@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js

@@ -29,6 +29,11 @@ var ChangeLogInitializer = class {
 	#sources;
 	#resultCounter = getOrCreateCounter("replica", "sqlite_change_log.init_compare_result", "Comparisons of the replica-derived stream initialization parameters against the Postgres-derived ones, by classification.");
 	#lastComparison;
+	/**
+	* The (restored) replica's resume point that the Postgres change log must
+	* cover, read once. See {@link #pgChangeLog}.
+	*/
+	#restoredReplica;
 	constructor(lc, { initFromPgChangeLog, initFromReplica }, sources) {
 		assert(initFromPgChangeLog || initFromReplica, `At least one of initFromPgChangeLog or initFromReplica must be true`);
 		this.#lc = lc.withContext("component", "change-log-initializer");
@@ -54,7 +59,7 @@ var ChangeLogInitializer = class {
 	*/
 	async initialize() {
 		this.#lastComparison = void 0;
-		const pg = this.#initFromPgChangeLog ? await this.#sources.pgChangeLog() : void 0;
+		const pg = this.#initFromPgChangeLog ? await this.#pgChangeLog() : void 0;
 		const replica = this.#initFromReplica ? this.#readReplica(pg === void 0) : void 0;
 		const reconciled = this.#sources.reconcileChangeLog(pg && {
 			resumeWatermark: pg.lastWatermark,
@@ -72,6 +77,44 @@ var ChangeLogInitializer = class {
 		};
 	}
 	/**
+	* Reads the Postgres change log's initialization parameters, first
+	* re-initializing it from the replica (taking ownership of it) if it may be
+	* missing transactions that the replication slot will not stream, e.g. if it
+	* is no longer maintained after running in high-availability mode. This is
+	* the Postgres counterpart of seeding a new SQLite change log from the
+	* replica.
+	*
+	* This only applies when the change log's head is behind the slot's position
+	* (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is
+	* supplied), and the change log is only re-initialized if its head is also
+	* behind the replica's. Otherwise, it contains every transaction before the
+	* slot's position: one that was acked after being stored would be in the
+	* change log, and one that was acked after being backed up would be in the
+	* replica, which the change log's head covers.
+	*
+	* The replica is read only once, i.e. as it was restored from the backup,
+	* before any stream is started. The live replica is no reference: it is
+	* caught up from the forwarded changes, which can be ahead of what has been
+	* stored, e.g. by the change log's previous owner. Against that fixed
+	* resume point, re-initialization is idempotent: afterwards, the change log's
+	* head (which only advances) covers it, so subsequent stream connections
+	* leave it untouched.
+	*/
+	async #pgChangeLog() {
+		const pg = await this.#sources.pgChangeLog();
+		const reinitialize = this.#sources.reinitializePgChangeLog;
+		if (!reinitialize) return pg;
+		this.#restoredReplica ??= this.#sources.replica();
+		const { lastWatermark, cookies } = this.#restoredReplica;
+		if (pg.lastWatermark >= lastWatermark) return pg;
+		const result = await reinitialize({
+			watermark: lastWatermark,
+			cookies
+		});
+		this.#lc.info?.(`Postgres change log at ${pg.lastWatermark} is behind the replica at ${lastWatermark}: ${result}`);
+		return this.#sources.pgChangeLog();
+	}
+	/**
 	* Compares both sources and records the result. Postgres remains
 	* authoritative when both sources are enabled.
 	*/

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#readReplica","#compare","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#sources.pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkJA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKE,SAAS,YAAY,IAChC,KAAA;EAGJ,MAAM,UAAU,KAAKD,mBACjB,KAAKI,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKH,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKE,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKC,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKR,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKK,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKN,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKM,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKT,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}
+{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#pgChangeLog","#readReplica","#compare","#restoredReplica","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n  /**\n   * Re-initializes the Postgres change log from the replica's resume point\n   * (taking ownership of it) if it does not contain all of the replica's\n   * changes, re-checking under lock. Typically\n   * `Storer.reinitializeFromReplica()`.\n   *\n   * Supplied only when the change log's head is behind the position of the\n   * replication slot from which replication resumes, i.e. when it may be\n   * missing transactions that the slot will not stream. Otherwise, a replica\n   * that is ahead of the change log (as changes are forwarded before they are\n   * stored) is caught up by the change stream, which resumes from the change\n   * log's head.\n   */\n  readonly reinitializePgChangeLog?:\n    | ((params: {\n        watermark: string;\n        cookies: CookieSet;\n      }) => Promise<'reinitialized' | 'maintained'>)\n    | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  /**\n   * The (restored) replica's resume point that the Postgres change log must\n   * cover, read once. See {@link #pgChangeLog}.\n   */\n  #restoredReplica: InitializationParameters | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Reads the Postgres change log's initialization parameters, first\n   * re-initializing it from the replica (taking ownership of it) if it may be\n   * missing transactions that the replication slot will not stream, e.g. if it\n   * is no longer maintained after running in high-availability mode. This is\n   * the Postgres counterpart of seeding a new SQLite change log from the\n   * replica.\n   *\n   * This only applies when the change log's head is behind the slot's position\n   * (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is\n   * supplied), and the change log is only re-initialized if its head is also\n   * behind the replica's. Otherwise, it contains every transaction before the\n   * slot's position: one that was acked after being stored would be in the\n   * change log, and one that was acked after being backed up would be in the\n   * replica, which the change log's head covers.\n   *\n   * The replica is read only once, i.e. as it was restored from the backup,\n   * before any stream is started. The live replica is no reference: it is\n   * caught up from the forwarded changes, which can be ahead of what has been\n   * stored, e.g. by the change log's previous owner. Against that fixed\n   * resume point, re-initialization is idempotent: afterwards, the change log's\n   * head (which only advances) covers it, so subsequent stream connections\n   * leave it untouched.\n   */\n  async #pgChangeLog(): Promise<InitializationParameters> {\n    const pg = await this.#sources.pgChangeLog();\n    const reinitialize = this.#sources.reinitializePgChangeLog;\n    if (!reinitialize) {\n      return pg;\n    }\n    this.#restoredReplica ??= this.#sources.replica();\n    const {lastWatermark, cookies} = this.#restoredReplica;\n    if (pg.lastWatermark >= lastWatermark) {\n      return pg;\n    }\n    const result = await reinitialize({watermark: lastWatermark, cookies});\n    this.#lc.info?.(\n      `Postgres change log at ${pg.lastWatermark} is behind the replica at ` +\n        `${lastWatermark}: ${result}`,\n    );\n    // Either way, the change log has changed since it was read.\n    return this.#sources.pgChangeLog();\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAqKA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;;;;;CAMA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKK,aAAa,IACxB,KAAA;EAGJ,MAAM,UAAU,KAAKJ,mBACjB,KAAKK,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKJ,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKG,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;;;;;;;;;;;;;;;;;;;;;CA0BA,MAAMF,eAAkD;EACtD,MAAM,KAAK,MAAM,KAAKH,SAAS,YAAY;EAC3C,MAAM,eAAe,KAAKA,SAAS;EACnC,IAAI,CAAC,cACH,OAAO;EAET,KAAKM,qBAAqB,KAAKN,SAAS,QAAQ;EAChD,MAAM,EAAC,eAAe,YAAW,KAAKM;EACtC,IAAI,GAAG,iBAAiB,eACtB,OAAO;EAET,MAAM,SAAS,MAAM,aAAa;GAAC,WAAW;GAAe;EAAO,CAAC;EACrE,KAAKT,IAAI,OACP,0BAA0B,GAAG,cAAc,4BACtC,cAAc,IAAI,QACzB;EAEA,OAAO,KAAKG,SAAS,YAAY;CACnC;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKO,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKO,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKR,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKQ,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKX,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -1,26 +1,74 @@
 import type { IncomingMessage } from 'node:http';
 import type { LogContext } from '@rocicorp/logger';
+import type { NormalizedZeroConfig } from '../../config/normalize.ts';
 import { type Worker } from '../../types/processes.ts';
 import { type ShardID } from '../../types/shards.ts';
-import { type Source } from '../../types/streams.ts';
+import { type Sized, type Source } from '../../types/streams.ts';
+import { Subscription } from '../../types/subscription.ts';
 import { HttpService, type Options as HttpOptions } from '../http-service.ts';
-import { type ChangeStreamer, type ChangeStreamerService, type SizedDownstream, type SubscriberContext } from './change-streamer.ts';
-import { type SnapshotMessage } from './snapshot.ts';
+import { type ChangeStreamer, type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
+import { type LocalReplica, type SnapshotStatus, type SubscribeDownstream, type SubscribeUpstream } from './subscribe.ts';
 type Options = HttpOptions & {
     startupDelayMs: number;
+    config?: Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'> | undefined;
+    getProfileWorker?: (() => Promise<Worker>) | undefined;
 };
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
     readonly id = "change-streamer-http-server";
     constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamerService);
     protected _onStart(): void;
+    /**
+     * Signals (drainable) subscribers to switch over to another
+     * replication-manager, resolving when they have disconnected. Subscribers
+     * continue to be served in the meantime.
+     */
+    drain(): Promise<void>;
     protected _onStop(): Promise<void>;
 }
-export declare class ChangeStreamerHttpClient implements ChangeStreamer {
+export interface SnapshotReserver {
+    reserveSnapshot(taskID: string, localReplica?: LocalReplica | undefined): Promise<{
+        reserved: SnapshotStatus;
+        followup: ReservationFollowup;
+    }>;
+}
+export interface SubscriptionStarter {
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
+}
+export interface ReservationFollowup extends SubscriptionStarter {
+    /**
+     * A signal to indicate that the followup is no longer valid (i.e. the
+     * connection to the reserving task has been severed).
+     */
+    readonly signal: AbortSignal;
+    /**
+     * Abandons the reservation without subscribing, closing the connection that
+     * was held open to pin the change-log floor. Used when the caller cannot
+     * proceed to `subscribe()` (e.g. process shutdown while a restore is in
+     * progress) and must not leave the reservation's connection open forever.
+     */
+    cancel(reason?: Error): void;
+}
+export declare class ChangeStreamerHttpClient implements ChangeStreamer, SnapshotReserver, SubscriptionStarter {
     #private;
     constructor(lc: LogContext, shardID: ShardID, changeDB: string, changeStreamerURI: string | undefined);
-    reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>>;
-    subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;
+    reserveSnapshot(taskID: string, localReplica?: LocalReplica | undefined): Promise<{
+        reserved: SnapshotStatus;
+        followup: ReservationFollowup;
+    }>;
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
+    /**
+     * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the
+     * inbound change/reservation stream and an `outbound` sink into which the
+     * caller pushes the `reserve-snapshot` / `start-subscription` control
+     * messages that drive the connection's phases. The single connection spans
+     * the reservation, backup restore, and subscription, so the reserving
+     * replication-manager is the one that serves the subscription.
+     */
+    connect(taskID: string): Promise<{
+        instream: Source<Sized<SubscribeDownstream>>;
+        outbound: Subscription<SubscribeUpstream>;
+    }>;
 }
 type RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;
 export declare function getSubscriberContext(req: RequestHeaders): SubscriberContext;

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuBtC;IAyED,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,qBAAa,wBAAyB,YAAW,cAAc;;IAM7D,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAOtE;IAEK,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAOxE;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAkB3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,KAAK,EACV,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAIzD,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,KAAK,YAAY,EAIjB,KAAK,cAAc,EAEnB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AAmBxB,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EACH,IAAI,CAAC,oBAAoB,EAAE,eAAe,GAAG,kBAAkB,CAAC,GAChE,SAAS,CAAC;IACd,gBAAgB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuCtC;IAmLD,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3B;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CACb,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,GACtC,OAAO,CAAC;QACT,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,wBACX,YAAW,cAAc,EAAE,gBAAgB,EAAE,mBAAmB;;IAOhE,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CACnB,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,GACtC,OAAO,CAAC;QACT,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CA2BD;IAEK,SAAS,CACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAI7C;IAED;;;;;;;OAOG;IAEG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC7C,QAAQ,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;KAC3C,CAAC,CAuBD;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAiB3E"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -1,24 +1,29 @@
-import { assert } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
 import "../../types/processes.js";
 import { pgClient } from "../../types/pg.js";
 import { HttpService } from "../http-service.js";
+import { URLParams } from "../../types/url-params.js";
+import { handleProfzRequest } from "../profz.js";
 import { PROTOCOL_ERROR, closeWithError } from "../../types/ws.js";
 import { installWebSocketReceiver } from "../../types/websocket-handoff.js";
-import { streamIn, streamInWithSize, streamOut, streamOutStringified } from "../../types/streams.js";
+import { Subscription } from "../../types/subscription.js";
+import { readHead, streamInternal, streamInternalStringified, streamInternalWithSize } from "../../types/streams.js";
 import { discoverChangeStreamerAddress } from "./schema/tables.js";
-import { URLParams } from "../../types/url-params.js";
-import { downstreamSchema } from "./change-streamer.js";
-import { snapshotMessageSchema } from "./snapshot.js";
+import "./change-streamer.js";
+import "./snapshot-message.js";
+import { subscribeDownstreamSchema, subscribeUpstreamSchema } from "./subscribe.js";
 import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
+var MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;
 var SNAPSHOT_PATH_PATTERN = "/replication/:version/snapshot";
 var CHANGES_PATH_PATTERN = "/replication/:version/changes";
-var PATH_REGEX = /\/replication\/v(?<version>\d+)\/(changes|snapshot)$/;
-var SNAPSHOT_PATH = `/replication/v6/snapshot`;
-var CHANGES_PATH = `/replication/v6/changes`;
+var SUBSCRIBE_PATH_PATTERN = "/replication/:version/subscribe";
+var PATH_REGEX = /\/replication\/v(?<version>\d+)\/(changes|snapshot|subscribe)$/;
+var SUBSCRIBE_PATH = `/replication/v8/subscribe`;
+var CONNECT_TIMEOUT_MS = 1e4;
 var ChangeStreamerHttpServer = class extends HttpService {
 	id = "change-streamer-http-server";
 	#lc;
@@ -27,8 +32,13 @@ var ChangeStreamerHttpServer = class extends HttpService {
 	constructor(lc, opts, parent, changeStreamer) {
 		super("change-streamer-http-server", lc, opts, async (fastify) => {
 			await fastify.register(websocket);
-			fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#subscribe);
+			fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#changes);
 			fastify.get(SNAPSHOT_PATH_PATTERN, { websocket: true }, this.#reserveSnapshot);
+			fastify.get(SUBSCRIBE_PATH_PATTERN, { websocket: true }, this.#subscribe);
+			fastify.get("/profz", (req, res) => handleProfzRequest(lc, opts.config ?? {
+				adminPassword: void 0,
+				operatorPassword: void 0
+			}, req, res, opts.getProfileWorker, void 0, "change-streamer"));
 			installWebSocketReceiver(lc, fastify.websocketServer, this.#receiveWebsocket, parent);
 		});
 		this.#lc = lc;
@@ -38,7 +48,8 @@ var ChangeStreamerHttpServer = class extends HttpService {
 	#receiveWebsocket = (ws, action, msg) => {
 		switch (action) {
 			case "snapshot": return this.#reserveSnapshot(ws, msg);
-			case "changes": return this.#subscribe(ws, msg);
+			case "changes": return this.#changes(ws, msg);
+			case "subscribe": return this.#subscribe(ws, msg);
 			default:
 				closeWithError(this._lc, ws, `invalid action "${action}" received in handoff`);
 				return;
@@ -52,38 +63,98 @@ var ChangeStreamerHttpServer = class extends HttpService {
 			const taskID = url.searchParams.get("taskID");
 			if (!taskID) throw new Error("Missing taskID in snapshot request");
 			const downstream = await this.#changeStreamer.startSnapshotReservation(taskID);
-			streamOut(this._lc, downstream, ws);
+			streamInternal(this._lc, ws, void 0, downstream);
 		} catch (err) {
 			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
 		}
 	};
-	#subscribe = async (ws, req) => {
+	#changes = async (ws, req) => {
 		try {
 			const ctx = getSubscriberContext(req);
 			if (ctx.mode === "serving") this.#ensureChangeStreamerStarted("incoming subscription");
 			const downstream = await this.#changeStreamer.subscribe(ctx);
-			streamOutStringified(this._lc, downstream, ws);
+			streamInternalStringified(this._lc, ws, void 0, downstream, { batched: ctx.wsBatched });
+		} catch (err) {
+			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
+		}
+	};
+	/**
+	* The merged v7 `/subscribe` handler. A single bidirectional connection
+	* carries, in sequence: an optional reservation phase (`reserve-snapshot` →
+	* `reserved`) and then the subscription phase (`start-subscription` → change
+	* stream). This ensures that the subscriber subscribes to the same task from
+	* which it restored the backup.
+	*/
+	#subscribe = async (ws, req) => {
+		let reservation;
+		const outbound = Subscription.create();
+		try {
+			const protocolVersion = checkProtocolVersion(new URL(req.url ?? "", req.headers.origin ?? "http://localhost").pathname);
+			if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) throw new Error(`the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} (client is at v${protocolVersion})`);
+			const instream = await streamInternalStringified(this._lc, ws, subscribeUpstreamSchema, outbound, { batched: true });
+			for await (const msg of instream) if (msg[0] === "reserve-snapshot") {
+				const { taskID, localReplica } = msg[1];
+				this.#ensureChangeStreamerStarted("incoming snapshot reservation");
+				reservation = await this.#changeStreamer.startSnapshotReservation(taskID, { localReplica });
+				this.#forwardReservation(reservation, outbound);
+			} else if (msg[0] === "start-subscription") {
+				const ctx = {
+					...msg[1],
+					protocolVersion,
+					wsBatched: true
+				};
+				if (ctx.mode === "serving") this.#ensureChangeStreamerStarted("incoming subscription");
+				reservation = void 0;
+				await this.#changeStreamer.subscribe(ctx, outbound);
+			}
 		} catch (err) {
 			closeWithError(this._lc, ws, err, PROTOCOL_ERROR);
+		} finally {
+			reservation?.cancel();
 		}
 	};
+	/**
+	* Forwards a snapshot reservation's single `['status', ...]` message onto
+	* the merged connection's `outbound` sink as a `['reserved', ...]` message.
+	* The loop then blocks on the held-open reservation until it is cancelled,
+	* by `subscribe()` at the subscription handoff, or by the reservation-phase
+	* teardown on disconnect.
+	*/
+	async #forwardReservation(reservation, outbound) {
+		try {
+			for await (const [, status] of reservation) outbound.push(BigIntJSON.stringify(["reserved", {
+				...status,
+				tag: "snapshot"
+			}]));
+		} catch (e) {
+			this._lc.warn?.(`error forwarding snapshot reservation`, e);
+			outbound.fail(e instanceof Error ? e : new Error(String(e)));
+		}
+	}
 	#changeStreamerStarted = false;
 	#ensureChangeStreamerStarted(reason) {
 		if (!this.#changeStreamerStarted && this._state.shouldRun()) {
-			this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);
+			this.#lc.info?.(`starting ChangeStreamerService ${reason ? `(${reason})` : ""}`);
 			this.#changeStreamer.run().catch((e) => this.#lc.warn?.(`ChangeStreamerService ended with error`, e)).finally(() => this.stop());
 			this.#changeStreamerStarted = true;
 		}
 	}
 	_onStart() {
 		const { startupDelayMs, readinessGate = promiseVoid } = this.#opts;
-		readinessGate.then(() => {
-			if (startupDelayMs > 0) {
-				this.#lc.info?.(`waiting ${startupDelayMs}ms before taking over the change log`);
-				this._state.setTimeout(() => this.#ensureChangeStreamerStarted(`startup delay elapsed (${startupDelayMs} ms)`), startupDelayMs);
-			} else this.#ensureChangeStreamerStarted("no startup delay configured");
+		if (startupDelayMs === 0) this.#ensureChangeStreamerStarted();
+		else readinessGate.then(() => {
+			this.#lc.info?.(`waiting ${startupDelayMs}ms before taking over the change log`);
+			this._state.setTimeout(() => this.#ensureChangeStreamerStarted(`startup delay elapsed (${startupDelayMs} ms)`), startupDelayMs);
 		});
 	}
+	/**
+	* Signals (drainable) subscribers to switch over to another
+	* replication-manager, resolving when they have disconnected. Subscribers
+	* continue to be served in the meantime.
+	*/
+	async drain() {
+		if (this.#changeStreamerStarted && this.#changeStreamer.drain) await this.#changeStreamer.drain();
+	}
 	async _onStop() {
 		if (this.#changeStreamerStarted) await this.#changeStreamer.stop();
 	}
@@ -113,13 +184,49 @@ var ChangeStreamerHttpClient = class {
 		this.#lc.info?.(`connecting to change-streamer@${uri}`);
 		return uri;
 	}
-	async reserveSnapshot(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
-		return streamIn(this.#lc, ws, snapshotMessageSchema);
+	async reserveSnapshot(taskID, localReplica) {
+		const { instream, outbound } = await this.connect(taskID);
+		outbound.push(["reserve-snapshot", {
+			taskID,
+			...localReplica ? { localReplica } : {}
+		}]);
+		const { head, rest } = readHead(instream);
+		const item = await head.next();
+		if (!item.value) throw new Error("stream closed without snapshot reservation");
+		const { data: msg } = item.value;
+		if (msg[0] !== "reserved") throw new Error(`expected "reserved" message but got ${msg[0]}`);
+		return {
+			reserved: msg[1],
+			followup: {
+				subscribe: (ctx) => {
+					outbound.push(["start-subscription", toStartSubscriptionContext(ctx)]);
+					return Promise.resolve(rest);
+				},
+				signal: outbound.signal,
+				cancel: (reason) => outbound.cancel(reason)
+			}
+		};
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
-		return streamInWithSize(this.#lc, ws, downstreamSchema);
+		const { instream, outbound } = await this.connect(ctx.taskID);
+		outbound.push(["start-subscription", toStartSubscriptionContext(ctx)]);
+		return instream;
+	}
+	/**
+	* Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the
+	* inbound change/reservation stream and an `outbound` sink into which the
+	* caller pushes the `reserve-snapshot` / `start-subscription` control
+	* messages that drive the connection's phases. The single connection spans
+	* the reservation, backup restore, and subscription, so the reserving
+	* replication-manager is the one that serves the subscription.
+	*/
+	async connect(taskID) {
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
+		const outbound = Subscription.create();
+		return {
+			instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),
+			outbound
+		};
 	}
 };
 function getSubscriberContext(req) {
@@ -133,25 +240,28 @@ function getSubscriberContext(req) {
 		mode: params.get("mode", false) === "backup" ? "backup" : "serving",
 		replicaVersion: params.get("replicaVersion", true),
 		watermark: params.get("watermark", true),
-		initial: params.getBoolean("initial"),
-		logsChangeStream: params.getBoolean("logsChangeStream")
+		wsBatched: params.getBoolean("wsBatched")
 	};
 }
 function checkProtocolVersion(pathname) {
 	const match = PATH_REGEX.exec(pathname);
 	if (!match) throw new Error(`invalid path: ${pathname}`);
 	const v = Number(match.groups?.version);
-	if (Number.isNaN(v) || v > 6 || v < MIN_SUPPORTED_PROTOCOL_VERSION) throw new Error(`Cannot service client at protocol v${v}. Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v6]`);
+	if (Number.isNaN(v) || v > 8 || v < MIN_SUPPORTED_PROTOCOL_VERSION) throw new Error(`Cannot service client at protocol v${v}. Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v8]`);
 	return v;
 }
-function getParams(ctx) {
-	const { protocolVersion, ...stringParams } = ctx;
-	assert(protocolVersion === 6, `replicator should be setting protocolVersion to 6`);
-	return new URLSearchParams({
-		...stringParams,
-		initial: ctx.initial ? "true" : "false",
-		logsChangeStream: ctx.logsChangeStream ? "true" : "false"
-	});
+function toStartSubscriptionContext(ctx) {
+	const { taskID, id, mode, replicaVersion, watermark, backfills, drainable, switchover } = ctx;
+	return {
+		taskID,
+		id,
+		mode,
+		replicaVersion,
+		watermark,
+		backfills,
+		...drainable !== void 0 ? { drainable } : {},
+		...switchover !== void 0 ? { switchover } : {}
+	};
 }
 //#endregion
 export { ChangeStreamerHttpClient, ChangeStreamerHttpServer };