@rocicorp/zero 1.11.0-canary.26 → 1.11.0-canary.28

package/out/zero-cache/src/scripts/permissions.js

@@ -10,7 +10,7 @@ import { fileURLToPath } from "node:url";
 //#region ../zero-cache/src/scripts/permissions.ts
 var deployPermissionsOptions = {
 	schema: { path: {
-		type: valita_exports.string().default("schema.ts"),
+		type: valita_exports.string().optional(() => "schema.ts"),
 		desc: ["Relative path to the file containing the schema definition."],
 		alias: "p"
 	} },
@@ -30,7 +30,7 @@ var deployPermissionsOptions = {
 			desc: [`Outputs the permissions to a file with the requested {bold output-format}.`]
 		},
 		format: {
-			type: literalUnion("sql", "json", "pretty").default("sql"),
+			type: literalUnion("sql", "json", "pretty").optional(() => "sql"),
 			desc: [
 				`The desired format of the output file.`,
 				``,
@@ -43,7 +43,7 @@ var deployPermissionsOptions = {
 		}
 	},
 	force: {
-		type: valita_exports.boolean().default(false),
+		type: valita_exports.boolean().optional(() => false),
 		desc: [`Deploy to upstream without validation. Use at your own risk.`],
 		alias: "f"
 	}

package/out/zero-cache/src/scripts/permissions.js.map

@@ -1 +1 @@
-{"version":3,"file":"permissions.js","names":[],"sources":["../../../../../zero-cache/src/scripts/permissions.ts"],"sourcesContent":["import {existsSync} from 'node:fs';\nimport {basename, dirname, join, relative, resolve, sep} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {logOptions} from '../../../otel/src/log-options.ts';\nimport {colorConsole} from '../../../shared/src/logging.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {\n  permissionsConfigSchema,\n  type PermissionsConfig,\n} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {isSchemaConfig} from '../../../zero-schema/src/schema-config.ts';\nimport type {Schema} from '../../../zero-types/src/schema.ts';\nimport {appOptions, shardOptions, zeroOptions} from '../config/zero-config.ts';\n\nexport const deployPermissionsOptions = {\n  schema: {\n    path: {\n      type: v.string().default('schema.ts'),\n      desc: ['Relative path to the file containing the schema definition.'],\n      alias: 'p',\n    },\n  },\n\n  upstream: {\n    db: {\n      type: v.string().optional(),\n      desc: [\n        `The upstream Postgres database to deploy permissions to.`,\n        `This is ignored if an {bold output-file} is specified.`,\n      ],\n    },\n\n    type: zeroOptions.upstream.type,\n  },\n\n  app: {id: appOptions.id},\n\n  shard: shardOptions,\n\n  log: logOptions,\n\n  output: {\n    file: {\n      type: v.string().optional(),\n      desc: [\n        `Outputs the permissions to a file with the requested {bold output-format}.`,\n      ],\n    },\n\n    format: {\n      type: v.literalUnion('sql', 'json', 'pretty').default('sql'),\n      desc: [\n        `The desired format of the output file.`,\n        ``,\n        `A {bold sql} file can be executed via \"psql -f <file.sql>\", or \"\\\\\\\\i <file.sql>\"`,\n        `from within the psql console, or copied and pasted into a migration script.`,\n        ``,\n        `The {bold json} and {bold pretty} formats are available for non-pg backends`,\n        `and general debugging.`,\n      ],\n    },\n  },\n\n  force: {\n    type: v.boolean().default(false),\n    desc: [`Deploy to upstream without validation. Use at your own risk.`],\n    alias: 'f',\n  },\n};\n\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing: true,\n): Promise<{schema: Schema; permissions: PermissionsConfig} | undefined>;\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing?: false,\n): Promise<{schema: Schema; permissions: PermissionsConfig}>;\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing: boolean | undefined,\n): Promise<{schema: Schema; permissions: PermissionsConfig} | undefined> {\n  const typeModuleErrorMessage = () =>\n    `\\n\\nYou may need to add \\` \"type\": \"module\" \\` to the package.json file for ${schemaPath}.\\n`;\n\n  colorConsole.info(`Loading schema from ${schemaPath}`);\n\n  const dir = dirname(fileURLToPath(import.meta.url));\n  const absoluteSchemaPath = resolve(schemaPath);\n  const relativeDir = relative(dir, dirname(absoluteSchemaPath));\n  let relativePath =\n    relativeDir.length && relativeDir !== '.'\n      ? join(relativeDir, basename(absoluteSchemaPath))\n      : `.${sep}${basename(absoluteSchemaPath)}`;\n\n  // tsImport doesn't expect to receive slashes in the Windows format when running\n  // on Windows. They need to be converted to *nix format.\n  relativePath = relativePath.replace(/\\\\/g, '/');\n\n  if (!existsSync(absoluteSchemaPath)) {\n    if (allowMissing) {\n      return undefined;\n    }\n    colorConsole.error(`Schema file ${schemaPath} does not exist.`);\n    process.exit(1);\n  }\n\n  let module;\n  try {\n    module = await import(relativePath);\n  } catch (e) {\n    colorConsole.error(\n      `Failed to load zero schema from ${absoluteSchemaPath}` +\n        typeModuleErrorMessage(),\n    );\n    throw e;\n  }\n\n  if (!isSchemaConfig(module)) {\n    colorConsole.error(\n      `Schema file ${schemaPath} must export [schema].` +\n        typeModuleErrorMessage(),\n    );\n    process.exit(1);\n  }\n  try {\n    const schemaConfig = module;\n    const perms =\n      await (schemaConfig.permissions as unknown as Promise<unknown>);\n    const {schema} = schemaConfig;\n\n    if (perms) {\n      colorConsole.warn?.(\n        'Permissions are deprecated and will be removed in an upcoming release. See: https://zero.rocicorp.dev/docs/auth.',\n      );\n    }\n\n    return {\n      schema,\n      permissions: v.parse(perms ?? {}, permissionsConfigSchema),\n    };\n  } catch (e) {\n    colorConsole.error(`Failed to parse Permissions object`);\n    throw e;\n  }\n}\n"],"mappings":";;;;;;;;;;AAcA,IAAa,2BAA2B;CACtC,QAAQ,EACN,MAAM;EACJ,MAAM,eAAE,OAAO,EAAE,QAAQ,WAAW;EACpC,MAAM,CAAC,6DAA6D;EACpE,OAAO;CACT,EACF;CAEA,UAAU;EACR,IAAI;GACF,MAAM,eAAE,OAAO,EAAE,SAAS;GAC1B,MAAM,CACJ,4DACA,wDACF;EACF;EAEA,MAAM,YAAY,SAAS;CAC7B;CAEA,KAAK,EAAC,IAAI,WAAW,GAAE;CAEvB,OAAO;CAEP,KAAK;CAEL,QAAQ;EACN,MAAM;GACJ,MAAM,eAAE,OAAO,EAAE,SAAS;GAC1B,MAAM,CACJ,4EACF;EACF;EAEA,QAAQ;GACN,MAAM,aAAe,OAAO,QAAQ,QAAQ,EAAE,QAAQ,KAAK;GAC3D,MAAM;IACJ;IACA;IACA;IACA;IACA;IACA;IACA;GACF;EACF;CACF;CAEA,OAAO;EACL,MAAM,eAAE,QAAQ,EAAE,QAAQ,KAAK;EAC/B,MAAM,CAAC,8DAA8D;EACrE,OAAO;CACT;AACF;AAUA,eAAsB,yBACpB,YACA,cACuE;CACvE,MAAM,+BACJ,+EAA+E,WAAW;CAE5F,aAAa,KAAK,uBAAuB,YAAY;CAErD,MAAM,MAAM,QAAQ,cAAc,OAAO,KAAK,GAAG,CAAC;CAClD,MAAM,qBAAqB,QAAQ,UAAU;CAC7C,MAAM,cAAc,SAAS,KAAK,QAAQ,kBAAkB,CAAC;CAC7D,IAAI,eACF,YAAY,UAAU,gBAAgB,MAClC,KAAK,aAAa,SAAS,kBAAkB,CAAC,IAC9C,IAAI,MAAM,SAAS,kBAAkB;CAI3C,eAAe,aAAa,QAAQ,OAAO,GAAG;CAE9C,IAAI,CAAC,WAAW,kBAAkB,GAAG;EACnC,IAAI,cACF;EAEF,aAAa,MAAM,eAAe,WAAW,iBAAiB;EAC9D,QAAQ,KAAK,CAAC;CAChB;CAEA,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,OAAO;CACxB,SAAS,GAAG;EACV,aAAa,MACX,mCAAmC,uBACjC,uBAAuB,CAC3B;EACA,MAAM;CACR;CAEA,IAAI,CAAC,eAAe,MAAM,GAAG;EAC3B,aAAa,MACX,eAAe,WAAW,0BACxB,uBAAuB,CAC3B;EACA,QAAQ,KAAK,CAAC;CAChB;CACA,IAAI;EACF,MAAM,eAAe;EACrB,MAAM,QACJ,MAAO,aAAa;EACtB,MAAM,EAAC,WAAU;EAEjB,IAAI,OACF,aAAa,OACX,kHACF;EAGF,OAAO;GACL;GACA,aAAa,MAAQ,SAAS,CAAC,GAAG,uBAAuB;EAC3D;CACF,SAAS,GAAG;EACV,aAAa,MAAM,oCAAoC;EACvD,MAAM;CACR;AACF"}
+{"version":3,"file":"permissions.js","names":[],"sources":["../../../../../zero-cache/src/scripts/permissions.ts"],"sourcesContent":["import {existsSync} from 'node:fs';\nimport {basename, dirname, join, relative, resolve, sep} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {logOptions} from '../../../otel/src/log-options.ts';\nimport {colorConsole} from '../../../shared/src/logging.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {\n  permissionsConfigSchema,\n  type PermissionsConfig,\n} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {isSchemaConfig} from '../../../zero-schema/src/schema-config.ts';\nimport type {Schema} from '../../../zero-types/src/schema.ts';\nimport {appOptions, shardOptions, zeroOptions} from '../config/zero-config.ts';\n\nexport const deployPermissionsOptions = {\n  schema: {\n    path: {\n      type: v.string().optional(() => 'schema.ts'),\n      desc: ['Relative path to the file containing the schema definition.'],\n      alias: 'p',\n    },\n  },\n\n  upstream: {\n    db: {\n      type: v.string().optional(),\n      desc: [\n        `The upstream Postgres database to deploy permissions to.`,\n        `This is ignored if an {bold output-file} is specified.`,\n      ],\n    },\n\n    type: zeroOptions.upstream.type,\n  },\n\n  app: {id: appOptions.id},\n\n  shard: shardOptions,\n\n  log: logOptions,\n\n  output: {\n    file: {\n      type: v.string().optional(),\n      desc: [\n        `Outputs the permissions to a file with the requested {bold output-format}.`,\n      ],\n    },\n\n    format: {\n      type: v.literalUnion('sql', 'json', 'pretty').optional(() => 'sql'),\n      desc: [\n        `The desired format of the output file.`,\n        ``,\n        `A {bold sql} file can be executed via \"psql -f <file.sql>\", or \"\\\\\\\\i <file.sql>\"`,\n        `from within the psql console, or copied and pasted into a migration script.`,\n        ``,\n        `The {bold json} and {bold pretty} formats are available for non-pg backends`,\n        `and general debugging.`,\n      ],\n    },\n  },\n\n  force: {\n    type: v.boolean().optional(() => false),\n    desc: [`Deploy to upstream without validation. Use at your own risk.`],\n    alias: 'f',\n  },\n};\n\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing: true,\n): Promise<{schema: Schema; permissions: PermissionsConfig} | undefined>;\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing?: false,\n): Promise<{schema: Schema; permissions: PermissionsConfig}>;\nexport async function loadSchemaAndPermissions(\n  schemaPath: string,\n  allowMissing: boolean | undefined,\n): Promise<{schema: Schema; permissions: PermissionsConfig} | undefined> {\n  const typeModuleErrorMessage = () =>\n    `\\n\\nYou may need to add \\` \"type\": \"module\" \\` to the package.json file for ${schemaPath}.\\n`;\n\n  colorConsole.info(`Loading schema from ${schemaPath}`);\n\n  const dir = dirname(fileURLToPath(import.meta.url));\n  const absoluteSchemaPath = resolve(schemaPath);\n  const relativeDir = relative(dir, dirname(absoluteSchemaPath));\n  let relativePath =\n    relativeDir.length && relativeDir !== '.'\n      ? join(relativeDir, basename(absoluteSchemaPath))\n      : `.${sep}${basename(absoluteSchemaPath)}`;\n\n  // tsImport doesn't expect to receive slashes in the Windows format when running\n  // on Windows. They need to be converted to *nix format.\n  relativePath = relativePath.replace(/\\\\/g, '/');\n\n  if (!existsSync(absoluteSchemaPath)) {\n    if (allowMissing) {\n      return undefined;\n    }\n    colorConsole.error(`Schema file ${schemaPath} does not exist.`);\n    process.exit(1);\n  }\n\n  let module;\n  try {\n    module = await import(relativePath);\n  } catch (e) {\n    colorConsole.error(\n      `Failed to load zero schema from ${absoluteSchemaPath}` +\n        typeModuleErrorMessage(),\n    );\n    throw e;\n  }\n\n  if (!isSchemaConfig(module)) {\n    colorConsole.error(\n      `Schema file ${schemaPath} must export [schema].` +\n        typeModuleErrorMessage(),\n    );\n    process.exit(1);\n  }\n  try {\n    const schemaConfig = module;\n    const perms =\n      await (schemaConfig.permissions as unknown as Promise<unknown>);\n    const {schema} = schemaConfig;\n\n    if (perms) {\n      colorConsole.warn?.(\n        'Permissions are deprecated and will be removed in an upcoming release. See: https://zero.rocicorp.dev/docs/auth.',\n      );\n    }\n\n    return {\n      schema,\n      permissions: v.parse(perms ?? {}, permissionsConfigSchema),\n    };\n  } catch (e) {\n    colorConsole.error(`Failed to parse Permissions object`);\n    throw e;\n  }\n}\n"],"mappings":";;;;;;;;;;AAcA,IAAa,2BAA2B;CACtC,QAAQ,EACN,MAAM;EACJ,MAAM,eAAE,OAAO,EAAE,eAAe,WAAW;EAC3C,MAAM,CAAC,6DAA6D;EACpE,OAAO;CACT,EACF;CAEA,UAAU;EACR,IAAI;GACF,MAAM,eAAE,OAAO,EAAE,SAAS;GAC1B,MAAM,CACJ,4DACA,wDACF;EACF;EAEA,MAAM,YAAY,SAAS;CAC7B;CAEA,KAAK,EAAC,IAAI,WAAW,GAAE;CAEvB,OAAO;CAEP,KAAK;CAEL,QAAQ;EACN,MAAM;GACJ,MAAM,eAAE,OAAO,EAAE,SAAS;GAC1B,MAAM,CACJ,4EACF;EACF;EAEA,QAAQ;GACN,MAAM,aAAe,OAAO,QAAQ,QAAQ,EAAE,eAAe,KAAK;GAClE,MAAM;IACJ;IACA;IACA;IACA;IACA;IACA;IACA;GACF;EACF;CACF;CAEA,OAAO;EACL,MAAM,eAAE,QAAQ,EAAE,eAAe,KAAK;EACtC,MAAM,CAAC,8DAA8D;EACrE,OAAO;CACT;AACF;AAUA,eAAsB,yBACpB,YACA,cACuE;CACvE,MAAM,+BACJ,+EAA+E,WAAW;CAE5F,aAAa,KAAK,uBAAuB,YAAY;CAErD,MAAM,MAAM,QAAQ,cAAc,OAAO,KAAK,GAAG,CAAC;CAClD,MAAM,qBAAqB,QAAQ,UAAU;CAC7C,MAAM,cAAc,SAAS,KAAK,QAAQ,kBAAkB,CAAC;CAC7D,IAAI,eACF,YAAY,UAAU,gBAAgB,MAClC,KAAK,aAAa,SAAS,kBAAkB,CAAC,IAC9C,IAAI,MAAM,SAAS,kBAAkB;CAI3C,eAAe,aAAa,QAAQ,OAAO,GAAG;CAE9C,IAAI,CAAC,WAAW,kBAAkB,GAAG;EACnC,IAAI,cACF;EAEF,aAAa,MAAM,eAAe,WAAW,iBAAiB;EAC9D,QAAQ,KAAK,CAAC;CAChB;CAEA,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,OAAO;CACxB,SAAS,GAAG;EACV,aAAa,MACX,mCAAmC,uBACjC,uBAAuB,CAC3B;EACA,MAAM;CACR;CAEA,IAAI,CAAC,eAAe,MAAM,GAAG;EAC3B,aAAa,MACX,eAAe,WAAW,0BACxB,uBAAuB,CAC3B;EACA,QAAQ,KAAK,CAAC;CAChB;CACA,IAAI;EACF,MAAM,eAAe;EACrB,MAAM,QACJ,MAAO,aAAa;EACtB,MAAM,EAAC,WAAU;EAEjB,IAAI,OACF,aAAa,OACX,kHACF;EAGF,OAAO;GACL;GACA,aAAa,MAAQ,SAAS,CAAC,GAAG,uBAAuB;EAC3D;CACF,SAAS,GAAG;EACV,aAAa,MAAM,oCAAoC;EACvD,MAAM;CACR;AACF"}

package/out/zero-cache/src/server/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA4CA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAiaf"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA4CA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAmaf"}

package/out/zero-cache/src/server/change-streamer.js

@@ -92,7 +92,7 @@ async function runWorker(parent, env, ...argv) {
 		};
 		try {
 			const { pgReplicationEpoch: epoch, pgReplicationSlotPerReplica: slotPerReplica, pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs } = upstream;
-			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, walKeeper: initWalKeeper } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
+			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, walKeeper: initWalKeeper, pgChangeLogBehindSlot } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
 				...initialSync,
 				replicationSlotFailover: upstream.pgReplicationSlotFailover,
 				installPartialIndexTriggers: upstream.pgPartialIndexTriggers
@@ -110,6 +110,7 @@ async function runWorker(parent, env, ...argv) {
 				replicaFile: replica.file
 			} : null, purgeLock, autoReset ?? false, {
 				pgChangeLogEnabled,
+				pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,
 				backPressureLimitHeapProportion,
 				flowControlConsensusTimeoutProportion,
 				flowControlSlowSubscriberGracePeriodMs: flowControlSlowSubscriberGracePeriodSeconds > 0 ? flowControlSlowSubscriberGracePeriodSeconds * 1e3 : void 0,

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  let walKeeper: WalKeeper | undefined;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      pgChangeLogEnabled && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        walKeeper: initWalKeeper,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n      walKeeper = initWalKeeper;\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      // * The WAL keeper of a restore that prepared a backup lineage to\n      //   continue, which would otherwise hold the (deleted) replica open.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'supporting',\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n  // By the initial backup, litestream holds the restored WAL itself.\n  void backupReady.then(\n    () => walKeeper?.release('initial backup confirmed'),\n    () => walKeeper?.release('initial backup failed'),\n  );\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    walKeeper?.release('shutting down');\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,kBACT,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,UACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,cACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,YAAiB,WACT,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  let walKeeper: WalKeeper | undefined;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      pgChangeLogEnabled && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        walKeeper: initWalKeeper,\n        pgChangeLogBehindSlot,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n      walKeeper = initWalKeeper;\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      // * The WAL keeper of a restore that prepared a backup lineage to\n      //   continue, which would otherwise hold the (deleted) replica open.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'supporting',\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n  // By the initial backup, litestream holds the restored WAL itself.\n  void backupReady.then(\n    () => walKeeper?.release('initial backup confirmed'),\n    () => walKeeper?.release('initial backup failed'),\n  );\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    walKeeper?.release('shutting down');\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,eACX,0BACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,UACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,cACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,YAAiB,WACT,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts

@@ -10,6 +10,11 @@ export interface PurgeLock {
 }
 /** A PurgeLock that also constrains the replica that can be restored. */
 export interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {
+    /**
+     * The head of the change log when the lock was acquired, from which (or
+     * from past which) the change log is resumed.
+     */
+    readonly headWatermark: string;
 }
 /**
  * Acquires a purge lock on the PG change-log (if it is not empty).

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAC9D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;CAAG;AAE/E;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAE1B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAOF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,EAC3C,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,GAChC,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAuDtC"}
+{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAC9D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;IAC1E;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAE1B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAOF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,EAC3C,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,GAChC,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAuDtC"}

package/out/zero-cache/src/services/change-source/common/replica-restore.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {WalKeeper} from '../../litestream/wal-keeper.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n  release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n  slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  /**\n   * With the PG change-log enabled, purge-locks the change-log before the\n   * replica is restored, which constrains the replica to one from which the\n   * change-log can be resumed. The lock is acquired as late as possible (i.e.\n   * after a replication slot is created, if applicable) because, if the\n   * change-db is the upstream db, a transaction holding the lock would block\n   * the creation of a replication slot.\n   */\n  acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n  /**\n   * Resources acquired by the initialization (e.g. a claimed replication slot)\n   * are registered here to be released if the initialization attempt fails.\n   * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n   * here by the locker, so that it is released on failure.\n   */\n  cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n\n  /**\n   * Whether the PG change-log's head was behind the position of the\n   * replication slot from which replication resumes (see\n   * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n   * the restored replica if it does not contain the replica's changes.\n   */\n  pgChangeLogBehindSlot?: boolean | undefined;\n\n  /**\n   * Whether the litestream backup to `destinationBackupURL` starts a new\n   * (litestream v5) backup lineage, i.e. one that no local litestream state\n   * belongs to. Any such state (e.g. from a previous run of a restarted\n   * container, whose restore reused the existing replica) describes a\n   * different lineage and must be discarded before replicating, or litestream\n   * resolves its position from it and fails (and auto-recovers) on its first\n   * sync to the empty lineage.\n   *\n   * This is false when the backup continues a lineage from the local\n   * litestream state (see {@link ResumableBackup}).\n   */\n  newBackupLineage: boolean;\n\n  /**\n   * Set when the backup to `destinationBackupURL` continues from the local\n   * litestream state (see {@link ResumableBackup}). It is registered with the\n   * initialization's {@link InitCleanup}, which releases it if the attempt\n   * fails; otherwise the caller must release it once the litestream backup\n   * has started.\n   */\n  walKeeper?: WalKeeper | undefined;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n *   the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state, whose WAL `walKeeper` holds.\n */\nexport type ResumableBackup = {\n  backupPath: string;\n  walKeeper: WalKeeper;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists). Its WAL keeper is registered with `cleanup`.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  destination?: BackupDestination | undefined,\n  cleanup?: InitCleanup | undefined,\n): Promise<ResumableBackup | undefined> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      // Each attempt seeds a new subfolder, as a failed attempt can leave\n      // files in its subfolder.\n      const backupPath =\n        destination?.type === 'fork' ? String(Date.now()) : undefined;\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n          undefined,\n          backupPath\n            ? {\n                url: new URL(backupPath, destination?.baseURL).toString(),\n                cleanup,\n              }\n            : undefined,\n        );\n        result = attempt.result;\n        return backupPath && attempt.walKeeper\n          ? {backupPath, walKeeper: attempt.walKeeper}\n          : undefined;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          break;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n    return undefined;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AAsIA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;;;;;;;AAQ/B,eAAsB,eACpB,IACA,QACA,aACA,oBACA,aACA,SACsC;CACtC,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAAc;GAGzE,MAAM,aACJ,aAAa,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,IAAI,KAAA;GACtD,IAAI;IACF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,aACA,oBACA,uBACA,KAAA,GACA,aACI;KACE,KAAK,IAAI,IAAI,YAAY,aAAa,OAAO,EAAE,SAAS;KACxD;IACF,IACA,KAAA,CACN;IACA,SAAS,QAAQ;IACjB,OAAO,cAAc,QAAQ,YACzB;KAAC;KAAY,WAAW,QAAQ;IAAS,IACzC,KAAA;GACN,SAAS,GAAG;IACV,IAAI,eAAe,sBAAsB;KACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;KACA;IACF;IAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;GACpC;EACF;EACA;CACF,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}
+{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {WalKeeper} from '../../litestream/wal-keeper.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n  release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {\n  /**\n   * The head of the change log when the lock was acquired, from which (or\n   * from past which) the change log is resumed.\n   */\n  readonly headWatermark: string;\n}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n  slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  /**\n   * With the PG change-log enabled, purge-locks the change-log before the\n   * replica is restored, which constrains the replica to one from which the\n   * change-log can be resumed. The lock is acquired as late as possible (i.e.\n   * after a replication slot is created, if applicable) because, if the\n   * change-db is the upstream db, a transaction holding the lock would block\n   * the creation of a replication slot.\n   */\n  acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n  /**\n   * Resources acquired by the initialization (e.g. a claimed replication slot)\n   * are registered here to be released if the initialization attempt fails.\n   * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n   * here by the locker, so that it is released on failure.\n   */\n  cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n\n  /**\n   * Whether the PG change-log's head was behind the position of the\n   * replication slot from which replication resumes (see\n   * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n   * the restored replica if it does not contain the replica's changes.\n   */\n  pgChangeLogBehindSlot?: boolean | undefined;\n\n  /**\n   * Whether the litestream backup to `destinationBackupURL` starts a new\n   * (litestream v5) backup lineage, i.e. one that no local litestream state\n   * belongs to. Any such state (e.g. from a previous run of a restarted\n   * container, whose restore reused the existing replica) describes a\n   * different lineage and must be discarded before replicating, or litestream\n   * resolves its position from it and fails (and auto-recovers) on its first\n   * sync to the empty lineage.\n   *\n   * This is false when the backup continues a lineage from the local\n   * litestream state (see {@link ResumableBackup}).\n   */\n  newBackupLineage: boolean;\n\n  /**\n   * Set when the backup to `destinationBackupURL` continues from the local\n   * litestream state (see {@link ResumableBackup}). It is registered with the\n   * initialization's {@link InitCleanup}, which releases it if the attempt\n   * fails; otherwise the caller must release it once the litestream backup\n   * has started.\n   */\n  walKeeper?: WalKeeper | undefined;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n *   the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state, whose WAL `walKeeper` holds.\n */\nexport type ResumableBackup = {\n  backupPath: string;\n  walKeeper: WalKeeper;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists). Its WAL keeper is registered with `cleanup`.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  destination?: BackupDestination | undefined,\n  cleanup?: InitCleanup | undefined,\n): Promise<ResumableBackup | undefined> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      // Each attempt seeds a new subfolder, as a failed attempt can leave\n      // files in its subfolder.\n      const backupPath =\n        destination?.type === 'fork' ? String(Date.now()) : undefined;\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n          undefined,\n          backupPath\n            ? {\n                url: new URL(backupPath, destination?.baseURL).toString(),\n                cleanup,\n              }\n            : undefined,\n        );\n        result = attempt.result;\n        return backupPath && attempt.walKeeper\n          ? {backupPath, walKeeper: attempt.walKeeper}\n          : undefined;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          break;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n    return undefined;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AA4IA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;;;;;;;AAQ/B,eAAsB,eACpB,IACA,QACA,aACA,oBACA,aACA,SACsC;CACtC,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAAc;GAGzE,MAAM,aACJ,aAAa,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,IAAI,KAAA;GACtD,IAAI;IACF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,aACA,oBACA,uBACA,KAAA,GACA,aACI;KACE,KAAK,IAAI,IAAI,YAAY,aAAa,OAAO,EAAE,SAAS;KACxD;IACF,IACA,KAAA,CACN;IACA,SAAS,QAAQ;IACjB,OAAO,cAAc,QAAQ,YACzB;KAAC;KAAY,WAAW,QAAQ;IAAS,IACzC,KAAA;GACN,SAAS,GAAG;IACV,IAAI,eAAe,sBAAsB;KACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;KACA;IACF;IAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;GACpC;EACF;EACA;CACF,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -238,7 +238,8 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 	try {
 		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, {
 			slotName,
-			temporary: true
+			temporary: true,
+			diagnosticsDB: db
 		});
 		const { init, imported } = importSnapshot(snapshot);
 		const tx = new TransactionPool(lc, {

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randomCharacters} from '../../../../../shared/src/random-values.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport {resumePoint} from '../protocol/backfill-progress.ts';\nimport type {\n  BackfillCompleted,\n  BackfillProgressMark,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag' | 'progressMarks'>;\n\n/**\n * Determines how the progress of a backfill is marked, and from where it is\n * resumed.\n */\ntype Progress = {\n  /**\n   * The timeline of the progress marks. For plain tables, this is the table's\n   * `relfilenode`, which identifies the physical \"generation\" of the table's\n   * data (changed by heap rewrites like `VACUUM FULL`, `CLUSTER`, or\n   * `TRUNCATE`), within which `ctid`s are comparable.\n   */\n  timeline: string;\n\n  /**\n   * Whether the progress marks are resumable `ctid`s. Tables without\n   * (their own) storage, such as partitioned tables, are not resumable and\n   * are instead marked with the number of rows streamed, on a timeline that\n   * is unique to the run.\n   */\n  resumable: boolean;\n\n  /** The mark from which the backfill resumes, if any. */\n  start: BackfillProgressMark | undefined;\n};\n\n/**\n * Encodes a `ctid` text representation, e.g. `(123,4)`, as a\n * fixed-width string that sorts lexicographically in `ctid` order.\n */\nconst CTID_RE = /^\\((\\d+),(\\d+)\\)$/;\nconst CTID_MARK_RE = /^(\\d{10})\\.(\\d{5})$/;\n\nexport function ctidToProgressMark(ctid: string): string {\n  const match = CTID_RE.exec(ctid);\n  assert(match, () => `invalid ctid: ${ctid}`);\n  return `${match[1].padStart(10, '0')}.${match[2].padStart(5, '0')}`;\n}\n\n/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */\nexport function progressMarkToCtid(mark: string): string {\n  const match = CTID_MARK_RE.exec(mark);\n  assert(match, () => `invalid ctid progress mark: ${mark}`);\n  return `(${Number(match[1])},${Number(match[2])})`;\n}\n\nfunction progressMarks(\n  previous: BackfillProgressMark | undefined,\n  current: BackfillProgressMark,\n) {\n  return previous ? {previous, current} : {current};\n}\n\n/** The `ctid` of each row is appended as the last column of the download. */\nconst CTID_SELECT = 'ctid::text';\n\n/**\n * Settings that ensure that the COPY scans the table in physical (i.e.\n * `ctid`) order, which is necessary for the progress marks of a resumable\n * backfill to be meaningful. (Monotonicity is nonetheless verified as rows\n * are streamed.)\n *\n * * Synchronized seqscans can start in the middle of the table.\n * * Parallel scans interleave the rows of different workers.\n * * Index scans follow index order. (A publication row filter could\n *   otherwise be satisfied with an index.)\n */\nconst PHYSICAL_ORDER_SETTINGS = [\n  'synchronize_seqscans = off',\n  'max_parallel_workers_per_gather = 0',\n  'enable_indexscan = off',\n  'enable_indexonlyscan = off',\n  'enable_bitmapscan = off',\n];\n\nfunction rowCountMark(rows: number): string {\n  return String(rows).padStart(16, '0');\n}\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n\n  /**\n   * @visibleForTesting\n   * Called after the snapshot transaction is opened, before the table is\n   * locked, e.g. to run a concurrent heap rewrite in between.\n   */\n  afterSnapshotForTesting?: (() => Promise<void>) | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {\n    flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE,\n    textCopy = false,\n    afterSnapshotForTesting,\n  } = opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    await afterSnapshotForTesting?.();\n    const {tableSpec, backfill, progress} = await validateSchema(\n      lc,\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    // Resumable backfills are streamed in physical (i.e. ctid) order, which\n    // is the natural order of a sequential or TID range scan (as enforced by\n    // the settings in `stream()`). An ORDER BY is not used, as Postgres does\n    // not consider either scan to be ordered, and would unnecessarily sort\n    // the table.\n    const order =\n      progress.resumable && progress.start\n        ? {\n            after: /*sql*/ `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid`,\n          }\n        : undefined;\n    const stmts = makeDownloadStatements(\n      tableSpec,\n      cols,\n      undefined,\n      undefined,\n      [...cols.map(col => id(col)), CTID_SELECT],\n      order,\n    );\n    if (progress.start) {\n      lc.info?.(\n        `resuming backfill from ${progress.start.progressMark} ` +\n          `(timeline ${progress.timeline})`,\n      );\n    }\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        (text: string) => text,\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT],\n        order,\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        (buf: Buffer) => buf.toString('utf8'), // ctid::text\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {timeline, resumable, start: startMark}: Progress,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  ctidDecoder: (field: T) => string,\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    // SET LOCAL applies to the (snapshot) transaction of this read task.\n    for (const setting of PHYSICAL_ORDER_SETTINGS) {\n      await sql.unsafe(`SET LOCAL ${setting}`);\n    }\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // The mark of the last row that was streamed, and that of the last row of\n  // the previous message.\n  let current: BackfillProgressMark | undefined = startMark;\n  let previous: BackfillProgressMark | undefined = startMark;\n  const markOf = (progressMark: string) => ({progressMark, timeline});\n\n  // The number of columns of each row, followed by the ctid.\n  const numCols = decoders.length + 1;\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      if (col < decoders.length) {\n        row[col] = field === null ? null : decoders[col](field);\n      } else {\n        const mark = resumable\n          ? ctidToProgressMark(ctidDecoder(field as T))\n          : rowCountMark(status.rows + 1);\n        if (current !== undefined && mark <= current.progressMark) {\n          // This should not happen given the PHYSICAL_ORDER_SETTINGS, but\n          // the correctness of the progress marks depends on it.\n          throw new Error(\n            `backfill rows are not in ctid order ` +\n              `(${mark} after ${current.progressMark})`,\n          );\n        }\n        current = markOf(mark);\n      }\n\n      if (++col === numCols) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {\n      yield {\n        message: {\n          tag: 'backfill',\n          ...backfill,\n          rowValues,\n          status,\n          progressMarks: progressMarks(previous, must(current)),\n        },\n        byteSize: bufferedBytes,\n      };\n      previous = current;\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {\n        tag: 'backfill',\n        ...backfill,\n        rowValues,\n        status,\n        progressMarks: progressMarks(previous, must(current)),\n      },\n      byteSize: bufferedBytes,\n    };\n    previous = current;\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {\n      tag: 'backfill-completed',\n      ...backfill,\n      status,\n      // `previous` is only unset for a backfill from scratch with no rows.\n      progressMarks: previous ? {previous} : {},\n    },\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  lc: LogContext,\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n  progress: Progress;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns).map(([col, {id}]) => [col, id] as const),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    const progress = await getProgress(lc, sql, spec.oid, bf);\n    return {tableSpec: spec, backfill, progress};\n  });\n}\n\n/**\n * Determines the timeline of the table's progress marks (as of the snapshot),\n * and whether the backfill can be resumed from the requested progress.\n *\n * The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest\n * of the transaction) so that its storage cannot be rewritten (by `VACUUM\n * FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which\n * require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note\n * that `pg_relation_filenode()` reflects the table's current storage (only\n * refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at\n * the transaction snapshot. If they differ, the storage was rewritten after the\n * snapshot, and the COPY would scan the new storage with the old snapshot,\n * which is not MVCC-safe for all rewrites (e.g. the table appears empty after\n * a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.\n */\nasync function getProgress(\n  lc: LogContext,\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<Progress> {\n  const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;\n  await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);\n  const [{lockedOID, relkind, snapshotFilenode, filenode}] = await sql<\n    {\n      lockedOID: number | null;\n      relkind: string;\n      snapshotFilenode: string;\n      filenode: string | null;\n    }[]\n  >`\n    SELECT to_regclass(${table})::oid::int8 AS \"lockedOID\",\n           relkind,\n           relfilenode::text AS \"snapshotFilenode\",\n           pg_relation_filenode(oid)::text AS filenode\n      FROM pg_class WHERE oid = ${relationOID}`;\n  if (Number(lockedOID) !== relationOID) {\n    // The (current) table with the name is not the one at the snapshot.\n    throw new SchemaIncompatibilityError(\n      bf,\n      `Table has been renamed or replaced`,\n    );\n  }\n  if (relkind !== 'r' || filenode === null) {\n    // Not a plain table (e.g. a partitioned table), for which ctids are not\n    // unique. Such backfills are not resumable, which is achieved by using a\n    // timeline that is unique to the run.\n    return {\n      timeline: `${relkind}:${randomCharacters(12)}`,\n      resumable: false,\n      start: undefined,\n    };\n  }\n  if (filenode !== snapshotFilenode) {\n    throw new Error(\n      `${bf.table.schema}.${bf.table.name} was rewritten after the backfill ` +\n        `snapshot (relfilenode ${snapshotFilenode} => ${filenode})`,\n    );\n  }\n  const timeline = filenode;\n  const requested = resumePoint(bf);\n  if (requested && requested.timeline !== timeline) {\n    lc.info?.(\n      `restarting backfill from scratch: the requested progress is on ` +\n        `timeline ${requested.timeline} but the table is on ${timeline}`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  if (requested && !(await canResume(sql, relationOID, bf))) {\n    lc.info?.(\n      `restarting backfill from scratch: the table has TOASTed values ` +\n        `and backfilling columns [${Object.keys(bf.columns).join(',')}] ` +\n        `are TOAST-able`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  return {timeline, resumable: true, start: requested};\n}\n\n/**\n * Whether resuming a backfill (in a new snapshot) is guaranteed to deliver\n * the values of every row that the preceding snapshot(s) had not reached.\n *\n * Within a single snapshot, every row is scanned exactly once, since the\n * version visible to the snapshot stays in place while it is held. Across\n * snapshots, however, a row that had not yet been reached (i.e. beyond the\n * resume mark `M`) can be updated between the snapshots, with its new\n * version placed at or before `M` (a non-HOT update into free space in an\n * earlier page, or a HOT update into a lower line pointer of `M`'s page).\n * Neither snapshot then scans the row, and it is only delivered by the\n * replication stream. The stream's UPDATE contains the full row, _except_\n * for unchanged values stored out-of-line in TOAST, which pgoutput omits\n * (unless the table has REPLICA IDENTITY FULL). A backfilling column with\n * such a value would be published without it.\n *\n * Resuming is therefore only done if no backfilling column can be TOASTed\n * (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the\n * table has no TOAST data. The latter is safe because an omitted value must\n * still be in the TOAST relation at the new snapshot (unless its row has since\n * been deleted, which the replication stream does deliver), and conservative\n * because the TOAST relation does not shrink to empty without a heap rewrite,\n * which changes the timeline.\n *\n * Potential optimization for TOAST-able backfills: have the resumed run also\n * deliver the rows at or before `M` that were modified since the snapshot of\n * the run that started the backfill from scratch. That run's snapshot `xmin`\n * (`T_origin`, as an xid8) would be encoded in the timeline, i.e.\n * `timeline = \"<relfilenode>:<T_origin>\"`, and carried over (rather than\n * replaced) by resumed runs, so that:\n * * Within a chain of runs, every row version with `xmin < T_origin` stayed\n *   in place across all of the runs' snapshots and was therefore scanned\n *   exactly once, and all other rows are re-delivered by the resumed run\n *   (`ctid > M OR xmin >= T_origin`).\n * * Continuations from a different chain (i.e. with a newer `T_origin`) are\n *   not accepted by subscribers, and requests from different chains are\n *   merged into a backfill from scratch, as the timelines differ. Note that\n *   using the _latest_ `T` would be incorrect: rows that moved before `M`\n *   between the earlier snapshots would never be re-delivered.\n * The resumed run would scan the whole table (but only deliver the modified\n * rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which\n * requires falling back to a backfill from scratch once the current xid is\n * 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the\n * `xmin` system column returning the raw xmin of frozen tuples, which should\n * be verified.\n */\nasync function canResume(\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<boolean> {\n  const [{toastable, toasted}] = await sql<\n    {toastable: boolean; toasted: boolean}[]\n  >`\n    SELECT\n      EXISTS (\n        SELECT 1 FROM pg_attribute\n          WHERE attrelid = ${relationOID}\n            AND attname IN ${sql(Object.keys(bf.columns))}\n            AND attstorage <> 'p'\n      ) AS toastable,\n      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted\n      FROM pg_class WHERE oid = ${relationOID}`;\n  return !toastable || !toasted;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,UAAU;AAChB,IAAM,eAAe;AAErB,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,QAAQ,KAAK,IAAI;CAC/B,OAAO,aAAa,iBAAiB,MAAM;CAC3C,OAAO,GAAG,MAAM,GAAG,SAAS,IAAI,GAAG,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,GAAG;AAClE;;AAGA,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,aAAa,KAAK,IAAI;CACpC,OAAO,aAAa,+BAA+B,MAAM;CACzD,OAAO,IAAI,OAAO,MAAM,EAAE,EAAE,GAAG,OAAO,MAAM,EAAE,EAAE;AAClD;AAEA,SAAS,cACP,UACA,SACA;CACA,OAAO,WAAW;EAAC;EAAU;CAAO,IAAI,EAAC,QAAO;AAClD;;AAGA,IAAM,cAAc;;;;;;;;;;;;AAapB,IAAM,0BAA0B;CAC9B;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aAAa,MAAsB;CAC1C,OAAO,OAAO,IAAI,EAAE,SAAS,IAAI,GAAG;AACtC;AA4BA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EACJ,sBAAsB,0BACtB,WAAW,OACX,4BACE;CACJ,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,0BAA0B;EAChC,MAAM,EAAC,WAAW,UAAU,aAAY,MAAM,eAC5C,IACA,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EAMpD,MAAM,QACJ,SAAS,aAAa,SAAS,QAC3B,EACE,OAAe,WAAW,mBAAmB,SAAS,MAAM,YAAY,EAAE,QAC5E,IACA,KAAA;EACN,MAAM,QAAQ,uBACZ,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,KAAK,KAAI,QAAO,GAAG,GAAG,CAAC,GAAG,WAAW,GACzC,KACF;EACA,IAAI,SAAS,OACX,GAAG,OACD,0BAA0B,SAAS,MAAM,aAAa,aACvC,SAAS,SAAS,EACnC;EAGF,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,IACA,SAAiB,MAClB,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,sBAAsB,WAAW,IAAI,GAAG,WAAW,GACvD,KACF;GAEA,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,IACA,QAAgB,IAAI,SAAS,MAAM,GACpC,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,UAAU,WAAW,OAAO,aAC7B,EACE,cACA,iBAEF,aACA,QACA,UACA,aACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAE/C,KAAK,MAAM,WAAW,yBACpB,MAAM,IAAI,OAAO,aAAa,SAAS;EAEzC,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAIA,IAAI,UAA4C;CAChD,IAAI,WAA6C;CACjD,MAAM,UAAU,kBAA0B;EAAC;EAAc;CAAQ;CAGjE,MAAM,UAAU,SAAS,SAAS;CAGlC,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,MAAM,SAAS,QACjB,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;QACjD;IACL,MAAM,OAAO,YACT,mBAAmB,YAAY,KAAU,CAAC,IAC1C,aAAa,OAAO,OAAO,CAAC;IAChC,IAAI,YAAY,KAAA,KAAa,QAAQ,QAAQ,cAG3C,MAAM,IAAI,MACR,wCACM,KAAK,SAAS,QAAQ,aAAa,EAC3C;IAEF,UAAU,OAAO,IAAI;GACvB;GAEA,IAAI,EAAE,QAAQ,SAAS;IACrB,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,uBAAuB,UAAU,SAAS,GAAG;GAChE,MAAM;IACJ,SAAS;KACP,KAAK;KACL,GAAG;KACH;KACA;KACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;IACtD;IACA,UAAU;GACZ;GACA,WAAW;GACX;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IACP,KAAK;IACL,GAAG;IACH;IACA;IACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;GACtD;GACA,UAAU;EACZ;EACA,WAAW;EACX;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GACP,KAAK;GACL,GAAG;GACH;GAEA,eAAe,WAAW,EAAC,SAAQ,IAAI,CAAC;EAC1C;EACA,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,IACA,cACA,IACA,WAKC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAAC,KAAK,EAAE,CAAU,CACvE;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAaA,OAAO;GAAC,WAAW;GAAM,UAAA;IAXvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAGuB;GAAU,UAAA,MADZ,YAAY,IAAI,KAAK,KAAK,KAAK,EAAE;EACb;CAC7C,CAAC;AACH;;;;;;;;;;;;;;;;AAiBA,eAAe,YACb,IACA,KACA,aACA,IACmB;CACnB,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,MAAM,EAAE,GAAG,GAAG,GAAG,MAAM,IAAI;CACxD,MAAM,IAAI,OAAO,cAAc,MAAM,sBAAsB;CAC3D,MAAM,CAAC,EAAC,WAAW,SAAS,kBAAkB,cAAa,MAAM,GAOhE;yBACsB,MAAM;;;;kCAIG;CAChC,IAAI,OAAO,SAAS,MAAM,aAExB,MAAM,IAAI,2BACR,IACA,oCACF;CAEF,IAAI,YAAY,OAAO,aAAa,MAIlC,OAAO;EACL,UAAU,GAAG,QAAQ,GAAG,iBAAiB,EAAE;EAC3C,WAAW;EACX,OAAO,KAAA;CACT;CAEF,IAAI,aAAa,kBACf,MAAM,IAAI,MACR,GAAG,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,0DACT,iBAAiB,MAAM,SAAS,EAC7D;CAEF,MAAM,WAAW;CACjB,MAAM,YAAY,YAAY,EAAE;CAChC,IAAI,aAAa,UAAU,aAAa,UAAU;EAChD,GAAG,OACD,2EACc,UAAU,SAAS,uBAAuB,UAC1D;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,IAAI,aAAa,CAAE,MAAM,UAAU,KAAK,aAAa,EAAE,GAAI;EACzD,GAAG,OACD,2FAC8B,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,iBAElE;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,OAAO;EAAC;EAAU,WAAW;EAAM,OAAO;CAAS;AACrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,eAAe,UACb,KACA,aACA,IACkB;CAClB,MAAM,CAAC,EAAC,WAAW,aAAY,MAAM,GAEpC;;;;6BAI0B,YAAY;6BACZ,IAAI,OAAO,KAAK,GAAG,OAAO,CAAC,EAAE;;;;kCAIxB;CAChC,OAAO,CAAC,aAAa,CAAC;AACxB"}
+{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randomCharacters} from '../../../../../shared/src/random-values.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport {resumePoint} from '../protocol/backfill-progress.ts';\nimport type {\n  BackfillCompleted,\n  BackfillProgressMark,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag' | 'progressMarks'>;\n\n/**\n * Determines how the progress of a backfill is marked, and from where it is\n * resumed.\n */\ntype Progress = {\n  /**\n   * The timeline of the progress marks. For plain tables, this is the table's\n   * `relfilenode`, which identifies the physical \"generation\" of the table's\n   * data (changed by heap rewrites like `VACUUM FULL`, `CLUSTER`, or\n   * `TRUNCATE`), within which `ctid`s are comparable.\n   */\n  timeline: string;\n\n  /**\n   * Whether the progress marks are resumable `ctid`s. Tables without\n   * (their own) storage, such as partitioned tables, are not resumable and\n   * are instead marked with the number of rows streamed, on a timeline that\n   * is unique to the run.\n   */\n  resumable: boolean;\n\n  /** The mark from which the backfill resumes, if any. */\n  start: BackfillProgressMark | undefined;\n};\n\n/**\n * Encodes a `ctid` text representation, e.g. `(123,4)`, as a\n * fixed-width string that sorts lexicographically in `ctid` order.\n */\nconst CTID_RE = /^\\((\\d+),(\\d+)\\)$/;\nconst CTID_MARK_RE = /^(\\d{10})\\.(\\d{5})$/;\n\nexport function ctidToProgressMark(ctid: string): string {\n  const match = CTID_RE.exec(ctid);\n  assert(match, () => `invalid ctid: ${ctid}`);\n  return `${match[1].padStart(10, '0')}.${match[2].padStart(5, '0')}`;\n}\n\n/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */\nexport function progressMarkToCtid(mark: string): string {\n  const match = CTID_MARK_RE.exec(mark);\n  assert(match, () => `invalid ctid progress mark: ${mark}`);\n  return `(${Number(match[1])},${Number(match[2])})`;\n}\n\nfunction progressMarks(\n  previous: BackfillProgressMark | undefined,\n  current: BackfillProgressMark,\n) {\n  return previous ? {previous, current} : {current};\n}\n\n/** The `ctid` of each row is appended as the last column of the download. */\nconst CTID_SELECT = 'ctid::text';\n\n/**\n * Settings that ensure that the COPY scans the table in physical (i.e.\n * `ctid`) order, which is necessary for the progress marks of a resumable\n * backfill to be meaningful. (Monotonicity is nonetheless verified as rows\n * are streamed.)\n *\n * * Synchronized seqscans can start in the middle of the table.\n * * Parallel scans interleave the rows of different workers.\n * * Index scans follow index order. (A publication row filter could\n *   otherwise be satisfied with an index.)\n */\nconst PHYSICAL_ORDER_SETTINGS = [\n  'synchronize_seqscans = off',\n  'max_parallel_workers_per_gather = 0',\n  'enable_indexscan = off',\n  'enable_indexonlyscan = off',\n  'enable_bitmapscan = off',\n];\n\nfunction rowCountMark(rows: number): string {\n  return String(rows).padStart(16, '0');\n}\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n\n  /**\n   * @visibleForTesting\n   * Called after the snapshot transaction is opened, before the table is\n   * locked, e.g. to run a concurrent heap rewrite in between.\n   */\n  afterSnapshotForTesting?: (() => Promise<void>) | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {\n    flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE,\n    textCopy = false,\n    afterSnapshotForTesting,\n  } = opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    await afterSnapshotForTesting?.();\n    const {tableSpec, backfill, progress} = await validateSchema(\n      lc,\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    // Resumable backfills are streamed in physical (i.e. ctid) order, which\n    // is the natural order of a sequential or TID range scan (as enforced by\n    // the settings in `stream()`). An ORDER BY is not used, as Postgres does\n    // not consider either scan to be ordered, and would unnecessarily sort\n    // the table.\n    const order =\n      progress.resumable && progress.start\n        ? {\n            after: /*sql*/ `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid`,\n          }\n        : undefined;\n    const stmts = makeDownloadStatements(\n      tableSpec,\n      cols,\n      undefined,\n      undefined,\n      [...cols.map(col => id(col)), CTID_SELECT],\n      order,\n    );\n    if (progress.start) {\n      lc.info?.(\n        `resuming backfill from ${progress.start.progressMark} ` +\n          `(timeline ${progress.timeline})`,\n      );\n    }\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        (text: string) => text,\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT],\n        order,\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        (buf: Buffer) => buf.toString('utf8'), // ctid::text\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {timeline, resumable, start: startMark}: Progress,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  ctidDecoder: (field: T) => string,\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    // SET LOCAL applies to the (snapshot) transaction of this read task.\n    for (const setting of PHYSICAL_ORDER_SETTINGS) {\n      await sql.unsafe(`SET LOCAL ${setting}`);\n    }\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // The mark of the last row that was streamed, and that of the last row of\n  // the previous message.\n  let current: BackfillProgressMark | undefined = startMark;\n  let previous: BackfillProgressMark | undefined = startMark;\n  const markOf = (progressMark: string) => ({progressMark, timeline});\n\n  // The number of columns of each row, followed by the ctid.\n  const numCols = decoders.length + 1;\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      if (col < decoders.length) {\n        row[col] = field === null ? null : decoders[col](field);\n      } else {\n        const mark = resumable\n          ? ctidToProgressMark(ctidDecoder(field as T))\n          : rowCountMark(status.rows + 1);\n        if (current !== undefined && mark <= current.progressMark) {\n          // This should not happen given the PHYSICAL_ORDER_SETTINGS, but\n          // the correctness of the progress marks depends on it.\n          throw new Error(\n            `backfill rows are not in ctid order ` +\n              `(${mark} after ${current.progressMark})`,\n          );\n        }\n        current = markOf(mark);\n      }\n\n      if (++col === numCols) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {\n      yield {\n        message: {\n          tag: 'backfill',\n          ...backfill,\n          rowValues,\n          status,\n          progressMarks: progressMarks(previous, must(current)),\n        },\n        byteSize: bufferedBytes,\n      };\n      previous = current;\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {\n        tag: 'backfill',\n        ...backfill,\n        rowValues,\n        status,\n        progressMarks: progressMarks(previous, must(current)),\n      },\n      byteSize: bufferedBytes,\n    };\n    previous = current;\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {\n      tag: 'backfill-completed',\n      ...backfill,\n      status,\n      // `previous` is only unset for a backfill from scratch with no rows.\n      progressMarks: previous ? {previous} : {},\n    },\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n        diagnosticsDB: db,\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  lc: LogContext,\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n  progress: Progress;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns).map(([col, {id}]) => [col, id] as const),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    const progress = await getProgress(lc, sql, spec.oid, bf);\n    return {tableSpec: spec, backfill, progress};\n  });\n}\n\n/**\n * Determines the timeline of the table's progress marks (as of the snapshot),\n * and whether the backfill can be resumed from the requested progress.\n *\n * The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest\n * of the transaction) so that its storage cannot be rewritten (by `VACUUM\n * FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which\n * require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note\n * that `pg_relation_filenode()` reflects the table's current storage (only\n * refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at\n * the transaction snapshot. If they differ, the storage was rewritten after the\n * snapshot, and the COPY would scan the new storage with the old snapshot,\n * which is not MVCC-safe for all rewrites (e.g. the table appears empty after\n * a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.\n */\nasync function getProgress(\n  lc: LogContext,\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<Progress> {\n  const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;\n  await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);\n  const [{lockedOID, relkind, snapshotFilenode, filenode}] = await sql<\n    {\n      lockedOID: number | null;\n      relkind: string;\n      snapshotFilenode: string;\n      filenode: string | null;\n    }[]\n  >`\n    SELECT to_regclass(${table})::oid::int8 AS \"lockedOID\",\n           relkind,\n           relfilenode::text AS \"snapshotFilenode\",\n           pg_relation_filenode(oid)::text AS filenode\n      FROM pg_class WHERE oid = ${relationOID}`;\n  if (Number(lockedOID) !== relationOID) {\n    // The (current) table with the name is not the one at the snapshot.\n    throw new SchemaIncompatibilityError(\n      bf,\n      `Table has been renamed or replaced`,\n    );\n  }\n  if (relkind !== 'r' || filenode === null) {\n    // Not a plain table (e.g. a partitioned table), for which ctids are not\n    // unique. Such backfills are not resumable, which is achieved by using a\n    // timeline that is unique to the run.\n    return {\n      timeline: `${relkind}:${randomCharacters(12)}`,\n      resumable: false,\n      start: undefined,\n    };\n  }\n  if (filenode !== snapshotFilenode) {\n    throw new Error(\n      `${bf.table.schema}.${bf.table.name} was rewritten after the backfill ` +\n        `snapshot (relfilenode ${snapshotFilenode} => ${filenode})`,\n    );\n  }\n  const timeline = filenode;\n  const requested = resumePoint(bf);\n  if (requested && requested.timeline !== timeline) {\n    lc.info?.(\n      `restarting backfill from scratch: the requested progress is on ` +\n        `timeline ${requested.timeline} but the table is on ${timeline}`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  if (requested && !(await canResume(sql, relationOID, bf))) {\n    lc.info?.(\n      `restarting backfill from scratch: the table has TOASTed values ` +\n        `and backfilling columns [${Object.keys(bf.columns).join(',')}] ` +\n        `are TOAST-able`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  return {timeline, resumable: true, start: requested};\n}\n\n/**\n * Whether resuming a backfill (in a new snapshot) is guaranteed to deliver\n * the values of every row that the preceding snapshot(s) had not reached.\n *\n * Within a single snapshot, every row is scanned exactly once, since the\n * version visible to the snapshot stays in place while it is held. Across\n * snapshots, however, a row that had not yet been reached (i.e. beyond the\n * resume mark `M`) can be updated between the snapshots, with its new\n * version placed at or before `M` (a non-HOT update into free space in an\n * earlier page, or a HOT update into a lower line pointer of `M`'s page).\n * Neither snapshot then scans the row, and it is only delivered by the\n * replication stream. The stream's UPDATE contains the full row, _except_\n * for unchanged values stored out-of-line in TOAST, which pgoutput omits\n * (unless the table has REPLICA IDENTITY FULL). A backfilling column with\n * such a value would be published without it.\n *\n * Resuming is therefore only done if no backfilling column can be TOASTed\n * (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the\n * table has no TOAST data. The latter is safe because an omitted value must\n * still be in the TOAST relation at the new snapshot (unless its row has since\n * been deleted, which the replication stream does deliver), and conservative\n * because the TOAST relation does not shrink to empty without a heap rewrite,\n * which changes the timeline.\n *\n * Potential optimization for TOAST-able backfills: have the resumed run also\n * deliver the rows at or before `M` that were modified since the snapshot of\n * the run that started the backfill from scratch. That run's snapshot `xmin`\n * (`T_origin`, as an xid8) would be encoded in the timeline, i.e.\n * `timeline = \"<relfilenode>:<T_origin>\"`, and carried over (rather than\n * replaced) by resumed runs, so that:\n * * Within a chain of runs, every row version with `xmin < T_origin` stayed\n *   in place across all of the runs' snapshots and was therefore scanned\n *   exactly once, and all other rows are re-delivered by the resumed run\n *   (`ctid > M OR xmin >= T_origin`).\n * * Continuations from a different chain (i.e. with a newer `T_origin`) are\n *   not accepted by subscribers, and requests from different chains are\n *   merged into a backfill from scratch, as the timelines differ. Note that\n *   using the _latest_ `T` would be incorrect: rows that moved before `M`\n *   between the earlier snapshots would never be re-delivered.\n * The resumed run would scan the whole table (but only deliver the modified\n * rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which\n * requires falling back to a backfill from scratch once the current xid is\n * 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the\n * `xmin` system column returning the raw xmin of frozen tuples, which should\n * be verified.\n */\nasync function canResume(\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<boolean> {\n  const [{toastable, toasted}] = await sql<\n    {toastable: boolean; toasted: boolean}[]\n  >`\n    SELECT\n      EXISTS (\n        SELECT 1 FROM pg_attribute\n          WHERE attrelid = ${relationOID}\n            AND attname IN ${sql(Object.keys(bf.columns))}\n            AND attstorage <> 'p'\n      ) AS toastable,\n      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted\n      FROM pg_class WHERE oid = ${relationOID}`;\n  return !toastable || !toasted;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,UAAU;AAChB,IAAM,eAAe;AAErB,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,QAAQ,KAAK,IAAI;CAC/B,OAAO,aAAa,iBAAiB,MAAM;CAC3C,OAAO,GAAG,MAAM,GAAG,SAAS,IAAI,GAAG,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,GAAG;AAClE;;AAGA,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,aAAa,KAAK,IAAI;CACpC,OAAO,aAAa,+BAA+B,MAAM;CACzD,OAAO,IAAI,OAAO,MAAM,EAAE,EAAE,GAAG,OAAO,MAAM,EAAE,EAAE;AAClD;AAEA,SAAS,cACP,UACA,SACA;CACA,OAAO,WAAW;EAAC;EAAU;CAAO,IAAI,EAAC,QAAO;AAClD;;AAGA,IAAM,cAAc;;;;;;;;;;;;AAapB,IAAM,0BAA0B;CAC9B;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aAAa,MAAsB;CAC1C,OAAO,OAAO,IAAI,EAAE,SAAS,IAAI,GAAG;AACtC;AA4BA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EACJ,sBAAsB,0BACtB,WAAW,OACX,4BACE;CACJ,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,0BAA0B;EAChC,MAAM,EAAC,WAAW,UAAU,aAAY,MAAM,eAC5C,IACA,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EAMpD,MAAM,QACJ,SAAS,aAAa,SAAS,QAC3B,EACE,OAAe,WAAW,mBAAmB,SAAS,MAAM,YAAY,EAAE,QAC5E,IACA,KAAA;EACN,MAAM,QAAQ,uBACZ,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,KAAK,KAAI,QAAO,GAAG,GAAG,CAAC,GAAG,WAAW,GACzC,KACF;EACA,IAAI,SAAS,OACX,GAAG,OACD,0BAA0B,SAAS,MAAM,aAAa,aACvC,SAAS,SAAS,EACnC;EAGF,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,IACA,SAAiB,MAClB,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,sBAAsB,WAAW,IAAI,GAAG,WAAW,GACvD,KACF;GAEA,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,IACA,QAAgB,IAAI,SAAS,MAAM,GACpC,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,UAAU,WAAW,OAAO,aAC7B,EACE,cACA,iBAEF,aACA,QACA,UACA,aACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAE/C,KAAK,MAAM,WAAW,yBACpB,MAAM,IAAI,OAAO,aAAa,SAAS;EAEzC,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAIA,IAAI,UAA4C;CAChD,IAAI,WAA6C;CACjD,MAAM,UAAU,kBAA0B;EAAC;EAAc;CAAQ;CAGjE,MAAM,UAAU,SAAS,SAAS;CAGlC,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,MAAM,SAAS,QACjB,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;QACjD;IACL,MAAM,OAAO,YACT,mBAAmB,YAAY,KAAU,CAAC,IAC1C,aAAa,OAAO,OAAO,CAAC;IAChC,IAAI,YAAY,KAAA,KAAa,QAAQ,QAAQ,cAG3C,MAAM,IAAI,MACR,wCACM,KAAK,SAAS,QAAQ,aAAa,EAC3C;IAEF,UAAU,OAAO,IAAI;GACvB;GAEA,IAAI,EAAE,QAAQ,SAAS;IACrB,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,uBAAuB,UAAU,SAAS,GAAG;GAChE,MAAM;IACJ,SAAS;KACP,KAAK;KACL,GAAG;KACH;KACA;KACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;IACtD;IACA,UAAU;GACZ;GACA,WAAW;GACX;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IACP,KAAK;IACL,GAAG;IACH;IACA;IACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;GACtD;GACA,UAAU;EACZ;EACA,WAAW;EACX;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GACP,KAAK;GACL,GAAG;GACH;GAEA,eAAe,WAAW,EAAC,SAAQ,IAAI,CAAC;EAC1C;EACA,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;GACX,eAAe;EACjB,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,IACA,cACA,IACA,WAKC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAAC,KAAK,EAAE,CAAU,CACvE;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAaA,OAAO;GAAC,WAAW;GAAM,UAAA;IAXvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAGuB;GAAU,UAAA,MADZ,YAAY,IAAI,KAAK,KAAK,KAAK,EAAE;EACb;CAC7C,CAAC;AACH;;;;;;;;;;;;;;;;AAiBA,eAAe,YACb,IACA,KACA,aACA,IACmB;CACnB,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,MAAM,EAAE,GAAG,GAAG,GAAG,MAAM,IAAI;CACxD,MAAM,IAAI,OAAO,cAAc,MAAM,sBAAsB;CAC3D,MAAM,CAAC,EAAC,WAAW,SAAS,kBAAkB,cAAa,MAAM,GAOhE;yBACsB,MAAM;;;;kCAIG;CAChC,IAAI,OAAO,SAAS,MAAM,aAExB,MAAM,IAAI,2BACR,IACA,oCACF;CAEF,IAAI,YAAY,OAAO,aAAa,MAIlC,OAAO;EACL,UAAU,GAAG,QAAQ,GAAG,iBAAiB,EAAE;EAC3C,WAAW;EACX,OAAO,KAAA;CACT;CAEF,IAAI,aAAa,kBACf,MAAM,IAAI,MACR,GAAG,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,0DACT,iBAAiB,MAAM,SAAS,EAC7D;CAEF,MAAM,WAAW;CACjB,MAAM,YAAY,YAAY,EAAE;CAChC,IAAI,aAAa,UAAU,aAAa,UAAU;EAChD,GAAG,OACD,2EACc,UAAU,SAAS,uBAAuB,UAC1D;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,IAAI,aAAa,CAAE,MAAM,UAAU,KAAK,aAAa,EAAE,GAAI;EACzD,GAAG,OACD,2FAC8B,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,iBAElE;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,OAAO;EAAC;EAAU,WAAW;EAAM,OAAO;CAAS;AACrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,eAAe,UACb,KACA,aACA,IACkB;CAClB,MAAM,CAAC,EAAC,WAAW,aAAY,MAAM,GAEpC;;;;6BAI0B,YAAY;6BACZ,IAAI,OAAO,KAAK,GAAG,OAAO,CAAC,EAAE;;;;kCAIxB;CAChC,OAAO,CAAC,aAAa,CAAC;AACxB"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -2,7 +2,7 @@ import type { Readable } from 'node:stream';
 import type { LogContext } from '@rocicorp/logger';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
-import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
+import { type ConstrainingPurgeLock, type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
 import { type LSN } from './lsn.ts';
 import { type ReplicaState } from './schema/shard.ts';
@@ -16,6 +16,31 @@ export type InitializeOptions = ReplicaOptions & {
  * stream.
  */
 export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
+/**
+ * The watermark below which the replication stream cannot (correctly) be
+ * started, i.e. from which it would skip transactions that the change log
+ * does not contain. It reflects the order in which each mode stores a
+ * transaction and then acks it to the slot, such that a correctly initialized
+ * stream always starts at or past it:
+ *
+ * * After an initial sync, the replica's watermark, which is the position
+ *   at which the new slot was created, and the change log is (re)initialized
+ *   at it.
+ * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored
+ *   replica's watermark and the slot's position. Transactions are only acked
+ *   once backed up (so a skipped interval ends at or before the replica), and
+ *   if the change log is behind the slot it is re-initialized from the
+ *   replica. Note that the slot can be past the replica (acked for keepalives,
+ *   with nothing skipped), and the replica past the change log (forwarded
+ *   before being stored, and streamed again from the slot), but the stream
+ *   never starts below both. (With RMv1.5's purge lock, the change log's head
+ *   is at or past the slot, so it does not lower the minimum.)
+ * * With a shared slot (RMv1), the change log's head when it was purge
+ *   locked, as the stream resumes from the head, which only advances. The
+ *   slot is not a reference: its owner acks it until it is taken over. Without
+ *   a purge lock (i.e. an empty change log), there is no floor.
+ */
+export declare function minStartWatermark(replicaWatermark: string, initialSynced: boolean, slotWatermark: string | undefined, purgeLock: ConstrainingPurgeLock | null): string | undefined;
 export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
     restoreFrom: ReplicaState;
     replicateTo: ReplicaState;

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAsI3B;AAqKD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CAiKA"}
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CA6I3B;AAkKD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,wBAAgB,iBAAiB,CAC/B,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,qBAAqB,GAAG,IAAI,GACtC,MAAM,GAAG,SAAS,CAQpB;AAkBD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CAiKA"}