@rocicorp/zero 1.11.0-canary.22 → 1.11.0-canary.24

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, acquirePurgeLock}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    // Without a replication slot, the change-log is never behind one.\n    const lock = await acquirePurgeLock?.();\n    const constraints =\n      lock === 'behind-slot' ? undefined : (lock ?? undefined);\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,qBAAoC,CAAC,GACvB;CAS3B,IAAI,YAAY,WAAW;EAEzB,MAAM,OAAO,MAAM,mBAAmB;EAGtC,MAAM,eAAe,IAAI,YAAY,eADnC,SAAS,gBAAgB,KAAA,IAAa,QAAQ,KAAA,CACe;CACjE;CAEA,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}
+{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, acquirePurgeLock}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    // Without a replication slot, the change-log is never behind one.\n    const lock = await acquirePurgeLock?.();\n    const constraints =\n      lock === 'behind-slot' ? undefined : (lock ?? undefined);\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n    // Backs up to the (non-lineage) base URL.\n    newBackupLineage: false,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,qBAAoC,CAAC,GACvB;CAS3B,IAAI,YAAY,WAAW;EAEzB,MAAM,OAAO,MAAM,mBAAmB;EAGtC,MAAM,eAAe,IAAI,YAAY,eADnC,SAAS,gBAAgB,KAAA,IAAa,QAAQ,KAAA,CACe;CACjE;CAEA,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;EAE5B,kBAAkB;CACpB;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -1,3 +1,4 @@
+import type { Readable } from 'node:stream';
 import type { LogContext } from '@rocicorp/logger';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
@@ -20,5 +21,10 @@ export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: Pos
     replicateTo: ReplicaState;
     /** The position of the slot from which replication resumes. */
     slotLSN: LSN;
+    /**
+     * The session holding the destination slot active until it is taken
+     * over by the PostgresChangeSource. Destroying it releases the slot.
+     */
+    slotSession: Readable;
 } | undefined>;
 //# sourceMappingURL=change-source-init.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAMxE,OAAO,EAEL,KAAK,gBAAgB,EAErB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAqID,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;CACd,GACD,SAAS,CACZ,CAiJA"}
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAmI3B;AAqJD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CAiKA"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js

@@ -35,17 +35,19 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
 		const { replica: restoredReplica, purgeLock, pgChangeLogBehindSlot } = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
 		let initialSyncedReplica;
 		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
-			purgeLock?.release();
+			await restoreOptions.cleanup?.release();
+			await purgeLock?.release();
 			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, {
 				epoch,
 				backupV5
-			});
+			}, restoreOptions.cleanup);
 		});
 		const replica = new Database(lc, replicaDbFile);
 		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
 		replica.close();
 		const { upstreamReplica, pgVersion } = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
-		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
+		const newBackupLineage = backupV5;
+		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : newBackupLineage ? String(Date.now()) : restoredReplica?.backupPath ?? null;
 		return {
 			subscriptionState,
 			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, pgVersion, {
@@ -55,7 +57,8 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
 			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
 			replicaID: upstreamReplica.id,
 			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null),
-			pgChangeLogBehindSlot
+			pgChangeLogBehindSlot,
+			newBackupLineage
 		};
 	} finally {
 		await db.end();
@@ -65,8 +68,7 @@ async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream
 	const lock = await acquirePurgeLock?.();
 	assert(lock !== "behind-slot", "unexpected purge-lock result");
 	const purgeLock = lock ?? null;
-	const constraints = purgeLock ?? void 0;
-	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
+	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (purgeLock?.replicaVersion ?? generation));
 	if (replicas.length === 0) {
 		lc.info?.(`no suitable replicas to restore from`, { replicas });
 		return {
@@ -76,6 +78,7 @@ async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream
 	}
 	const [replica] = replicas;
 	if (litestream?.backupURL) {
+		const constraints = restoreConstraints(purgeLock, replica);
 		const { backupURL: backupBaseURL } = litestream;
 		const { slot, backupPath, confirmedFlushLsn } = replica;
 		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
@@ -90,17 +93,18 @@ async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream
 		purgeLock
 	};
 }
-async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, acquirePurgeLock }, gracePeriodMs) {
+async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, acquirePurgeLock, cleanup }, gracePeriodMs) {
 	const result = await getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs);
 	if (!result) return {
 		replica: void 0,
 		purgeLock: null
 	};
-	const { restoreFrom, replicateTo, slotLSN } = result;
+	const { restoreFrom, replicateTo, slotLSN, slotSession } = result;
+	cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () => slotSession.destroy());
 	const lock = acquirePurgeLock ? await acquirePurgeLock(toStateVersionString(slotLSN)) : null;
 	const pgChangeLogBehindSlot = lock === "behind-slot";
 	const purgeLock = lock === "behind-slot" ? null : lock;
-	const constraints = purgeLock ?? void 0;
+	const constraints = restoreConstraints(purgeLock, restoreFrom);
 	if (litestream?.backupURL) {
 		const { backupURL: backupBaseURL } = litestream;
 		const { slot, backupPath, confirmedFlushLsn } = restoreFrom;
@@ -120,6 +124,12 @@ async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaF
 		pgChangeLogBehindSlot
 	};
 }
+function restoreConstraints(purgeLock, { generation }) {
+	return purgeLock ?? {
+		replicaVersion: generation,
+		minWatermark: ""
+	};
+}
 async function currentWalLSN(sql) {
 	const [{ lsn }] = await sql`SELECT pg_current_wal_lsn() AS lsn`;
 	return lsn;
@@ -167,10 +177,14 @@ async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailov
 								sourceID: replica.id,
 								destID: destination.replica.id
 							}), `replica ${destination.replica.id} was deleted`),
-							slotLSN: destination.slot.consistent_point
+							slotLSN: destination.slot.consistent_point,
+							slotSession: destination.initialSession
 						};
 					}
 				}
+				if (replica.stage === 2 && replica.active && replica.id !== destination?.replica.id) {
+					if (i % 12 === 0) lc.info?.(`waiting for replica ${replica.id}@${replica.slot} being restored by another task`, { replica });
+				}
 				if (replica.backupPath && !replica.active) {
 					const now = Date.now();
 					const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);
@@ -184,7 +198,8 @@ async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailov
 							return {
 								restoreFrom: replica,
 								replicateTo: reserved.replica,
-								slotLSN: replica.confirmedFlushLsn
+								slotLSN: replica.confirmedFlushLsn,
+								slotSession: reserved.reservation
 							};
 						}
 					}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type ConstrainingPurgeLock,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2.\n        void purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n        // recorded in the replicas table by the PostgresChangeSource.\n        backupV5\n        ? String(Date.now())\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const constraints = purgeLock ?? undefined;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the constraints, if present\n    ({generation}) =>\n      generation === (constraints?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN} = result;\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for changes outside of the publication (e.g. on an\n  // idle upstream). The change-log is then re-initialized unnecessarily, which\n  // resets subscribers behind the replica. (This could be avoided by recording\n  // the positions acked for such changes in the change-log, e.g. an\n  // `ackedThrough` column, and checking the slot's position against that.\n  // Resumption is uncommon, however, and involves downtime anyway.)\n  const lock = acquirePurgeLock\n    ? await acquirePurgeLock(toStateVersionString(slotLSN))\n    : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  const constraints = purgeLock ?? undefined;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n            };\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA6DA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,0BACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,WAAgB,QAAQ;GACxB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,CAClB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAEA,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,WACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;EACF;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAUA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,cAAc,aAAa,KAAA;CACjC,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBACA,gBAAgB,aAAa,kBAAkB,WACnD;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,oBACb,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,YAAW;CAkB5C,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAClD,MAAM,cAAc,aAAa,KAAA;CAEjC,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BASjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;MAC5B;KACF;IACF;IAEA,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;OACnB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to a new backup\n    // lineage.\n    const newBackupLineage = backupV5;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n        // recorded in the replicas table by the PostgresChangeSource.\n        newBackupLineage\n        ? String(Date.now())\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for changes outside of the publication (e.g. on an\n  // idle upstream). The change-log is then re-initialized unnecessarily, which\n  // resets subscribers behind the replica. (This could be avoided by recording\n  // the positions acked for such changes in the change-log, e.g. an\n  // `ackedThrough` column, and checking the slot's position against that.\n  // Resumption is uncommon, however, and involves downtime anyway.)\n  const lock = acquirePurgeLock\n    ? await acquirePurgeLock(toStateVersionString(slotLSN))\n    : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (i % 12 === 0) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,0BACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,GAChB,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB;EACzB,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,mBACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;EACF;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAUA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAKzD,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAkBA,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAGhC,IAAI,OAAO,GACb,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts

@@ -6,6 +6,7 @@ import type { IndexSpec, PublishedTableSpec } from '../../../db/specs.ts';
 import { type PostgresDB } from '../../../types/pg.ts';
 import type { ShardConfig } from '../../../types/shards.ts';
 import { ReplicationStatusPublisher } from '../../replicator/replication-status.ts';
+import type { InitCleanup } from '../common/init-cleanup.ts';
 import { type ReplicaState } from './schema/shard.ts';
 export type InitialSyncOptions = {
     tableCopyWorkers: number;
@@ -40,7 +41,7 @@ export type ServerContext = JSONObject;
  * @returns The {@link ReplicaState} of the initialized replica, or `undefined`
  *          for a shadow sync.
  */
-export declare function initialSync(lc: LogContext, shard: ShardConfig, tx: Database, upstreamURI: string, syncOptions: InitialSyncOptions, context: ServerContext, { epoch, backupV5 }?: ReplicaOptions): Promise<ReplicaState | undefined>;
+export declare function initialSync(lc: LogContext, shard: ShardConfig, tx: Database, upstreamURI: string, syncOptions: InitialSyncOptions, context: ServerContext, { epoch, backupV5 }?: ReplicaOptions, cleanup?: InitCleanup): Promise<ReplicaState | undefined>;
 export type ShadowSyncOptions = {
     sampleRate: number;
     maxRowsPerTable: number;

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAEL,0BAA0B,EAC3B,MAAM,wCAAwC,CAAC;AAQhD,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,2BAA2B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClD;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAE,cAA2C,GAC7D,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CA+RnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAgKD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,SAAS,EAAE,EACpB,eAAe,EAAE,0BAA0B,GAC1C,OAAO,CAAC,MAAM,CAAC,CAiCjB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAED;;;GAGG;AAEH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAY7E;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,GAChC,kBAAkB,CAkCpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}
+{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAEL,0BAA0B,EAC3B,MAAM,wCAAwC,CAAC;AAGhD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AAM3D,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,2BAA2B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClD;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAE,cAA2C,EAC9D,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAsSnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAgKD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,SAAS,EAAE,EACpB,eAAe,EAAE,0BAA0B,GAC1C,OAAO,CAAC,MAAM,CAAC,CAiCjB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAED;;;GAGG;AAEH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAY7E;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,GAChC,kBAAkB,CAkCpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -41,7 +41,7 @@ import { pipeline as pipeline$1 } from "node:stream/promises";
 async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context, { epoch, backupV5 } = {
 	epoch: 0,
 	backupV5: true
-}) {
+}, cleanup) {
 	if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) throw new Error("The App ID may only consist of lower-case letters, numbers, and the underscore character");
 	const { tableCopyWorkers, profileCopy, textCopy = false, replicationSlotFailover = false, installPartialIndexTriggers = true, shadow } = syncOptions;
 	const syncMode = shadow ? "shadow" : "initial";
@@ -99,6 +99,8 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context, { e
 			slotName = replication.slot.slot_name;
 			snapshotResult = replication.capturedSnapshot;
 			slotSession = replication.initialSession;
+			const session = slotSession;
+			cleanup?.onFailure(`replication slot ${slotName}`, () => session.destroy());
 		}
 		const { published, tables, indexes, numTables, copyPool, copiers } = snapshotResult;
 		const initialVersion = toStateVersionString(lsn);