@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n    opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    const stmts = makeDownloadStatements(tableSpec, cols);\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        makeBinarySelectExprs(tableSpec, cols),\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      row[col] = field === null ? null : decoders[col](field);\n\n      if (++col === decoders.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {\n        message: {tag: 'backfill', ...backfill, rowValues, status},\n        byteSize: bufferedBytes,\n      };\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {tag: 'backfill', ...backfill, rowValues, status},\n      byteSize: bufferedBytes,\n    };\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {tag: 'backfill-completed', ...backfill, status},\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAC/C,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IACJ,SAAS;KAAC,KAAK;KAAY,GAAG;KAAU;KAAW;IAAM;IACzD,UAAU;GACZ;GACA;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACzD,UAAU;EACZ;EACA;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GAAC,KAAK;GAAsB,GAAG;GAAU;EAAM;EACxD,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
+{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randomCharacters} from '../../../../../shared/src/random-values.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport {resumePoint} from '../protocol/backfill-progress.ts';\nimport type {\n  BackfillCompleted,\n  BackfillProgressMark,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag' | 'progressMarks'>;\n\n/**\n * Determines how the progress of a backfill is marked, and from where it is\n * resumed.\n */\ntype Progress = {\n  /**\n   * The timeline of the progress marks. For plain tables, this is the table's\n   * `relfilenode`, which identifies the physical \"generation\" of the table's\n   * data (changed by heap rewrites like `VACUUM FULL`, `CLUSTER`, or\n   * `TRUNCATE`), within which `ctid`s are comparable.\n   */\n  timeline: string;\n\n  /**\n   * Whether the progress marks are resumable `ctid`s. Tables without\n   * (their own) storage, such as partitioned tables, are not resumable and\n   * are instead marked with the number of rows streamed, on a timeline that\n   * is unique to the run.\n   */\n  resumable: boolean;\n\n  /** The mark from which the backfill resumes, if any. */\n  start: BackfillProgressMark | undefined;\n};\n\n/**\n * Encodes a `ctid` text representation, e.g. `(123,4)`, as a\n * fixed-width string that sorts lexicographically in `ctid` order.\n */\nconst CTID_RE = /^\\((\\d+),(\\d+)\\)$/;\nconst CTID_MARK_RE = /^(\\d{10})\\.(\\d{5})$/;\n\nexport function ctidToProgressMark(ctid: string): string {\n  const match = CTID_RE.exec(ctid);\n  assert(match, () => `invalid ctid: ${ctid}`);\n  return `${match[1].padStart(10, '0')}.${match[2].padStart(5, '0')}`;\n}\n\n/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */\nexport function progressMarkToCtid(mark: string): string {\n  const match = CTID_MARK_RE.exec(mark);\n  assert(match, () => `invalid ctid progress mark: ${mark}`);\n  return `(${Number(match[1])},${Number(match[2])})`;\n}\n\nfunction progressMarks(\n  previous: BackfillProgressMark | undefined,\n  current: BackfillProgressMark,\n) {\n  return previous ? {previous, current} : {current};\n}\n\n/** The `ctid` of each row is appended as the last column of the download. */\nconst CTID_SELECT = 'ctid::text';\n\n/**\n * Settings that ensure that the COPY scans the table in physical (i.e.\n * `ctid`) order, which is necessary for the progress marks of a resumable\n * backfill to be meaningful. (Monotonicity is nonetheless verified as rows\n * are streamed.)\n *\n * * Synchronized seqscans can start in the middle of the table.\n * * Parallel scans interleave the rows of different workers.\n * * Index scans follow index order. (A publication row filter could\n *   otherwise be satisfied with an index.)\n */\nconst PHYSICAL_ORDER_SETTINGS = [\n  'synchronize_seqscans = off',\n  'max_parallel_workers_per_gather = 0',\n  'enable_indexscan = off',\n  'enable_indexonlyscan = off',\n  'enable_bitmapscan = off',\n];\n\nfunction rowCountMark(rows: number): string {\n  return String(rows).padStart(16, '0');\n}\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n\n  /**\n   * @visibleForTesting\n   * Called after the snapshot transaction is opened, before the table is\n   * locked, e.g. to run a concurrent heap rewrite in between.\n   */\n  afterSnapshotForTesting?: (() => Promise<void>) | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {\n    flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE,\n    textCopy = false,\n    afterSnapshotForTesting,\n  } = opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  let completed = false;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    await afterSnapshotForTesting?.();\n    const {tableSpec, backfill, progress} = await validateSchema(\n      lc,\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    // Resumable backfills are streamed in physical (i.e. ctid) order, which\n    // is the natural order of a sequential or TID range scan (as enforced by\n    // the settings in `stream()`). An ORDER BY is not used, as Postgres does\n    // not consider either scan to be ordered, and would unnecessarily sort\n    // the table.\n    const order =\n      progress.resumable && progress.start\n        ? {\n            after: /*sql*/ `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid`,\n          }\n        : undefined;\n    const stmts = makeDownloadStatements(\n      tableSpec,\n      cols,\n      undefined,\n      undefined,\n      [...cols.map(col => id(col)), CTID_SELECT],\n      order,\n    );\n    if (progress.start) {\n      lc.info?.(\n        `resuming backfill from ${progress.start.progressMark} ` +\n          `(timeline ${progress.timeline})`,\n      );\n    }\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        (text: string) => text,\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT],\n        order,\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        (buf: Buffer) => buf.toString('utf8'), // ctid::text\n        flushThresholdBytes,\n      );\n    }\n    completed = true;\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    if (completed) {\n      await tx?.done();\n      await db.end();\n    } else {\n      // The stream failed or was closed early, in which case the COPY read\n      // task may be blocked on a stream that is no longer consumed. Close the\n      // connection in the background, forcibly after a timeout.\n      void db\n        .end({timeout: 5})\n        .catch(e => lc.warn?.(`error closing backfill connection`, e));\n    }\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {timeline, resumable, start: startMark}: Progress,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  ctidDecoder: (field: T) => string,\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    // SET LOCAL applies to the (snapshot) transaction of this read task.\n    for (const setting of PHYSICAL_ORDER_SETTINGS) {\n      await sql.unsafe(`SET LOCAL ${setting}`);\n    }\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // The mark of the last row that was streamed, and that of the last row of\n  // the previous message.\n  let current: BackfillProgressMark | undefined = startMark;\n  let previous: BackfillProgressMark | undefined = startMark;\n  const markOf = (progressMark: string) => ({progressMark, timeline});\n\n  // The number of columns of each row, followed by the ctid.\n  const numCols = decoders.length + 1;\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      if (col < decoders.length) {\n        row[col] = field === null ? null : decoders[col](field);\n      } else {\n        const mark = resumable\n          ? ctidToProgressMark(ctidDecoder(field as T))\n          : rowCountMark(status.rows + 1);\n        if (current !== undefined && mark <= current.progressMark) {\n          // This should not happen given the PHYSICAL_ORDER_SETTINGS, but\n          // the correctness of the progress marks depends on it.\n          throw new Error(\n            `backfill rows are not in ctid order ` +\n              `(${mark} after ${current.progressMark})`,\n          );\n        }\n        current = markOf(mark);\n      }\n\n      if (++col === numCols) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {\n      yield {\n        message: {\n          tag: 'backfill',\n          ...backfill,\n          rowValues,\n          status,\n          progressMarks: progressMarks(previous, must(current)),\n        },\n        byteSize: bufferedBytes,\n      };\n      previous = current;\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {\n        tag: 'backfill',\n        ...backfill,\n        rowValues,\n        status,\n        progressMarks: progressMarks(previous, must(current)),\n      },\n      byteSize: bufferedBytes,\n    };\n    previous = current;\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {\n      tag: 'backfill-completed',\n      ...backfill,\n      status,\n      // `previous` is only unset for a backfill from scratch with no rows.\n      progressMarks: previous ? {previous} : {},\n    },\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n        diagnosticsDB: db,\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    // With a timeout, in case a hung slot creation is still pending.\n    await replicationSession.end({timeout: 5});\n  }\n}\n\nfunction validateSchema(\n  lc: LogContext,\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n  progress: Progress;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns).map(([col, {id}]) => [col, id] as const),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    const progress = await getProgress(lc, sql, spec.oid, bf);\n    return {tableSpec: spec, backfill, progress};\n  });\n}\n\n/**\n * Determines the timeline of the table's progress marks (as of the snapshot),\n * and whether the backfill can be resumed from the requested progress.\n *\n * The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest\n * of the transaction) so that its storage cannot be rewritten (by `VACUUM\n * FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which\n * require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note\n * that `pg_relation_filenode()` reflects the table's current storage (only\n * refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at\n * the transaction snapshot. If they differ, the storage was rewritten after the\n * snapshot, and the COPY would scan the new storage with the old snapshot,\n * which is not MVCC-safe for all rewrites (e.g. the table appears empty after\n * a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.\n */\nasync function getProgress(\n  lc: LogContext,\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<Progress> {\n  const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;\n  await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);\n  const [{lockedOID, relkind, snapshotFilenode, filenode}] = await sql<\n    {\n      lockedOID: number | null;\n      relkind: string;\n      snapshotFilenode: string;\n      filenode: string | null;\n    }[]\n  >`\n    SELECT to_regclass(${table})::oid::int8 AS \"lockedOID\",\n           relkind,\n           relfilenode::text AS \"snapshotFilenode\",\n           pg_relation_filenode(oid)::text AS filenode\n      FROM pg_class WHERE oid = ${relationOID}`;\n  if (Number(lockedOID) !== relationOID) {\n    // The (current) table with the name is not the one at the snapshot.\n    throw new SchemaIncompatibilityError(\n      bf,\n      `Table has been renamed or replaced`,\n    );\n  }\n  if (relkind !== 'r' || filenode === null) {\n    // Not a plain table (e.g. a partitioned table), for which ctids are not\n    // unique. Such backfills are not resumable, which is achieved by using a\n    // timeline that is unique to the run.\n    return {\n      timeline: `${relkind}:${randomCharacters(12)}`,\n      resumable: false,\n      start: undefined,\n    };\n  }\n  if (filenode !== snapshotFilenode) {\n    throw new Error(\n      `${bf.table.schema}.${bf.table.name} was rewritten after the backfill ` +\n        `snapshot (relfilenode ${snapshotFilenode} => ${filenode})`,\n    );\n  }\n  const timeline = filenode;\n  const requested = resumePoint(bf);\n  if (requested && requested.timeline !== timeline) {\n    lc.info?.(\n      `restarting backfill from scratch: the requested progress is on ` +\n        `timeline ${requested.timeline} but the table is on ${timeline}`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  if (requested && !(await canResume(sql, relationOID, bf))) {\n    lc.info?.(\n      `restarting backfill from scratch: the table has TOASTed values ` +\n        `and backfilling columns [${Object.keys(bf.columns).join(',')}] ` +\n        `are TOAST-able`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  return {timeline, resumable: true, start: requested};\n}\n\n/**\n * Whether resuming a backfill (in a new snapshot) is guaranteed to deliver\n * the values of every row that the preceding snapshot(s) had not reached.\n *\n * Within a single snapshot, every row is scanned exactly once, since the\n * version visible to the snapshot stays in place while it is held. Across\n * snapshots, however, a row that had not yet been reached (i.e. beyond the\n * resume mark `M`) can be updated between the snapshots, with its new\n * version placed at or before `M` (a non-HOT update into free space in an\n * earlier page, or a HOT update into a lower line pointer of `M`'s page).\n * Neither snapshot then scans the row, and it is only delivered by the\n * replication stream. The stream's UPDATE contains the full row, _except_\n * for unchanged values stored out-of-line in TOAST, which pgoutput omits\n * (unless the table has REPLICA IDENTITY FULL). A backfilling column with\n * such a value would be published without it.\n *\n * Resuming is therefore only done if no backfilling column can be TOASTed\n * (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the\n * table has no TOAST data. The latter is safe because an omitted value must\n * still be in the TOAST relation at the new snapshot (unless its row has since\n * been deleted, which the replication stream does deliver), and conservative\n * because the TOAST relation does not shrink to empty without a heap rewrite,\n * which changes the timeline.\n *\n * Potential optimization for TOAST-able backfills: have the resumed run also\n * deliver the rows at or before `M` that were modified since the snapshot of\n * the run that started the backfill from scratch. That run's snapshot `xmin`\n * (`T_origin`, as an xid8) would be encoded in the timeline, i.e.\n * `timeline = \"<relfilenode>:<T_origin>\"`, and carried over (rather than\n * replaced) by resumed runs, so that:\n * * Within a chain of runs, every row version with `xmin < T_origin` stayed\n *   in place across all of the runs' snapshots and was therefore scanned\n *   exactly once, and all other rows are re-delivered by the resumed run\n *   (`ctid > M OR xmin >= T_origin`).\n * * Continuations from a different chain (i.e. with a newer `T_origin`) are\n *   not accepted by subscribers, and requests from different chains are\n *   merged into a backfill from scratch, as the timelines differ. Note that\n *   using the _latest_ `T` would be incorrect: rows that moved before `M`\n *   between the earlier snapshots would never be re-delivered.\n * The resumed run would scan the whole table (but only deliver the modified\n * rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which\n * requires falling back to a backfill from scratch once the current xid is\n * 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the\n * `xmin` system column returning the raw xmin of frozen tuples, which should\n * be verified.\n */\nasync function canResume(\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<boolean> {\n  const [{toastable, toasted}] = await sql<\n    {toastable: boolean; toasted: boolean}[]\n  >`\n    SELECT\n      EXISTS (\n        SELECT 1 FROM pg_attribute\n          WHERE attrelid = ${relationOID}\n            AND attname IN ${sql(Object.keys(bf.columns))}\n            AND attstorage <> 'p'\n      ) AS toastable,\n      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted\n      FROM pg_class WHERE oid = ${relationOID}`;\n  return !toastable || !toasted;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,UAAU;AAChB,IAAM,eAAe;AAErB,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,QAAQ,KAAK,IAAI;CAC/B,OAAO,aAAa,iBAAiB,MAAM;CAC3C,OAAO,GAAG,MAAM,GAAG,SAAS,IAAI,GAAG,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,GAAG;AAClE;;AAGA,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,aAAa,KAAK,IAAI;CACpC,OAAO,aAAa,+BAA+B,MAAM;CACzD,OAAO,IAAI,OAAO,MAAM,EAAE,EAAE,GAAG,OAAO,MAAM,EAAE,EAAE;AAClD;AAEA,SAAS,cACP,UACA,SACA;CACA,OAAO,WAAW;EAAC;EAAU;CAAO,IAAI,EAAC,QAAO;AAClD;;AAGA,IAAM,cAAc;;;;;;;;;;;;AAapB,IAAM,0BAA0B;CAC9B;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aAAa,MAAsB;CAC1C,OAAO,OAAO,IAAI,EAAE,SAAS,IAAI,GAAG;AACtC;AA4BA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EACJ,sBAAsB,0BACtB,WAAW,OACX,4BACE;CACJ,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI,YAAY;CAChB,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,0BAA0B;EAChC,MAAM,EAAC,WAAW,UAAU,aAAY,MAAM,eAC5C,IACA,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EAMpD,MAAM,QACJ,SAAS,aAAa,SAAS,QAC3B,EACE,OAAe,WAAW,mBAAmB,SAAS,MAAM,YAAY,EAAE,QAC5E,IACA,KAAA;EACN,MAAM,QAAQ,uBACZ,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,KAAK,KAAI,QAAO,GAAG,GAAG,CAAC,GAAG,WAAW,GACzC,KACF;EACA,IAAI,SAAS,OACX,GAAG,OACD,0BAA0B,SAAS,MAAM,aAAa,aACvC,SAAS,SAAS,EACnC;EAGF,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,IACA,SAAiB,MAClB,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,sBAAsB,WAAW,IAAI,GAAG,WAAW,GACvD,KACF;GAEA,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,IACA,QAAgB,IAAI,SAAS,MAAM,GACpC,mBACF;EACF;EACA,YAAY;CACd,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EACZ,IAAI,WAAW;GACb,MAAM,IAAI,KAAK;GACf,MAAM,GAAG,IAAI;EACf,OAIE,GACG,IAAI,EAAC,SAAS,EAAC,CAAC,EAChB,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAEnE;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,UAAU,WAAW,OAAO,aAC7B,EACE,cACA,iBAEF,aACA,QACA,UACA,aACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAE/C,KAAK,MAAM,WAAW,yBACpB,MAAM,IAAI,OAAO,aAAa,SAAS;EAEzC,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAIA,IAAI,UAA4C;CAChD,IAAI,WAA6C;CACjD,MAAM,UAAU,kBAA0B;EAAC;EAAc;CAAQ;CAGjE,MAAM,UAAU,SAAS,SAAS;CAGlC,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,MAAM,SAAS,QACjB,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;QACjD;IACL,MAAM,OAAO,YACT,mBAAmB,YAAY,KAAU,CAAC,IAC1C,aAAa,OAAO,OAAO,CAAC;IAChC,IAAI,YAAY,KAAA,KAAa,QAAQ,QAAQ,cAG3C,MAAM,IAAI,MACR,wCACM,KAAK,SAAS,QAAQ,aAAa,EAC3C;IAEF,UAAU,OAAO,IAAI;GACvB;GAEA,IAAI,EAAE,QAAQ,SAAS;IACrB,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,uBAAuB,UAAU,SAAS,GAAG;GAChE,MAAM;IACJ,SAAS;KACP,KAAK;KACL,GAAG;KACH;KACA;KACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;IACtD;IACA,UAAU;GACZ;GACA,WAAW;GACX;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IACP,KAAK;IACL,GAAG;IACH;IACA;IACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;GACtD;GACA,UAAU;EACZ;EACA,WAAW;EACX;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GACP,KAAK;GACL,GAAG;GACH;GAEA,eAAe,WAAW,EAAC,SAAQ,IAAI,CAAC;EAC1C;EACA,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;GACX,eAAe;EACjB,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EAER,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;CAC3C;AACF;AAEA,SAAS,eACP,IACA,IACA,cACA,IACA,WAKC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAAC,KAAK,EAAE,CAAU,CACvE;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAaA,OAAO;GAAC,WAAW;GAAM,UAAA;IAXvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAGuB;GAAU,UAAA,MADZ,YAAY,IAAI,KAAK,KAAK,KAAK,EAAE;EACb;CAC7C,CAAC;AACH;;;;;;;;;;;;;;;;AAiBA,eAAe,YACb,IACA,KACA,aACA,IACmB;CACnB,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,MAAM,EAAE,GAAG,GAAG,GAAG,MAAM,IAAI;CACxD,MAAM,IAAI,OAAO,cAAc,MAAM,sBAAsB;CAC3D,MAAM,CAAC,EAAC,WAAW,SAAS,kBAAkB,cAAa,MAAM,GAOhE;yBACsB,MAAM;;;;kCAIG;CAChC,IAAI,OAAO,SAAS,MAAM,aAExB,MAAM,IAAI,2BACR,IACA,oCACF;CAEF,IAAI,YAAY,OAAO,aAAa,MAIlC,OAAO;EACL,UAAU,GAAG,QAAQ,GAAG,iBAAiB,EAAE;EAC3C,WAAW;EACX,OAAO,KAAA;CACT;CAEF,IAAI,aAAa,kBACf,MAAM,IAAI,MACR,GAAG,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,0DACT,iBAAiB,MAAM,SAAS,EAC7D;CAEF,MAAM,WAAW;CACjB,MAAM,YAAY,YAAY,EAAE;CAChC,IAAI,aAAa,UAAU,aAAa,UAAU;EAChD,GAAG,OACD,2EACc,UAAU,SAAS,uBAAuB,UAC1D;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,IAAI,aAAa,CAAE,MAAM,UAAU,KAAK,aAAa,EAAE,GAAI;EACzD,GAAG,OACD,2FAC8B,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,iBAElE;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,OAAO;EAAC;EAAU,WAAW;EAAM,OAAO;CAAS;AACrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,eAAe,UACb,KACA,aACA,IACkB;CAClB,MAAM,CAAC,EAAC,WAAW,aAAY,MAAM,GAEpC;;;;6BAI0B,YAAY;6BACZ,IAAI,OAAO,KAAK,GAAG,OAAO,CAAC,EAAE;;;;kCAIxB;CAChC,OAAO,CAAC,aAAa,CAAC;AACxB"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -0,0 +1,65 @@
+import type { Readable } from 'node:stream';
+import type { LogContext } from '@rocicorp/logger';
+import { type PostgresDB } from '../../../types/pg.ts';
+import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
+import { type ConstrainingPurgeLock, type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
+import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
+import { type LSN } from './lsn.ts';
+import { type ReplicaState } from './schema/shard.ts';
+export type InitializeOptions = ReplicaOptions & {
+    slotPerReplica: boolean | undefined;
+    inactiveReplicaGracePeriodMs: number;
+    /**
+     * Whether this replication-manager maintains the PG change log, i.e. acks
+     * the replication slot only after transactions are stored in it. Recorded
+     * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is
+     * taken over.
+     */
+    pgChangeLog: boolean;
+};
+/**
+ * Initializes a Postgres change source, including the initial sync of the
+ * replica, before streaming changes from the corresponding logical replication
+ * stream.
+ */
+export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, pgChangeLog, }?: InitializeOptions, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
+/**
+ * The watermark below which the replication stream cannot (correctly) be
+ * started, i.e. from which it would skip transactions that the change log
+ * does not contain. It reflects the order in which each mode stores a
+ * transaction and then acks it to the slot, such that a correctly initialized
+ * stream always starts at or past it:
+ *
+ * * After an initial sync, the replica's watermark, which is the position
+ *   at which the new slot was created, and the change log is (re)initialized
+ *   at it.
+ * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored
+ *   replica's watermark and the slot's position. Transactions are only acked
+ *   once backed up (so a skipped interval ends at or before the replica), and
+ *   if the change log is behind the slot it is re-initialized from the
+ *   replica. Note that the slot can be past the replica (acked for keepalives,
+ *   with nothing skipped), and the replica past the change log (forwarded
+ *   before being stored, and streamed again from the slot), but the stream
+ *   never starts below both. (With RMv1.5's purge lock, the change log's head
+ *   is at or past the slot, so it does not lower the minimum.)
+ * * With a shared slot (RMv1), or a resumed slot that was acked only once
+ *   transactions were stored in the change log (RMv1.5), the change log's
+ *   head when it was purge locked, as the stream resumes from the head, which
+ *   only advances. The slot is not a reference: a shared slot is acked by its
+ *   owner until it is taken over, and a resumed RMv1.5 slot can be past the
+ *   head for acked keepalives. Without a purge lock (i.e. an empty change
+ *   log), there is no floor.
+ */
+export declare function minStartWatermark(replicaWatermark: string, initialSynced: boolean, slotWatermark: string | undefined, purgeLock: ConstrainingPurgeLock | null): string | undefined;
+export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
+    restoreFrom: ReplicaState;
+    replicateTo: ReplicaState;
+    /** The position of the slot from which replication resumes. */
+    slotLSN: LSN;
+    /**
+     * The session holding the destination slot active until it is taken
+     * over by the PostgresChangeSource. Destroying it releases the slot.
+     */
+    slotSession: Readable;
+} | undefined>;
+//# sourceMappingURL=change-source-init.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;IAErC;;;;;OAKG;IACH,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,EAC5B,WAAW,GACZ,GAAE,iBAMF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CA4I3B;AA6KD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,wBAAgB,iBAAiB,CAC/B,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,qBAAqB,GAAG,IAAI,GACtC,MAAM,GAAG,SAAS,CAQpB;AAsBD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CA+LA"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js

@@ -0,0 +1,312 @@
+import { assert } from "../../../../../shared/src/asserts.js";
+import { deepEqual } from "../../../../../shared/src/json.js";
+import { promiseVoid } from "../../../../../shared/src/resolved-promises.js";
+import { sleep } from "../../../../../shared/src/sleep.js";
+import { must } from "../../../../../shared/src/must.js";
+import { Database } from "../../../../../zqlite/src/db.js";
+import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
+import { StatementRunner } from "../../../db/statements.js";
+import { connectPgClient } from "../../../types/pg.js";
+import { toBigInt, toStateVersionString } from "./lsn.js";
+import "./schema/replica-stage-enum.js";
+import { dropShard, getActiveReplicas, getReplicaAtVersion, getRestoreCandidates, initRestoreReplica, internalPublicationPrefix } from "./schema/shard.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { restoreReplica } from "../common/replica-restore.js";
+import { initReplica } from "../common/replica-schema.js";
+import { endSession } from "./logical-replication/stream.js";
+import { claimSlotForResumption, createReplicaAndSlot } from "./replication-slots.js";
+import { ensureShardSchema } from "./schema/init.js";
+import { initialSync } from "./initial-sync.js";
+import { PostgresChangeSource } from "./change-source.js";
+//#region ../zero-cache/src/services/change-source/pg/change-source-init.ts
+/**
+* Initializes a Postgres change source, including the initial sync of the
+* replica, before streaming changes from the corresponding logical replication
+* stream.
+*/
+async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, pgChangeLog } = {
+	epoch: 0,
+	slotPerReplica: false,
+	backupV5: true,
+	inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,
+	pgChangeLog: false
+}, streamInboundTimeoutMs) {
+	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
+	try {
+		await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
+		const { replica: restoredReplica, purgeLock, pgChangeLogBehindSlot, resumableBackup, slotWatermark } = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
+		let initialSyncedReplica;
+		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
+			await restoreOptions.cleanup?.release();
+			await purgeLock?.release();
+			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, {
+				epoch,
+				backupV5,
+				pgChangeLog
+			}, restoreOptions.cleanup);
+		});
+		const replica = new Database(lc, replicaDbFile);
+		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
+		replica.close();
+		const { upstreamReplica, pgVersion } = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
+		const newBackupLineage = backupV5 && !resumableBackup;
+		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? resumableBackup?.backupPath ?? String(Date.now()) : restoredReplica?.backupPath ?? null;
+		return {
+			subscriptionState,
+			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, pgVersion, {
+				backupPath,
+				backupV5,
+				pgChangeLog
+			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs, minStartWatermark(subscriptionState.watermark, initialSyncedReplica !== void 0, slotWatermark, purgeLock)),
+			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
+			replicaID: upstreamReplica.id,
+			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null),
+			pgChangeLogBehindSlot,
+			newBackupLineage
+		};
+	} finally {
+		await db.end();
+	}
+}
+async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, acquirePurgeLock }) {
+	const lock = await acquirePurgeLock?.();
+	assert(lock !== "behind-slot", "unexpected purge-lock result");
+	const purgeLock = lock ?? null;
+	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (purgeLock?.replicaVersion ?? generation));
+	if (replicas.length === 0) {
+		lc.info?.(`no suitable replicas to restore from`, { replicas });
+		return {
+			replica: void 0,
+			purgeLock
+		};
+	}
+	const [replica] = replicas;
+	if (litestream?.backupURL) {
+		const constraints = restoreConstraints(purgeLock, replica);
+		const { backupURL: backupBaseURL } = litestream;
+		const { slot, backupPath, confirmedFlushLsn } = replica;
+		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
+		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
+		return {
+			replica,
+			purgeLock,
+			resumableBackup: await restoreReplica(lc, {
+				...litestream,
+				backupURL
+			}, replicaFile, constraints, litestream.forkBackup ? {
+				type: "fork",
+				baseURL: backupBaseURL
+			} : void 0)
+		};
+	}
+	return {
+		replica,
+		purgeLock
+	};
+}
+async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, acquirePurgeLock, cleanup }, gracePeriodMs) {
+	const result = await getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs);
+	if (!result) return {
+		replica: void 0,
+		purgeLock: null
+	};
+	const { restoreFrom, replicateTo, slotLSN, slotSession } = result;
+	const slotWatermark = restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog ? void 0 : toStateVersionString(slotLSN);
+	cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () => slotSession.destroy());
+	const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;
+	const pgChangeLogBehindSlot = lock === "behind-slot";
+	const purgeLock = lock === "behind-slot" ? null : lock;
+	const constraints = restoreConstraints(purgeLock, restoreFrom);
+	if (litestream?.backupURL) {
+		const { backupURL: backupBaseURL } = litestream;
+		const { slot, backupPath, confirmedFlushLsn } = restoreFrom;
+		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
+		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, {
+			restoreFrom,
+			replicateTo
+		});
+		return {
+			replica: replicateTo,
+			purgeLock,
+			pgChangeLogBehindSlot,
+			resumableBackup: await restoreReplica(lc, {
+				...litestream,
+				backupURL
+			}, replicaFile, constraints, litestream.forkBackup ? {
+				type: "fork",
+				baseURL: backupBaseURL
+			} : void 0),
+			slotWatermark
+		};
+	}
+	return {
+		replica: replicateTo,
+		purgeLock,
+		pgChangeLogBehindSlot,
+		slotWatermark
+	};
+}
+/**
+* The watermark below which the replication stream cannot (correctly) be
+* started, i.e. from which it would skip transactions that the change log
+* does not contain. It reflects the order in which each mode stores a
+* transaction and then acks it to the slot, such that a correctly initialized
+* stream always starts at or past it:
+*
+* * After an initial sync, the replica's watermark, which is the position
+*   at which the new slot was created, and the change log is (re)initialized
+*   at it.
+* * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored
+*   replica's watermark and the slot's position. Transactions are only acked
+*   once backed up (so a skipped interval ends at or before the replica), and
+*   if the change log is behind the slot it is re-initialized from the
+*   replica. Note that the slot can be past the replica (acked for keepalives,
+*   with nothing skipped), and the replica past the change log (forwarded
+*   before being stored, and streamed again from the slot), but the stream
+*   never starts below both. (With RMv1.5's purge lock, the change log's head
+*   is at or past the slot, so it does not lower the minimum.)
+* * With a shared slot (RMv1), or a resumed slot that was acked only once
+*   transactions were stored in the change log (RMv1.5), the change log's
+*   head when it was purge locked, as the stream resumes from the head, which
+*   only advances. The slot is not a reference: a shared slot is acked by its
+*   owner until it is taken over, and a resumed RMv1.5 slot can be past the
+*   head for acked keepalives. Without a purge lock (i.e. an empty change
+*   log), there is no floor.
+*/
+function minStartWatermark(replicaWatermark, initialSynced, slotWatermark, purgeLock) {
+	if (initialSynced) return replicaWatermark;
+	if (slotWatermark !== void 0) return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;
+	return purgeLock?.headWatermark;
+}
+function restoreConstraints(purgeLock, { generation }) {
+	return purgeLock ?? {
+		replicaVersion: generation,
+		minWatermark: ""
+	};
+}
+async function currentWalLSN(sql) {
+	const [{ lsn }] = await sql`SELECT pg_current_wal_lsn() AS lsn`;
+	return lsn;
+}
+var REPLICA_POLL_INTERVAL_MS = 500;
+var PROGRESS_LOG_INTERVAL_MS = 5e3;
+var SLOW_PROGRESS_LOG_INTERVAL_MS = 6e4;
+var DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 2e4;
+async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs, pollIntervalMs = REPLICA_POLL_INTERVAL_MS) {
+	const inactiveSince = /* @__PURE__ */ new Map();
+	let destination;
+	let forkLSN;
+	const lastLogged = /* @__PURE__ */ new Map();
+	const shouldLog = (key, intervalMs) => {
+		const now = Date.now();
+		const last = lastLogged.get(key);
+		if (last !== void 0 && now - last < intervalMs) return false;
+		lastLogged.set(key, now);
+		return true;
+	};
+	try {
+		for (let i = 0;; i++) {
+			if (i > 0) await sleep(pollIntervalMs);
+			const replicas = await getRestoreCandidates(sql, shard, epoch);
+			if (shouldLog("replicas", PROGRESS_LOG_INTERVAL_MS)) lc.info?.(`current replicas at epoch ${epoch}`, { replicas });
+			if (replicas.length === 0) {
+				lc.info?.(`no suitable replicas to restore from`, { replicas });
+				await endSession(destination?.initialSession);
+				return;
+			}
+			for (const replica of replicas) {
+				if (replica.active) inactiveSince.delete(replica.id);
+				else if (!inactiveSince.has(replica.id)) inactiveSince.set(replica.id, Date.now());
+				if (replica.stage === 0) {
+					if (shouldLog(`initial-sync:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)) lc.info?.(`waiting for initial sync of ${replica.id}`, { replica });
+					break;
+				}
+				if (replica.stage === 1) {
+					const { active, confirmedFlushLsn } = replica;
+					if (active) {
+						destination ??= await createReplicaAndSlot(lc, sql, "fork-replica-session", shard, epoch, Date.now().toString(), slotFailover, {
+							backupPath: null,
+							backupV5: true,
+							pgChangeLog: false
+						}, () => promiseVoid, 2);
+						forkLSN ??= await currentWalLSN(sql);
+						if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {
+							if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) lc.info?.(`waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`, { replica });
+							break;
+						}
+					}
+					if (destination && forkLSN && toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)) {
+						lc.info?.(`forking replica ${replica.id}@${replica.slot}`, { replica });
+						return {
+							restoreFrom: replica,
+							replicateTo: must(await initRestoreReplica(sql, shard, {
+								sourceID: replica.id,
+								destID: destination.replica.id
+							}), `replica ${destination.replica.id} was deleted`),
+							slotLSN: destination.slot.consistent_point,
+							slotSession: destination.initialSession
+						};
+					}
+				}
+				if (replica.stage === 2 && replica.active && replica.id !== destination?.replica.id) {
+					if (shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)) lc.info?.(`waiting for replica ${replica.id}@${replica.slot} being restored by another task`, { replica });
+				}
+				if (replica.backupPath && !replica.active) {
+					const now = Date.now();
+					const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);
+					if (inactiveMs >= gracePeriodMs || shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) lc.info?.(`replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`, { replica });
+					if (inactiveMs >= gracePeriodMs) {
+						const reserved = await claimSlotForResumption(lc, sql, shard, "resume-replica", replica);
+						if (!reserved) lc.warn?.(`unable to resume replica ${replica.id}@${replica.slot}`);
+						else {
+							lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);
+							await endSession(destination?.initialSession);
+							return {
+								restoreFrom: replica,
+								replicateTo: reserved.replica,
+								slotLSN: replica.confirmedFlushLsn,
+								slotSession: reserved.reservation
+							};
+						}
+					}
+				}
+			}
+		}
+	} catch (e) {
+		await endSession(destination?.initialSession);
+		throw e;
+	}
+}
+async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
+	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
+	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
+	const requested = shard.publications.toSorted();
+	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
+	if (!deepEqual(requested, replicated)) {
+		lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
+		await sql.unsafe(dropShard(shard.appID, shard.shardNum));
+		throw new AutoResetSignal(`Requested publications [${requested}] do not match configured publications: [${replicated}]`);
+	}
+	if (!deepEqual(upstreamReplica.publications, subscribed)) throw new AutoResetSignal(`Upstream publications [${upstreamReplica.publications}] do not match subscribed publications [${subscribed}]`);
+	const exists = await sql`
+    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};
+  `.values();
+	if (exists.length !== subscribed.length) throw new AutoResetSignal(`Upstream publications [${exists.flat()}] do not contain all subscribed publications [${subscribed}]`);
+	const { slot } = upstreamReplica;
+	const result = await sql`
+    SELECT restart_lsn as "restartLSN", wal_status as "walStatus" FROM pg_replication_slots
+      WHERE slot_name = ${slot}`;
+	if (result.length === 0) throw new AutoResetSignal(`replication slot ${slot} is missing`);
+	const [{ restartLSN, walStatus }] = result;
+	if (restartLSN === null || walStatus === "lost") throw new AutoResetSignal(`replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`);
+	const [{ pgVersion }] = await sql`
+    SELECT current_setting('server_version_num')::int as "pgVersion"`;
+	return {
+		upstreamReplica,
+		pgVersion
+	};
+}
+//#endregion
+export { initializePostgresChangeSource };
+
+//# sourceMappingURL=change-source-init.js.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n  type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {endSession} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n\n  /**\n   * Whether this replication-manager maintains the PG change log, i.e. acks\n   * the replication slot only after transactions are stored in it. Recorded\n   * with the replica (see `pgChangeLog` in `ReplicaState`) when its slot is\n   * taken over.\n   */\n  pgChangeLog: boolean;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n    pgChangeLog,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n    pgChangeLog: false,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5, pgChangeLog},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to its own\n    // backup lineage.\n    const newBackupLineage = backupV5 && !resumableBackup;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use the lineage prepared by the restore, or a new, unique\n        // path when backing up with litestream v5. This will be recorded in the\n        // replicas table by the PostgresChangeSource.\n        backupV5\n        ? (resumableBackup?.backupPath ?? String(Date.now()))\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5, pgChangeLog},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n      minStartWatermark(\n        subscriptionState.watermark,\n        initialSyncedReplica !== undefined,\n        slotWatermark,\n        purgeLock,\n      ),\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n  resumableBackup?: ResumableBackup | undefined;\n  /**\n   * The position (as a watermark) of the replication slot from which\n   * replication resumes, when it was created (for a fork) or claimed (for a\n   * resumption), i.e. while it is held by this process. Absent for a shared\n   * slot (RMv1), whose position is advanced by its current owner until it is\n   * taken over, and for a resumed slot that was acked only once transactions\n   * were stored in the PG change-log (RMv1.5), whose position may be past the\n   * change-log's head for acked keepalives.\n   */\n  slotWatermark?: string | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n    );\n    return {replica, purgeLock, resumableBackup};\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // A resumed slot (i.e. of the same replica) that was acked only once\n  // transactions were stored in the PG change-log (RMv1.5) cannot have been\n  // acked past a transaction missing from the change-log. Its position past\n  // the change-log's head only reflects acked keepalives, so it is not\n  // compared with the head (see below).\n  const resumedAfterPgChangeLog =\n    restoreFrom.id === replicateTo.id && restoreFrom.pgChangeLog;\n  const slotWatermark = resumedAfterPgChangeLog\n    ? undefined\n    : toStateVersionString(slotLSN);\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for keepalives (e.g. for changes outside of the\n  // publication, or on an idle upstream). If the slot was acked by a\n  // replication-manager that maintains the change-log, it is therefore not\n  // checked against the slot's position, which would otherwise re-initialize\n  // the change-log unnecessarily, resetting subscribers behind the replica.\n  // Otherwise (e.g. RMv2, or unknown), it is: the change-log is then\n  // re-initialized from the replica if it does not contain the replica's\n  // changes.\n  const lock = acquirePurgeLock ? await acquirePurgeLock(slotWatermark) : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n    );\n    return {\n      replica: replicateTo,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n      slotWatermark,\n    };\n  }\n  return {\n    replica: replicateTo,\n    purgeLock,\n    pgChangeLogBehindSlot,\n    slotWatermark,\n  };\n}\n\n/**\n * The watermark below which the replication stream cannot (correctly) be\n * started, i.e. from which it would skip transactions that the change log\n * does not contain. It reflects the order in which each mode stores a\n * transaction and then acks it to the slot, such that a correctly initialized\n * stream always starts at or past it:\n *\n * * After an initial sync, the replica's watermark, which is the position\n *   at which the new slot was created, and the change log is (re)initialized\n *   at it.\n * * With a slot per replica (RMv1.5 and RMv2), the minimum of the restored\n *   replica's watermark and the slot's position. Transactions are only acked\n *   once backed up (so a skipped interval ends at or before the replica), and\n *   if the change log is behind the slot it is re-initialized from the\n *   replica. Note that the slot can be past the replica (acked for keepalives,\n *   with nothing skipped), and the replica past the change log (forwarded\n *   before being stored, and streamed again from the slot), but the stream\n *   never starts below both. (With RMv1.5's purge lock, the change log's head\n *   is at or past the slot, so it does not lower the minimum.)\n * * With a shared slot (RMv1), or a resumed slot that was acked only once\n *   transactions were stored in the change log (RMv1.5), the change log's\n *   head when it was purge locked, as the stream resumes from the head, which\n *   only advances. The slot is not a reference: a shared slot is acked by its\n *   owner until it is taken over, and a resumed RMv1.5 slot can be past the\n *   head for acked keepalives. Without a purge lock (i.e. an empty change\n *   log), there is no floor.\n */\n// Exported for testing.\nexport function minStartWatermark(\n  replicaWatermark: string,\n  initialSynced: boolean,\n  slotWatermark: string | undefined,\n  purgeLock: ConstrainingPurgeLock | null,\n): string | undefined {\n  if (initialSynced) {\n    return replicaWatermark;\n  }\n  if (slotWatermark !== undefined) {\n    return replicaWatermark < slotWatermark ? replicaWatermark : slotWatermark;\n  }\n  return purgeLock?.headWatermark;\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 500;\n// Polling is frequent so that a fork proceeds as soon as the source replica\n// passes the fork LSN; logging is throttled by time rather than per poll.\nconst PROGRESS_LOG_INTERVAL_MS = 5_000;\nconst SLOW_PROGRESS_LOG_INTERVAL_MS = 60_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n  const lastLogged = new Map<string, number>();\n  const shouldLog = (key: string, intervalMs: number) => {\n    const now = Date.now();\n    const last = lastLogged.get(key);\n    if (last !== undefined && now - last < intervalMs) {\n      return false;\n    }\n    lastLogged.set(key, now);\n    return true;\n  };\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(sql, shard, epoch);\n      if (shouldLog('replicas', PROGRESS_LOG_INTERVAL_MS)) {\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas});\n      }\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        await endSession(destination?.initialSession);\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (\n            shouldLog(\n              `initial-sync:${replica.id}`,\n              SLOW_PROGRESS_LOG_INTERVAL_MS,\n            )\n          ) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n                // The slot is not acked until it is taken over, which sets\n                // this. Until then, it is unknown.\n                pgChangeLog: false,\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              if (shouldLog(`fork:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)) {\n                lc.info?.(\n                  `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                  {replica},\n                );\n              }\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (\n            shouldLog(`restoring:${replica.id}`, SLOW_PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          if (\n            inactiveMs >= gracePeriodMs ||\n            shouldLog(`inactive:${replica.id}`, PROGRESS_LOG_INTERVAL_MS)\n          ) {\n            lc.info?.(\n              `replica ${replica.id}@${replica.slot} has been inactive for ${inactiveMs}ms`,\n              {replica},\n            );\n          }\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              await endSession(destination?.initialSession);\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    await endSession(destination?.initialSession);\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,8BACA,gBACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;CAC9B,aAAa;AACf,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,iBACA,kBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;IAAU;GAAW,GAC7B,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EA2BtC,OAAO;GACL;GACA,cAAA,IA3BuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;IAAU;GAAW,GAClC,SACA,qBACA,YAAY,UACZ,wBACA,kBACE,kBAAkB,WAClB,yBAAyB,KAAA,GACzB,eACA,SACF,CAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;EACF;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAqBA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAUA,OAAO;GAAC;GAAS;GAAW,iBAAA,MATE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAQzD,MAAM,gBADJ,YAAY,OAAO,YAAY,MAAM,YAAY,cAE/C,KAAA,IACA,qBAAqB,OAAO;CAKhC,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAoBA,MAAM,OAAO,mBAAmB,MAAM,iBAAiB,aAAa,IAAI;CACxE,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAUA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAb4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,CACN;GAME;EACF;CACF;CACA,OAAO;EACL,SAAS;EACT;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,kBACd,kBACA,eACA,eACA,WACoB;CACpB,IAAI,eACF,OAAO;CAET,IAAI,kBAAkB,KAAA,GACpB,OAAO,mBAAmB,gBAAgB,mBAAmB;CAE/D,OAAO,WAAW;AACpB;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AAGjC,IAAM,2BAA2B;AACjC,IAAM,gCAAgC;AACtC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CACJ,MAAM,6BAAa,IAAI,IAAoB;CAC3C,MAAM,aAAa,KAAa,eAAuB;EACrD,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAO,WAAW,IAAI,GAAG;EAC/B,IAAI,SAAS,KAAA,KAAa,MAAM,OAAO,YACrC,OAAO;EAET,WAAW,IAAI,KAAK,GAAG;EACvB,OAAO;CACT;CAEA,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,KAAK,OAAO,KAAK;GAC7D,IAAI,UAAU,YAAY,wBAAwB,GAChD,GAAG,OAAO,6BAA6B,SAAS,EAAC,SAAQ,CAAC;GAE5D,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,MAAM,WAAW,aAAa,cAAc;IAC5C;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IACE,UACE,gBAAgB,QAAQ,MACxB,6BACF,GAEA,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;OAGV,aAAa;MACf,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,IAAI,UAAU,QAAQ,QAAQ,MAAM,wBAAwB,GAC1D,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OAEF;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAIlC,UAAU,aAAa,QAAQ,MAAM,6BAA6B,GAElE,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,IACE,cAAc,iBACd,UAAU,YAAY,QAAQ,MAAM,wBAAwB,GAE5D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,yBAAyB,WAAW,KAC1E,EAAC,QAAO,CACV;KAEF,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,MAAM,WAAW,aAAa,cAAc;OAC5C,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,MAAM,WAAW,aAAa,cAAc;EAC5C,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}