@rocicorp/zero 1.5.0-canary.4 → 1.6.0-canary.1

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  createReplicationSlot,\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n    opts;\n  const db = pgClient(lc, upstreamURI, 'backfill-stream', {\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    const stmts = makeDownloadStatements(tableSpec, cols);\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        makeBinarySelectExprs(tableSpec, cols),\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(copyCommand).readable(),\n  );\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const field of parser.parse(chunk)) {\n      row[col] = field === null ? null : decoders[col](field);\n\n      if (++col === decoders.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, tempSlot);\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported;\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${tempSlot}'`,\n    );\n    lc.error?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,SAAS,IAAI,aAAa,mBAAmB,GACrD,iBAAiB,MACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EAGD,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;EACrD,MAAM,QAAQ,uBAAuB,WAAW,KAAK;AAErD,MAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;AAClE,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,WAAW,EACf,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ;AAClE,YAAQ,SAAiB,OAAO,KAAK;KACrC,EACF,oBACD;SACI;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,KAAK,CACvC;AAED,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,kBAAkB,EACtB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,KAAK,GAClC,kBAAkB,KAAK,GACvB;AACJ,YAAQ,QAAgB,QAAQ,IAAI;KACpC,EACF,oBACD;;UAEI,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;AAGrD,QACE,CAAC,mBAAmB,KAAK,YAAY,EACrC,2CAA2C,cAC5C;CACD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,QACD,CACF;CACD,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,YAAY,CAAC,UAAU,CACnC;CAED,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;CAC5D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE;AACvC,OAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,MAAM;AAEvD,OAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;AAC3C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CACF;CACD,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;EAE/D,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;GAAK,CAAC,CAAC,IAAI,GAAG;AAClE,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,QAAQ,sCAAsC,EAAE;AACnD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}
+{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n    opts;\n  const db = pgClient(lc, upstreamURI, 'backfill-stream', {\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    const stmts = makeDownloadStatements(tableSpec, cols);\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        makeBinarySelectExprs(tableSpec, cols),\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(copyCommand).readable(),\n  );\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const field of parser.parse(chunk)) {\n      row[col] = field === null ? null : decoders[col](field);\n\n      if (++col === decoders.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {slotName});\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported;\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${slotName}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${slotName}'`,\n    );\n    lc.warn?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,SAAS,IAAI,aAAa,mBAAmB,GACrD,iBAAiB,MACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EAGD,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;EACrD,MAAM,QAAQ,uBAAuB,WAAW,KAAK;AAErD,MAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;AAClE,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,WAAW,EACf,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ;AAClE,YAAQ,SAAiB,OAAO,KAAK;KACrC,EACF,oBACD;SACI;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,KAAK,CACvC;AAED,UAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,kBAAkB,EACtB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,KAAK,GAClC,kBAAkB,KAAK,GACvB;AACJ,YAAQ,QAAgB,QAAQ,IAAI;KACpC,EACF,oBACD;;UAEI,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;AAGrD,QACE,CAAC,mBAAmB,KAAK,YAAY,EACrC,2CAA2C,cAC5C;CACD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,QACD,CACF;CACD,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,YAAY,CAAC,UAAU,CACnC;CAED,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;CAC5D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,SAAS,OAAO,MAAM,MAAM,EAAE;AACvC,OAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,MAAM;AAEvD,OAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,QAAO,CAAC;AAC3C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CACF;CACD,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,EAAC,UAAS,CAAC;EAEjE,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;GAAK,CAAC,CAAC,IAAI,GAAG;AAClE,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,OAAO,sCAAsC,EAAE;AAClD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAGhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAuBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AA4cD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAqzBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAGhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAyBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AA+ZD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAmzBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -1,7 +1,6 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../../shared/src/json.js";
 import { AbortError } from "../../../../../shared/src/abort-error.js";
-import { sleep } from "../../../../../shared/src/sleep.js";
 import { parse, valita_exports } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
 import { mapValues } from "../../../../../shared/src/objects.js";
@@ -15,24 +14,25 @@ import { StatementRunner } from "../../../db/statements.js";
 import { isPostgresError, pgClient } from "../../../types/pg.js";
 import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
 import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
-import { runTx } from "../../../db/run-transaction.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { replicationEventSchema } from "./schema/ddl.js";
 import { validate } from "./schema/validation.js";
-import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, legacyReplicationSlot, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression } from "./schema/shard.js";
+import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
+import { dropOldReplicasAndSlots } from "./replication-slots.js";
 import { updateShardSchema } from "./schema/init.js";
 import { initialSync } from "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
 import { subscribe } from "./logical-replication/stream.js";
 import { nanoid } from "nanoid";
 import postgres from "postgres";
-import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE, PG_OBJECT_IN_USE } from "@drdgvhbh/postgres-error-codes";
+import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
+var REPLICA_SLOT_CLEANUP_INTERVAL_MS = 3e4;
 /**
 * Initializes a Postgres change source, including the initial sync of the
 * replica, before streaming changes from the corresponding logical replication
@@ -107,6 +107,7 @@ var PostgresChangeSource = class {
 	}
 	async stop() {
 		this.#lagReporter?.stop();
+		clearTimeout(this.#cleanupTimer);
 		await this.#db.end();
 	}
 	async startLagReporter() {
@@ -121,9 +122,9 @@ var PostgresChangeSource = class {
 		return null;
 	}
 	async startStream(clientWatermark, backfillRequests = []) {
-		const { slot } = this.#replica;
-		await this.#stopExistingReplicationSlotSubscribers(slot);
+		await this.#stopExistingReplicationSlotSubscriber();
 		const config = await getInternalShardConfig(this.#db, this.#shard);
+		const { slot } = this.#replica;
 		this.#lc.info?.(`starting replication stream@${slot}`);
 		return this.#startStream(slot, clientWatermark, config, backfillRequests);
 	}
@@ -214,73 +215,55 @@ var PostgresChangeSource = class {
 		}
 	}
 	/**
-	* Stops replication slots associated with this shard, and returns
-	* a `cleanup` task that drops any slot other than the specified
-	* `slotToKeep`.
-	*
-	* Note that replication slots created after `slotToKeep` (as indicated by
-	* the timestamp suffix) are preserved, as those are newly syncing replicas
-	* that will soon take over the slot.
+	* Stops replication slots associated with this shard, and asynchronously
+	* runs a cleanup task that drops older replicas and slots.
 	*/
-	async #stopExistingReplicationSlotSubscribers(slotToKeep) {
-		const slotExpression = replicationSlotExpression(this.#shard);
-		const legacySlotName = legacyReplicationSlot(this.#shard);
-		const result = await runTx(this.#db, async (sql) => {
-			const result = await sql`
-      SELECT slot_name as slot, pg_terminate_backend(active_pid) as terminated, active_pid as pid
-        FROM pg_replication_slots 
-        WHERE (slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName})
-              AND slot_name <= ${slotToKeep}`;
-			this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
-			const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
-			const replicasBefore = await sql`
-        SELECT slot, version, "initialSyncContext", "subscriberContext" 
-          FROM ${sql(replicasTable)} ORDER BY slot`;
-			if (result.length === 0) {
-				const shardSlots = await sql`
+	async #stopExistingReplicationSlotSubscriber() {
+		const sql = this.#db;
+		const { id: replicaID, slot } = this.#replica;
+		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		const result = await sql`
+      SELECT pg_terminate_backend(active_pid) as terminated, active_pid as pid
+        FROM pg_replication_slots
+        WHERE slot_name = ${slot}
+    `;
+		if (result.length === 0) {
+			const slotExpression = replicationSlotPrefix(this.#shard);
+			const replicas = await sql`
+        SELECT id, rank, slot, version, "initialSyncContext", "subscriberContext" 
+          FROM ${sql(replicasTable)} ORDER BY rank DESC`;
+			const slots = await sql`
         SELECT slot_name as slot, active, active_pid as pid
           FROM pg_replication_slots
-          WHERE slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName}
+          WHERE slot_name LIKE ${slotExpression}
           ORDER BY slot_name`;
-				this.#lc.warn?.(`slot ${slotToKeep} not found while cleaning subscribers`, {
-					slots: shardSlots,
-					replicas: replicasBefore
-				});
-				throw new AbortError(`replication slot ${slotToKeep} is missing. A different replication-manager should now be running on a new replication slot.`);
+			this.#lc.warn?.(`slot ${slot} not found while cleaning subscribers`, {
+				slots,
+				replicas
+			});
+			throw new AbortError(`replication slot ${slot} is missing. A different replication-manager should now be running on a new replication slot.`);
+		}
+		this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
+		await sql`
+      UPDATE ${sql(replicasTable)} 
+        SET "subscriberContext" = ${this.#context}
+        WHERE id = ${replicaID}`;
+		this.#cleanUpOlderReplicasAndSlots();
+	}
+	#cleanupTimer;
+	async #cleanUpOlderReplicasAndSlots() {
+		clearTimeout(this.#cleanupTimer);
+		try {
+			const result = await dropOldReplicasAndSlots(this.#lc, this.#db, this.#shard, this.#replica.rank);
+			if (result.draining === 0) {
+				this.#lc.info?.(`finished cleaning up replicas and slots`, { result });
+				return;
 			}
-			this.#lc.info?.(`replicas before cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(replicasBefore)}`);
-			await sql`
-        DELETE FROM ${sql(replicasTable)} WHERE slot < ${slotToKeep}`;
-			await sql`
-        UPDATE ${sql(replicasTable)} 
-          SET "subscriberContext" = ${this.#context}
-          WHERE slot = ${slotToKeep}`;
-			const replicasAfter = await sql`
-      SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;
-			this.#lc.info?.(`replicas after cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(replicasAfter)}`);
-			return result;
-		});
-		const pids = result.filter(({ pid }) => pid !== null).map(({ pid }) => pid);
-		if (pids.length) this.#lc.info?.(`signaled subscriber ${pids} to shut down`);
-		const otherSlots = result.filter(({ slot }) => slot !== slotToKeep).map(({ slot }) => slot);
-		if (otherSlots.length) this.#dropReplicationSlots(otherSlots).catch((e) => this.#lc.warn?.(`error dropping replication slots`, e));
-	}
-	async #dropReplicationSlots(slots) {
-		this.#lc.info?.(`dropping other replication slot(s) ${slots}`);
-		const sql = this.#db;
-		for (let i = 0; i < 5; i++) try {
-			await sql`
-          SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-            WHERE slot_name IN ${sql(slots)}
-        `;
-			this.#lc.info?.(`successfully dropped ${slots}`);
-			return;
+			this.#lc.info?.(`old slots still draining`, { result });
 		} catch (e) {
-			if (e instanceof postgres.PostgresError && e.code === PG_OBJECT_IN_USE) this.#lc.debug?.(`attempt ${i + 1}: ${String(e)}`, e);
-			else this.#lc.warn?.(`error dropping ${slots}`, e);
-			await sleep(1e3);
+			this.#lc.warn?.(`error dropping replication slots`, e);
 		}
-		this.#lc.warn?.(`maximum attempts exceeded dropping ${slots}`);
+		this.#cleanupTimer = setTimeout(() => this.#cleanUpOlderReplicasAndSlots(), REPLICA_SLOT_CLEANUP_INTERVAL_MS);
 	}
 };
 var Acker = class {
@@ -438,7 +421,7 @@ var LagReporter = class LagReporter {
 	}
 	processLagReportMessage(msg) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
-		const report = parseLogicalMessageContent(msg, lagReportSchema);
+		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
 		return this.#processLagReport(report, toStateVersionString(msg.messageLsn ?? "0/0"));
 	}
 	#processLagReport(report, watermark) {
@@ -580,7 +563,7 @@ var ChangeMaker = class {
 	}
 	#lastReplicationEvent;
 	#handleDdlMessage(lc, lsn, msg) {
-		const event = parseLogicalMessageContent(msg, replicationEventSchema);
+		const event = parseLogicalMessageContent(lc, msg, replicationEventSchema);
 		lc = lc.withContext("lsn", fromBigInt(lsn)).withContext("tag", event.event.tag).withContext("query", event.context.query);
 		clearTimeout(this.#replicaIdentityTimer);
 		const { type } = event;
@@ -927,9 +910,18 @@ var ShutdownSignal = class extends AbortError {
 		super("shutdown signal received (e.g. another zero-cache taking over the replication stream)", { cause });
 	}
 };
-function parseLogicalMessageContent({ content }, schema) {
+function parseLogicalMessageContent(lc, msg, schema) {
+	const { content } = msg;
 	const str = content instanceof Buffer ? content.toString("utf-8") : new TextDecoder().decode(content);
-	return parse(JSON.parse(str), schema, "passthrough");
+	try {
+		return parse(JSON.parse(str), schema, "passthrough");
+	} catch (e) {
+		lc.error?.(`unable to parse logical message content: ${String(e)}`, { message: {
+			...msg,
+			content: str
+		} });
+		throw e;
+	}
 }
 //#endregion
 export { initializePostgresChangeSource };