@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n    opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    const stmts = makeDownloadStatements(tableSpec, cols);\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        makeBinarySelectExprs(tableSpec, cols),\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      row[col] = field === null ? null : decoders[col](field);\n\n      if (++col === decoders.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {\n        message: {tag: 'backfill', ...backfill, rowValues, status},\n        byteSize: bufferedBytes,\n      };\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {tag: 'backfill', ...backfill, rowValues, status},\n      byteSize: bufferedBytes,\n    };\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {tag: 'backfill-completed', ...backfill, status},\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAC/C,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IACJ,SAAS;KAAC,KAAK;KAAY,GAAG;KAAU;KAAW;IAAM;IACzD,UAAU;GACZ;GACA;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACzD,UAAU;EACZ;EACA;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GAAC,KAAK;GAAsB,GAAG;GAAU;EAAM;EACxD,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
+{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randomCharacters} from '../../../../../shared/src/random-values.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\nimport {\n  SchemaIncompatibilityError,\n  type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport {resumePoint} from '../protocol/backfill-progress.ts';\nimport type {\n  BackfillCompleted,\n  BackfillProgressMark,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  makeBinarySelectExprs,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag' | 'progressMarks'>;\n\n/**\n * Determines how the progress of a backfill is marked, and from where it is\n * resumed.\n */\ntype Progress = {\n  /**\n   * The timeline of the progress marks. For plain tables, this is the table's\n   * `relfilenode`, which identifies the physical \"generation\" of the table's\n   * data (changed by heap rewrites like `VACUUM FULL`, `CLUSTER`, or\n   * `TRUNCATE`), within which `ctid`s are comparable.\n   */\n  timeline: string;\n\n  /**\n   * Whether the progress marks are resumable `ctid`s. Tables without\n   * (their own) storage, such as partitioned tables, are not resumable and\n   * are instead marked with the number of rows streamed, on a timeline that\n   * is unique to the run.\n   */\n  resumable: boolean;\n\n  /** The mark from which the backfill resumes, if any. */\n  start: BackfillProgressMark | undefined;\n};\n\n/**\n * Encodes a `ctid` text representation, e.g. `(123,4)`, as a\n * fixed-width string that sorts lexicographically in `ctid` order.\n */\nconst CTID_RE = /^\\((\\d+),(\\d+)\\)$/;\nconst CTID_MARK_RE = /^(\\d{10})\\.(\\d{5})$/;\n\nexport function ctidToProgressMark(ctid: string): string {\n  const match = CTID_RE.exec(ctid);\n  assert(match, () => `invalid ctid: ${ctid}`);\n  return `${match[1].padStart(10, '0')}.${match[2].padStart(5, '0')}`;\n}\n\n/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */\nexport function progressMarkToCtid(mark: string): string {\n  const match = CTID_MARK_RE.exec(mark);\n  assert(match, () => `invalid ctid progress mark: ${mark}`);\n  return `(${Number(match[1])},${Number(match[2])})`;\n}\n\nfunction progressMarks(\n  previous: BackfillProgressMark | undefined,\n  current: BackfillProgressMark,\n) {\n  return previous ? {previous, current} : {current};\n}\n\n/** The `ctid` of each row is appended as the last column of the download. */\nconst CTID_SELECT = 'ctid::text';\n\n/**\n * Settings that ensure that the COPY scans the table in physical (i.e.\n * `ctid`) order, which is necessary for the progress marks of a resumable\n * backfill to be meaningful. (Monotonicity is nonetheless verified as rows\n * are streamed.)\n *\n * * Synchronized seqscans can start in the middle of the table.\n * * Parallel scans interleave the rows of different workers.\n * * Index scans follow index order. (A publication row filter could\n *   otherwise be satisfied with an index.)\n */\nconst PHYSICAL_ORDER_SETTINGS = [\n  'synchronize_seqscans = off',\n  'max_parallel_workers_per_gather = 0',\n  'enable_indexscan = off',\n  'enable_indexonlyscan = off',\n  'enable_bitmapscan = off',\n];\n\nfunction rowCountMark(rows: number): string {\n  return String(rows).padStart(16, '0');\n}\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number | undefined;\n\n  /**\n   * Use text-format COPY instead of binary COPY.\n   * Binary is faster and handles all types (unknown types are cast to\n   * `::text` in the SELECT). This flag exists as an escape hatch to\n   * revert to the old code path if needed.\n   */\n  textCopy?: boolean | undefined;\n\n  /**\n   * @visibleForTesting\n   * Called after the snapshot transaction is opened, before the table is\n   * locked, e.g. to run a concurrent heap rewrite in between.\n   */\n  afterSnapshotForTesting?: (() => Promise<void>) | undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b|\\sLIMIT\\s+\\d/i;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {\n    flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE,\n    textCopy = false,\n    afterSnapshotForTesting,\n  } = opts;\n  const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n    // The COPY is a single stream that must outlive the entire table download,\n    // so allow a very long (24h) connection lifetime.\n    ['max_lifetime']: 24 * 60 * 60,\n    // A backfill COPY is a background process that can be preempted by\n    // upstream replication changes, potentially requiring it to sit idle for\n    // long stretches. Use TCP keepalive to detect dead connections instead\n    // of the inactivity watchdog, which could kill connnections that are\n    // taking a back seat for upstream replication changes.\n    liveness: 'keepalive',\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    await afterSnapshotForTesting?.();\n    const {tableSpec, backfill, progress} = await validateSchema(\n      lc,\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n    // Resumable backfills are streamed in physical (i.e. ctid) order, which\n    // is the natural order of a sequential or TID range scan (as enforced by\n    // the settings in `stream()`). An ORDER BY is not used, as Postgres does\n    // not consider either scan to be ordered, and would unnecessarily sort\n    // the table.\n    const order =\n      progress.resumable && progress.start\n        ? {\n            after: /*sql*/ `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid`,\n          }\n        : undefined;\n    const stmts = makeDownloadStatements(\n      tableSpec,\n      cols,\n      undefined,\n      undefined,\n      [...cols.map(col => id(col)), CTID_SELECT],\n      order,\n    );\n    if (progress.start) {\n      lc.info?.(\n        `resuming backfill from ${progress.start.progressMark} ` +\n          `(timeline ${progress.timeline})`,\n      );\n    }\n\n    if (textCopy) {\n      const types = await getTypeParsers(db, {returnJsonAsString: true});\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${stmts.select}) TO STDOUT`,\n        new TsvParser(),\n        cols.map(col => {\n          const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n          return (text: string) => parser(text) as JSONValue;\n        }),\n        (text: string) => text,\n        flushThresholdBytes,\n      );\n    } else {\n      const binaryStmts = makeDownloadStatements(\n        tableSpec,\n        cols,\n        undefined,\n        undefined,\n        [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT],\n        order,\n      );\n\n      yield* stream(\n        lc,\n        tx,\n        backfill,\n        progress,\n        stmts,\n        `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n        new BinaryCopyParser(),\n        cols.map(col => {\n          const spec = tableSpec.columns[col];\n          const decoder = hasBinaryDecoder(spec)\n            ? makeBinaryDecoder(spec)\n            : textCastDecoder;\n          return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n        }),\n        (buf: Buffer) => buf.toString('utf8'), // ctid::text\n        flushThresholdBytes,\n      );\n    }\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream<T>(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {timeline, resumable, start: startMark}: Progress,\n  {\n    getTotalRows,\n    getTotalBytes,\n  }: Pick<DownloadStatements, 'getTotalRows' | 'getTotalBytes'>,\n  copyCommand: string,\n  parser: {parse(chunk: Buffer): Iterable<T | null>},\n  decoders: ((field: T) => JSONValue)[],\n  ctidDecoder: (field: T) => string,\n  flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n  // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n  // sync and must never appear in a backfill COPY.\n  assert(\n    !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n    `backfill COPY must not sample or limit: ${copyCommand}`,\n  );\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n    {\n      status,\n    },\n  );\n\n  // Drain the COPY stream from *within* a single read task so that the\n  // TransactionPool worker holds the transaction for the entire duration of\n  // the COPY, rather than incorrectly considering the worker \"idle\" and\n  // attempting to send keepalives on it.\n  //\n  // Chunks are bridged out to this generator via a Queue, with a corresponding\n  // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n  // necessary because it is not possible to \"yield\" from within the read task.\n  const chunks = new Queue<Buffer | 'done'>();\n  const acks = new Queue<void>();\n\n  const copyDone = tx.processReadTask(async sql => {\n    // SET LOCAL applies to the (snapshot) transaction of this read task.\n    for (const setting of PHYSICAL_ORDER_SETTINGS) {\n      await sql.unsafe(`SET LOCAL ${setting}`);\n    }\n    const readable = await sql.unsafe(copyCommand).readable();\n    for await (const chunk of readable) {\n      chunks.enqueue(chunk as Buffer);\n      await acks.dequeue(); // wait for this generator to consume the chunk\n    }\n    chunks.enqueue('done');\n  });\n  copyDone.catch(e => chunks.enqueueRejection(e));\n\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // The mark of the last row that was streamed, and that of the last row of\n  // the previous message.\n  let current: BackfillProgressMark | undefined = startMark;\n  let previous: BackfillProgressMark | undefined = startMark;\n  const markOf = (progressMark: string) => ({progressMark, timeline});\n\n  // The number of columns of each row, followed by the ctid.\n  const numCols = decoders.length + 1;\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: decoders.length});\n  let col = 0;\n\n  for (;;) {\n    const chunk = await chunks.dequeue();\n    if (chunk === 'done') {\n      break;\n    }\n    for (const field of parser.parse(chunk)) {\n      if (col < decoders.length) {\n        row[col] = field === null ? null : decoders[col](field);\n      } else {\n        const mark = resumable\n          ? ctidToProgressMark(ctidDecoder(field as T))\n          : rowCountMark(status.rows + 1);\n        if (current !== undefined && mark <= current.progressMark) {\n          // This should not happen given the PHYSICAL_ORDER_SETTINGS, but\n          // the correctness of the progress marks depends on it.\n          throw new Error(\n            `backfill rows are not in ctid order ` +\n              `(${mark} after ${current.progressMark})`,\n          );\n        }\n        current = markOf(mark);\n      }\n\n      if (++col === numCols) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: decoders.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {\n      yield {\n        message: {\n          tag: 'backfill',\n          ...backfill,\n          rowValues,\n          status,\n          progressMarks: progressMarks(previous, must(current)),\n        },\n        byteSize: bufferedBytes,\n      };\n      previous = current;\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n\n    // Signal the read task to pull the next chunk (one-chunk backpressure).\n    acks.enqueue();\n  }\n\n  // Surface any error from the COPY read task (and confirm clean completion).\n  await copyDone;\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {\n      message: {\n        tag: 'backfill',\n        ...backfill,\n        rowValues,\n        status,\n        progressMarks: progressMarks(previous, must(current)),\n      },\n      byteSize: bufferedBytes,\n    };\n    previous = current;\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {\n    message: {\n      tag: 'backfill-completed',\n      ...backfill,\n      status,\n      // `previous` is only unset for a backfill from scratch with no rows.\n      progressMarks: previous ? {previous} : {},\n    },\n    byteSize: 0,\n  };\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(\n    lc,\n    upstreamURI,\n    'backfill-replication-session',\n    {\n      ['fetch_types']: false, // Necessary for the streaming protocol\n      connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n    },\n  );\n  const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        temporary: true, // deletes the slot when the replicationSession ends\n      });\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n    await imported.dequeue();\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  lc: LogContext,\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n  progress: Progress;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns).map(([col, {id}]) => [col, id] as const),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    const progress = await getProgress(lc, sql, spec.oid, bf);\n    return {tableSpec: spec, backfill, progress};\n  });\n}\n\n/**\n * Determines the timeline of the table's progress marks (as of the snapshot),\n * and whether the backfill can be resumed from the requested progress.\n *\n * The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest\n * of the transaction) so that its storage cannot be rewritten (by `VACUUM\n * FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which\n * require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note\n * that `pg_relation_filenode()` reflects the table's current storage (only\n * refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at\n * the transaction snapshot. If they differ, the storage was rewritten after the\n * snapshot, and the COPY would scan the new storage with the old snapshot,\n * which is not MVCC-safe for all rewrites (e.g. the table appears empty after\n * a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.\n */\nasync function getProgress(\n  lc: LogContext,\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<Progress> {\n  const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;\n  await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);\n  const [{lockedOID, relkind, snapshotFilenode, filenode}] = await sql<\n    {\n      lockedOID: number | null;\n      relkind: string;\n      snapshotFilenode: string;\n      filenode: string | null;\n    }[]\n  >`\n    SELECT to_regclass(${table})::oid::int8 AS \"lockedOID\",\n           relkind,\n           relfilenode::text AS \"snapshotFilenode\",\n           pg_relation_filenode(oid)::text AS filenode\n      FROM pg_class WHERE oid = ${relationOID}`;\n  if (Number(lockedOID) !== relationOID) {\n    // The (current) table with the name is not the one at the snapshot.\n    throw new SchemaIncompatibilityError(\n      bf,\n      `Table has been renamed or replaced`,\n    );\n  }\n  if (relkind !== 'r' || filenode === null) {\n    // Not a plain table (e.g. a partitioned table), for which ctids are not\n    // unique. Such backfills are not resumable, which is achieved by using a\n    // timeline that is unique to the run.\n    return {\n      timeline: `${relkind}:${randomCharacters(12)}`,\n      resumable: false,\n      start: undefined,\n    };\n  }\n  if (filenode !== snapshotFilenode) {\n    throw new Error(\n      `${bf.table.schema}.${bf.table.name} was rewritten after the backfill ` +\n        `snapshot (relfilenode ${snapshotFilenode} => ${filenode})`,\n    );\n  }\n  const timeline = filenode;\n  const requested = resumePoint(bf);\n  if (requested && requested.timeline !== timeline) {\n    lc.info?.(\n      `restarting backfill from scratch: the requested progress is on ` +\n        `timeline ${requested.timeline} but the table is on ${timeline}`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  if (requested && !(await canResume(sql, relationOID, bf))) {\n    lc.info?.(\n      `restarting backfill from scratch: the table has TOASTed values ` +\n        `and backfilling columns [${Object.keys(bf.columns).join(',')}] ` +\n        `are TOAST-able`,\n    );\n    return {timeline, resumable: true, start: undefined};\n  }\n  return {timeline, resumable: true, start: requested};\n}\n\n/**\n * Whether resuming a backfill (in a new snapshot) is guaranteed to deliver\n * the values of every row that the preceding snapshot(s) had not reached.\n *\n * Within a single snapshot, every row is scanned exactly once, since the\n * version visible to the snapshot stays in place while it is held. Across\n * snapshots, however, a row that had not yet been reached (i.e. beyond the\n * resume mark `M`) can be updated between the snapshots, with its new\n * version placed at or before `M` (a non-HOT update into free space in an\n * earlier page, or a HOT update into a lower line pointer of `M`'s page).\n * Neither snapshot then scans the row, and it is only delivered by the\n * replication stream. The stream's UPDATE contains the full row, _except_\n * for unchanged values stored out-of-line in TOAST, which pgoutput omits\n * (unless the table has REPLICA IDENTITY FULL). A backfilling column with\n * such a value would be published without it.\n *\n * Resuming is therefore only done if no backfilling column can be TOASTed\n * (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the\n * table has no TOAST data. The latter is safe because an omitted value must\n * still be in the TOAST relation at the new snapshot (unless its row has since\n * been deleted, which the replication stream does deliver), and conservative\n * because the TOAST relation does not shrink to empty without a heap rewrite,\n * which changes the timeline.\n *\n * Potential optimization for TOAST-able backfills: have the resumed run also\n * deliver the rows at or before `M` that were modified since the snapshot of\n * the run that started the backfill from scratch. That run's snapshot `xmin`\n * (`T_origin`, as an xid8) would be encoded in the timeline, i.e.\n * `timeline = \"<relfilenode>:<T_origin>\"`, and carried over (rather than\n * replaced) by resumed runs, so that:\n * * Within a chain of runs, every row version with `xmin < T_origin` stayed\n *   in place across all of the runs' snapshots and was therefore scanned\n *   exactly once, and all other rows are re-delivered by the resumed run\n *   (`ctid > M OR xmin >= T_origin`).\n * * Continuations from a different chain (i.e. with a newer `T_origin`) are\n *   not accepted by subscribers, and requests from different chains are\n *   merged into a backfill from scratch, as the timelines differ. Note that\n *   using the _latest_ `T` would be incorrect: rows that moved before `M`\n *   between the earlier snapshots would never be re-delivered.\n * The resumed run would scan the whole table (but only deliver the modified\n * rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which\n * requires falling back to a backfill from scratch once the current xid is\n * 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the\n * `xmin` system column returning the raw xmin of frozen tuples, which should\n * be verified.\n */\nasync function canResume(\n  sql: postgres.Sql,\n  relationOID: number,\n  bf: BackfillRequest,\n): Promise<boolean> {\n  const [{toastable, toasted}] = await sql<\n    {toastable: boolean; toasted: boolean}[]\n  >`\n    SELECT\n      EXISTS (\n        SELECT 1 FROM pg_attribute\n          WHERE attrelid = ${relationOID}\n            AND attname IN ${sql(Object.keys(bf.columns))}\n            AND attstorage <> 'p'\n      ) AS toastable,\n      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted\n      FROM pg_class WHERE oid = ${relationOID}`;\n  return !toastable || !toasted;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,UAAU;AAChB,IAAM,eAAe;AAErB,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,QAAQ,KAAK,IAAI;CAC/B,OAAO,aAAa,iBAAiB,MAAM;CAC3C,OAAO,GAAG,MAAM,GAAG,SAAS,IAAI,GAAG,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,GAAG;AAClE;;AAGA,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,aAAa,KAAK,IAAI;CACpC,OAAO,aAAa,+BAA+B,MAAM;CACzD,OAAO,IAAI,OAAO,MAAM,EAAE,EAAE,GAAG,OAAO,MAAM,EAAE,EAAE;AAClD;AAEA,SAAS,cACP,UACA,SACA;CACA,OAAO,WAAW;EAAC;EAAU;CAAO,IAAI,EAAC,QAAO;AAClD;;AAGA,IAAM,cAAc;;;;;;;;;;;;AAapB,IAAM,0BAA0B;CAC9B;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aAAa,MAAsB;CAC1C,OAAO,OAAO,IAAI,EAAE,SAAS,IAAI,GAAG;AACtC;AA4BA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EACJ,sBAAsB,0BACtB,WAAW,OACX,4BACE;CACJ,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,0BAA0B;EAChC,MAAM,EAAC,WAAW,UAAU,aAAY,MAAM,eAC5C,IACA,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EAMpD,MAAM,QACJ,SAAS,aAAa,SAAS,QAC3B,EACE,OAAe,WAAW,mBAAmB,SAAS,MAAM,YAAY,EAAE,QAC5E,IACA,KAAA;EACN,MAAM,QAAQ,uBACZ,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,KAAK,KAAI,QAAO,GAAG,GAAG,CAAC,GAAG,WAAW,GACzC,KACF;EACA,IAAI,SAAS,OACX,GAAG,OACD,0BAA0B,SAAS,MAAM,aAAa,aACvC,SAAS,SAAS,EACnC;EAGF,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,IACA,SAAiB,MAClB,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,sBAAsB,WAAW,IAAI,GAAG,WAAW,GACvD,KACF;GAEA,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,IACA,QAAgB,IAAI,SAAS,MAAM,GACpC,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,UAAU,WAAW,OAAO,aAC7B,EACE,cACA,iBAEF,aACA,QACA,UACA,aACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAE/C,KAAK,MAAM,WAAW,yBACpB,MAAM,IAAI,OAAO,aAAa,SAAS;EAEzC,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAIA,IAAI,UAA4C;CAChD,IAAI,WAA6C;CACjD,MAAM,UAAU,kBAA0B;EAAC;EAAc;CAAQ;CAGjE,MAAM,UAAU,SAAS,SAAS;CAGlC,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,MAAM,SAAS,QACjB,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;QACjD;IACL,MAAM,OAAO,YACT,mBAAmB,YAAY,KAAU,CAAC,IAC1C,aAAa,OAAO,OAAO,CAAC;IAChC,IAAI,YAAY,KAAA,KAAa,QAAQ,QAAQ,cAG3C,MAAM,IAAI,MACR,wCACM,KAAK,SAAS,QAAQ,aAAa,EAC3C;IAEF,UAAU,OAAO,IAAI;GACvB;GAEA,IAAI,EAAE,QAAQ,SAAS;IACrB,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,uBAAuB,UAAU,SAAS,GAAG;GAChE,MAAM;IACJ,SAAS;KACP,KAAK;KACL,GAAG;KACH;KACA;KACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;IACtD;IACA,UAAU;GACZ;GACA,WAAW;GACX;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IACP,KAAK;IACL,GAAG;IACH;IACA;IACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;GACtD;GACA,UAAU;EACZ;EACA,WAAW;EACX;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GACP,KAAK;GACL,GAAG;GACH;GAEA,eAAe,WAAW,EAAC,SAAQ,IAAI,CAAC;EAC1C;EACA,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,IACA,cACA,IACA,WAKC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAAC,KAAK,EAAE,CAAU,CACvE;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAaA,OAAO;GAAC,WAAW;GAAM,UAAA;IAXvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAGuB;GAAU,UAAA,MADZ,YAAY,IAAI,KAAK,KAAK,KAAK,EAAE;EACb;CAC7C,CAAC;AACH;;;;;;;;;;;;;;;;AAiBA,eAAe,YACb,IACA,KACA,aACA,IACmB;CACnB,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,MAAM,EAAE,GAAG,GAAG,GAAG,MAAM,IAAI;CACxD,MAAM,IAAI,OAAO,cAAc,MAAM,sBAAsB;CAC3D,MAAM,CAAC,EAAC,WAAW,SAAS,kBAAkB,cAAa,MAAM,GAOhE;yBACsB,MAAM;;;;kCAIG;CAChC,IAAI,OAAO,SAAS,MAAM,aAExB,MAAM,IAAI,2BACR,IACA,oCACF;CAEF,IAAI,YAAY,OAAO,aAAa,MAIlC,OAAO;EACL,UAAU,GAAG,QAAQ,GAAG,iBAAiB,EAAE;EAC3C,WAAW;EACX,OAAO,KAAA;CACT;CAEF,IAAI,aAAa,kBACf,MAAM,IAAI,MACR,GAAG,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,0DACT,iBAAiB,MAAM,SAAS,EAC7D;CAEF,MAAM,WAAW;CACjB,MAAM,YAAY,YAAY,EAAE;CAChC,IAAI,aAAa,UAAU,aAAa,UAAU;EAChD,GAAG,OACD,2EACc,UAAU,SAAS,uBAAuB,UAC1D;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,IAAI,aAAa,CAAE,MAAM,UAAU,KAAK,aAAa,EAAE,GAAI;EACzD,GAAG,OACD,2FAC8B,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,iBAElE;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,OAAO;EAAC;EAAU,WAAW;EAAM,OAAO;CAAS;AACrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,eAAe,UACb,KACA,aACA,IACkB;CAClB,MAAM,CAAC,EAAC,WAAW,aAAY,MAAM,GAEpC;;;;6BAI0B,YAAY;6BACZ,IAAI,OAAO,KAAK,GAAG,OAAO,CAAC,EAAE;;;;kCAIxB;CAChC,OAAO,CAAC,aAAa,CAAC;AACxB"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -0,0 +1,25 @@
+import type { LogContext } from '@rocicorp/logger';
+import { type PostgresDB } from '../../../types/pg.ts';
+import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
+import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
+import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
+import { type ReplicaState } from './schema/shard.ts';
+interface PurgeLock {
+    release(): Promise<void>;
+}
+export type InitializeOptions = ReplicaOptions & {
+    slotPerReplica: boolean | undefined;
+    inactiveReplicaGracePeriodMs: number;
+};
+/**
+ * Initializes a Postgres change source, including the initial sync of the
+ * replica, before streaming changes from the corresponding logical replication
+ * stream.
+ */
+export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
+export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
+    restoreFrom: ReplicaState;
+    replicateTo: ReplicaState;
+} | undefined>;
+export {};
+//# sourceMappingURL=change-source-init.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAMxE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAyFD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CAAC;IAAC,WAAW,EAAE,YAAY,CAAC;IAAC,WAAW,EAAE,YAAY,CAAA;CAAC,GAAG,SAAS,CAAC,CAoI7E"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js

@@ -0,0 +1,202 @@
+import { assert } from "../../../../../shared/src/asserts.js";
+import { deepEqual } from "../../../../../shared/src/json.js";
+import { promiseVoid } from "../../../../../shared/src/resolved-promises.js";
+import { sleep } from "../../../../../shared/src/sleep.js";
+import { must } from "../../../../../shared/src/must.js";
+import { Database } from "../../../../../zqlite/src/db.js";
+import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
+import { StatementRunner } from "../../../db/statements.js";
+import { connectPgClient } from "../../../types/pg.js";
+import { toBigInt } from "./lsn.js";
+import "./schema/replica-stage-enum.js";
+import { dropShard, getActiveReplicas, getReplicaAtVersion, getRestoreCandidates, initRestoreReplica, internalPublicationPrefix } from "./schema/shard.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { restoreReplica } from "../common/replica-restore.js";
+import { initReplica } from "../common/replica-schema.js";
+import { claimSlotForResumption, createReplicaAndSlot } from "./replication-slots.js";
+import { ensureShardSchema } from "./schema/init.js";
+import { initialSync } from "./initial-sync.js";
+import { PostgresChangeSource } from "./change-source.js";
+//#region ../zero-cache/src/services/change-source/pg/change-source-init.ts
+/**
+* Initializes a Postgres change source, including the initial sync of the
+* replica, before streaming changes from the corresponding logical replication
+* stream.
+*/
+async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs } = {
+	epoch: 0,
+	slotPerReplica: false,
+	backupV5: true,
+	inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS
+}, purgeLock, streamInboundTimeoutMs) {
+	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
+	try {
+		await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
+		if (slotPerReplica) assert(purgeLock === null, `There should be no purgeLock for RMv2`);
+		const restoredReplica = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
+		let initialSyncedReplica;
+		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
+			purgeLock?.release();
+			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, {
+				epoch,
+				backupV5
+			});
+		});
+		const replica = new Database(lc, replicaDbFile);
+		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
+		replica.close();
+		const { upstreamReplica, pgVersion } = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
+		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
+		return {
+			subscriptionState,
+			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, pgVersion, {
+				backupPath,
+				backupV5
+			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
+			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
+			replicaID: upstreamReplica.id,
+			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
+		};
+	} finally {
+		await db.end();
+	}
+}
+async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
+	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
+	if (replicas.length === 0) {
+		lc.info?.(`no suitable replicas to restore from`, { replicas });
+		return;
+	}
+	const [replica] = replicas;
+	if (litestream?.backupURL) {
+		const { backupURL: backupBaseURL } = litestream;
+		const { slot, backupPath, confirmedFlushLsn } = replica;
+		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
+		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
+		await restoreReplica(lc, {
+			...litestream,
+			backupURL
+		}, replicaFile, constraints);
+	}
+	return replica;
+}
+async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, constraints }, gracePeriodMs) {
+	const result = await getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs);
+	if (!result) return;
+	const { restoreFrom, replicateTo } = result;
+	if (litestream?.backupURL) {
+		const { backupURL: backupBaseURL } = litestream;
+		const { slot, backupPath, confirmedFlushLsn } = restoreFrom;
+		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
+		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, {
+			restoreFrom,
+			replicateTo
+		});
+		await restoreReplica(lc, {
+			...litestream,
+			backupURL
+		}, replicaFile, constraints);
+	}
+	return replicateTo;
+}
+var REPLICA_POLL_INTERVAL_MS = 5e3;
+var DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 2e4;
+async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs, pollIntervalMs = REPLICA_POLL_INTERVAL_MS) {
+	const inactiveSince = /* @__PURE__ */ new Map();
+	let destination;
+	try {
+		for (let i = 0;; i++) {
+			if (i > 0) await sleep(pollIntervalMs);
+			const replicas = await getRestoreCandidates(lc, sql, shard, epoch);
+			if (replicas.length === 0) {
+				lc.info?.(`no suitable replicas to restore from`, { replicas });
+				destination?.initialSession.destroy();
+				return;
+			}
+			for (const replica of replicas) {
+				if (replica.active) inactiveSince.delete(replica.id);
+				else if (!inactiveSince.has(replica.id)) inactiveSince.set(replica.id, Date.now());
+				if (replica.stage === 0) {
+					if (i % 12 === 0) lc.info?.(`waiting for initial sync of ${replica.id}`, { replica });
+					break;
+				}
+				if (replica.stage === 1) {
+					const { active, confirmedFlushLsn } = replica;
+					if (active) {
+						destination ??= await createReplicaAndSlot(lc, sql, "fork-replica-session", shard, epoch, Date.now().toString(), slotFailover, {
+							backupPath: null,
+							backupV5: true
+						}, () => promiseVoid, 2);
+						if (toBigInt(confirmedFlushLsn) < toBigInt(destination.slot.consistent_point)) {
+							lc.info?.(`waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`, { replica });
+							break;
+						}
+					}
+					if (destination && toBigInt(confirmedFlushLsn) >= toBigInt(destination.slot.consistent_point)) {
+						lc.info?.(`forking replica ${replica.id}@${replica.slot}`, { replica });
+						return {
+							restoreFrom: replica,
+							replicateTo: must(await initRestoreReplica(sql, shard, {
+								sourceID: replica.id,
+								destID: destination.replica.id
+							}), `replica ${destination.replica.id} was deleted`)
+						};
+					}
+				}
+				if (replica.backupPath && !replica.active) {
+					const now = Date.now();
+					const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);
+					lc.info?.(`replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`, { replica });
+					if (inactiveMs >= gracePeriodMs) {
+						const reserved = await claimSlotForResumption(lc, sql, shard, "resume-replica", replica.slot);
+						if (!reserved) lc.warn?.(`unable to resume replica ${replica.id}@${replica.slot}`);
+						else {
+							lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);
+							destination?.initialSession.destroy();
+							return {
+								restoreFrom: replica,
+								replicateTo: reserved.replica
+							};
+						}
+					}
+				}
+			}
+		}
+	} catch (e) {
+		destination?.initialSession.destroy();
+		throw e;
+	}
+}
+async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
+	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
+	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
+	const requested = shard.publications.toSorted();
+	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
+	if (!deepEqual(requested, replicated)) {
+		lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
+		await sql.unsafe(dropShard(shard.appID, shard.shardNum));
+		throw new AutoResetSignal(`Requested publications [${requested}] do not match configured publications: [${replicated}]`);
+	}
+	if (!deepEqual(upstreamReplica.publications, subscribed)) throw new AutoResetSignal(`Upstream publications [${upstreamReplica.publications}] do not match subscribed publications [${subscribed}]`);
+	const exists = await sql`
+    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};
+  `.values();
+	if (exists.length !== subscribed.length) throw new AutoResetSignal(`Upstream publications [${exists.flat()}] do not contain all subscribed publications [${subscribed}]`);
+	const { slot } = upstreamReplica;
+	const result = await sql`
+    SELECT restart_lsn as "restartLSN", wal_status as "walStatus" FROM pg_replication_slots
+      WHERE slot_name = ${slot}`;
+	if (result.length === 0) throw new AutoResetSignal(`replication slot ${slot} is missing`);
+	const [{ restartLSN, walStatus }] = result;
+	if (restartLSN === null || walStatus === "lost") throw new AutoResetSignal(`replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`);
+	const [{ pgVersion }] = await sql`
+    SELECT current_setting('server_version_num')::int as "pgVersion"`;
+	return {
+		upstreamReplica,
+		pgVersion
+	};
+}
+//#endregion
+export { initializePostgresChangeSource };
+
+//# sourceMappingURL=change-source-init.js.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  purgeLock?: PurgeLock | null,\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    if (slotPerReplica) {\n      // Sanity check: This should be disabled via pgChangeLogEnabled=false.\n      assert(purgeLock === null, `There should be no purgeLock for RMv2`);\n    }\n\n    const restoredReplica = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // In RMv1, the purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2.\n        void purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n        // recorded in the replicas table by the PostgresChangeSource.\n        backupV5\n        ? String(Date.now())\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n    };\n  } finally {\n    await db.end();\n  }\n}\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, constraints}: RestoreOptions,\n): Promise<ReplicaState | undefined> {\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the constraints, if present\n    ({generation}) =>\n      generation === (constraints?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return undefined;\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return replica;\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, constraints}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<ReplicaState | undefined> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    return undefined; // can't restore, must initial-sync\n  }\n  const {restoreFrom, replicateTo} = result;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return replicateTo;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<{restoreFrom: ReplicaState; replicateTo: ReplicaState} | undefined> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            if (\n              toBigInt(confirmedFlushLsn) <\n              toBigInt(destination.slot.consistent_point)\n            ) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            toBigInt(confirmedFlushLsn) >=\n              toBigInt(destination.slot.consistent_point)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {restoreFrom: replica, replicateTo};\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,WACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,IAAI,gBAEF,OAAO,cAAc,MAAM,uCAAuC;EAGpE,MAAM,kBAAkB,iBACpB,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,WAAgB,QAAQ;GACxB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,CAClB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAEA,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,WACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;EACnD;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAIA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,eACsB;CACnC,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBACA,gBAAgB,aAAa,kBAAkB,WACnD;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D;CACF;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,eACb,eACmC;CACnC,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QACH;CAEF,MAAM,EAAC,aAAa,gBAAe;CAEnC,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAC4D;CAC7E,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MACA,IACE,SAAS,iBAAiB,IAC1B,SAAS,YAAY,KAAK,gBAAgB,GAC1C;OACA,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,oBAC1G,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,SAAS,iBAAiB,KACxB,SAAS,YAAY,KAAK,gBAAgB,GAC5C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OAAC,aAAa;OAAS,aAPV,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAEN;MAAW;KAC3C;IACF;IAEA,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts

@@ -3,35 +3,25 @@ import * as v from '../../../../../shared/src/valita.ts';
 import type { PublishedIndexSpec, PublishedTableSpec } from '../../../db/specs.ts';
 import { type LexiVersion } from '../../../types/lexi-version.ts';
 import { type PostgresDB } from '../../../types/pg.ts';
-import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
+import { type ShardID } from '../../../types/shards.ts';
 import type { Sink } from '../../../types/streams.ts';
 import type { ChangeSource, ChangeStream } from '../change-source.ts';
 import { type Listener } from '../common/change-stream-multiplexer.ts';
-import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 import type { BackfillRequest, DownstreamStatusMessage } from '../protocol/current.ts';
 import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
 import { streamBackfill } from './backfill-stream.ts';
-import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
+import { type ServerContext } from './initial-sync.ts';
 import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
 import { subscribe } from './logical-replication/stream.ts';
 import { type PublishedTableWithReplicaIdentity } from './schema/published.ts';
 import { type BackupOptions, type InternalShardConfig, type Replica } from './schema/shard.ts';
-interface PurgeLock {
-    release(): Promise<void>;
-}
-/**
- * Initializes a Postgres change source, including the initial sync of the
- * replica, before streaming changes from the corresponding logical replication
- * stream.
- */
-export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { backupV5 }?: ReplicaOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
 /**
  * Postgres implementation of a {@link ChangeSource} backed by a logical
  * replication stream.
  */
 export declare class PostgresChangeSource implements ChangeSource {
     #private;
-    constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, deps?: {
+    constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, pgVersion: number, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, deps?: {
         subscribe?: typeof subscribe;
         streamBackfill?: typeof streamBackfill;
     });
@@ -45,7 +35,15 @@ export declare class PostgresChangeSource implements ChangeSource {
 }
 export declare class Acker implements Listener {
     #private;
-    constructor(acks: Sink<bigint>);
+    /**
+     * @param resumeWatermark the watermark the stream resumes after. What came
+     *     before it was received on an earlier connection, and only the
+     *     change-streamer knows whether it has persisted it, so keepalives are
+     *     not acked until the change-streamer has acked it. A change-streamer
+     *     whose SQLite change log is ahead of its backup would otherwise have
+     *     the slot moved past transactions that only that log holds.
+     */
+    constructor(acks: Sink<bigint>, resumeWatermark: string | null);
     onChange(change: ChangeStreamMessage): void;
     ack(watermark: LexiVersion): void;
 }
@@ -59,8 +57,8 @@ export declare class LagReporter {
     #private;
     static readonly MESSAGE_SUFFIX = "/lag-report/v1";
     readonly messagePrefix: string;
-    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
-    get pgVersion(): number | undefined;
+    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, pgVersion: number, lagIntervalMs: number);
+    get pgVersion(): number;
     initiateLagReport(log?: boolean): Promise<{
         firstCommitTimeMs: number;
         nextSendTimeMs: number;

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAMpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAM9E,OAAO,EAGL,KAAK,iCAAiC,EAEvC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAQL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EAEb,MAAM,mBAAmB,CAAC;AAK3B,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EAAC,QAAQ,EAAC,GAAE,cAAiC,EAC7C,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAuG3B;AA0HD;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAevD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAgCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAWvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAuGF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAE7B;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA0B1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EAMtB;IAWD,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAwElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAuCF;AA2pBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA0B1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAsCF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}