@rocicorp/zero 0.26.1-canary.4 → 0.26.1-canary.6

package/out/zero/package.json.js

@@ -1,4 +1,4 @@
-const version = "0.26.1-canary.4";
+const version = "0.26.1-canary.6";
 const packageJson = {
   version
 };

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EAGf,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAYhC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAwDrD"}
+{"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EAGf,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAYhC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAyDrD"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -17,7 +17,9 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
   lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
   const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE } = opts;
   const db = pgClient(lc, upstreamURI, {
-    connection: { ["application_name"]: "backfill-stream" }
+    connection: { ["application_name"]: "backfill-stream" },
+    ["max_lifetime"]: 120 * 60
+    // set a long (2h) limit for COPY streaming
   });
   const tx = new TransactionPool(lc, READONLY).run(db);
   try {

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  createReplicationSlot,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n  const db = pgClient(lc, upstreamURI, {\n    connection: {['application_name']: 'backfill-stream'},\n  });\n  const tx = new TransactionPool(lc, READONLY).run(db);\n  try {\n    const watermark = await setSnapshot(lc, upstreamURI, tx, slot);\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n    const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n\n    yield* stream(\n      lc,\n      tx,\n      backfill,\n      makeDownloadStatements(tableSpec, cols),\n      cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n      flushThresholdBytes,\n    );\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx.setDone();\n    // errors are already thrown and handled from processReadTask()\n    void tx.done().catch(() => {});\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {select, getTotalRows, getTotalBytes}: DownloadStatements,\n  colParsers: TypeParser[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {\n    status,\n  });\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n  );\n\n  const tsvParser = new TsvParser();\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: colParsers.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const text of tsvParser.parse(chunk)) {\n      row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n      if (++col === colParsers.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: colParsers.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function setSnapshot(\n  lc: LogContext,\n  upstreamURI: string,\n  tx: TransactionPool,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, tempSlot);\n\n    await tx.processReadTask(sql =>\n      sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`),\n    );\n    // Once the snapshot has been set, the replication session and slot can\n    // be closed / dropped.\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return watermark;\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${tempSlot}'`,\n    );\n    lc.error?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"names":["spec","v.parse"],"mappings":";;;;;;;;;;;;;;AAgDA,MAAM,2BAA2B,KAAK;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,IAC+B;AACrD,OAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;AAErC,QAAM,EAAC,sBAAsB,yBAAA,IAA4B;AACzD,QAAM,KAAK,SAAS,IAAI,aAAa;AAAA,IACnC,YAAY,EAAC,CAAC,kBAAkB,GAAG,kBAAA;AAAA,EAAiB,CACrD;AACD,QAAM,KAAK,IAAI,gBAAgB,IAAI,QAAQ,EAAE,IAAI,EAAE;AACnD,MAAI;AACF,UAAM,YAAY,MAAM,YAAY,IAAI,aAAa,IAAI,IAAI;AAC7D,UAAM,EAAC,WAAW,SAAA,IAAY,MAAM;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AAGjE,UAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,UAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;AAEpD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,uBAAuB,WAAW,IAAI;AAAA,MACtC,KAAK,IAAI,CAAA,QAAO,MAAM,cAAc,UAAU,QAAQ,GAAG,EAAE,OAAO,CAAC;AAAA,MACnE;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AASV,QACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAC7C;AACA,YAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,GAAE;AAAA,IAChE;AACA,UAAM;AAAA,EACR,UAAA;AACE,OAAG,QAAA;AAEH,SAAK,GAAG,OAAO,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7B,SAAK,GAAG,MAAM,MAAM,OAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;AAAA,EAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,QAAQ,cAAc,cAAA,GACvB,YACA,qBACqD;AACrD,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,GAAG;AAAA,IAAgB,CAAA,QAC7C,QAAQ,IAAI;AAAA,MACV,IAAI,OAA8B,YAAY;AAAA,MAC9C,IAAI,OAA+B,aAAa;AAAA,IAAA,CACjD;AAAA,EAAA;AAEH,QAAM,SAAyB;AAAA,IAC7B,MAAM;AAAA,IACN,WAAW,OAAO,KAAK,CAAC,EAAE,SAAS;AAAA,IACnC,YAAY,OAAO,MAAM,CAAC,EAAE,UAAU;AAAA,EAAA;AAGxC,MAAI,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACnD,KAAG,OAAO,sCAAsC,MAAM,KAAK,OAAO,QAAQ;AAAA,IACxE;AAAA,EAAA,CACD;AACD,QAAM,aAAa,MAAM,GAAG;AAAA,IAAgB,SAC1C,IAAI,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,EAAS;AAGpD,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,aAAa;AACjB,MAAI,YAAY;AAChB,MAAI,YAA2B,CAAA;AAC/B,MAAI,gBAAgB;AAEpB,QAAM,aAAa,MAAM;AACvB,OAAG;AAAA,MACD,WAAW,UAAU,MAAM,UAAU,aAAa,uBAChC,OAAO,IAAI,UAAU,SAAS,WAAW,UAAU;AAAA,IAAA;AAAA,EAEzE;AAGA,MAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC7D,MAAI,MAAM;AAEV,mBAAiB,QAAQ,YAAY;AACnC,UAAM,QAAQ;AACd,eAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,UAAI,GAAG,IAAI,SAAS,OAAO,OAAQ,WAAW,GAAG,EAAE,IAAI;AAEvD,UAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,kBAAU,KAAK,GAAG;AAClB,eAAO;AACP,cAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC5C,cAAM;AAAA,MACR;AAAA,IACF;AACA,qBAAiB,MAAM;AACvB,kBAAc,MAAM;AAEpB,QAAI,iBAAiB,qBAAqB;AACxC,YAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,iBAAA;AACA,kBAAY,CAAA;AACZ,sBAAgB;AAAA,IAClB;AAAA,EACF;AAGA,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,eAAA;AAAA,EACF;AAEA,QAAM,EAAC,KAAK,sBAAsB,GAAG,UAAU,OAAA;AAC/C,aAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AAC/C,KAAG;AAAA,IACD,sBAAsB,OAAO,IAAI,UAAU,SAAS,UAAU,UAAU,WAClE,OAAO;AAAA,EAAA;AAEjB;AAWA,eAAe,YACb,IACA,aACA,IACA,gBACA;AACA,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,GAAG,cAAc,OAAO,KAAK,KAAK;AACnD,MAAI;AACF,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AAE9D,UAAM,GAAG;AAAA,MAAgB,CAAA,QACvB,IAAI,OAAO,6BAA6B,QAAQ,GAAG;AAAA,IAAA;AAIrD,UAAM,mBAAmB,OAAO,0BAA0B,QAAQ,GAAG;AAErE,UAAM,YAAY,qBAAqB,GAAG;AAC1C,OAAG,OAAO,sCAAsC,GAAG,KAAK,SAAS,GAAG;AACpE,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,UAAM,mBAAmB;AAAA;AAAA,MAEvB;AAAA,8BACwB,QAAQ;AAAA,IAAA;AAElC,OAAG,QAAQ,sCAAsC,CAAC;AAClD,UAAM;AAAA,EACR,UAAA;AACE,UAAM,mBAAmB,IAAA;AAAA,EAC3B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,SAAO,GAAG,gBAAgB,OAAM,QAAO;AACrC,UAAM,EAAC,OAAA,IAAU,MAAM,mBAAmB,KAAK,YAAY;AAC3D,UAAM,OAAO,OAAO;AAAA,MAClB,CAAAA,UAAQA,MAAK,WAAW,GAAG,MAAM,UAAUA,MAAK,SAAS,GAAG,MAAM;AAAA,IAAA;AAEpE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,YAAYC,MAAQ,GAAG,MAAM,UAAU,mBAAmB;AAChE,QAAI,KAAK,cAAc,UAAU,WAAW;AAC1C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QAAI,KAAK,QAAQ,UAAU,aAAa;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QACE,CAAC;AAAA,MACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC;AAAA,MACrC,IAAI,IAAI,KAAK,sBAAsB;AAAA,IAAA,GAErC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,UAAU;AAAA,MACd,GAAG,OAAO,QAAQ,UAAU,MAAM;AAAA,MAClC,GAAG,OAAO,QAAQ,GAAG,OAAO;AAAA,IAAA;AAE9B,eAAW,CAAC,KAAK,GAAG,KAAK,SAAS;AAChC,YAAM,UAAU,KAAK,QAAQ,GAAG;AAChC,UAAI,CAAC,SAAS;AACZ,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AACA,YAAM,UAAUA,MAAQ,KAAK,oBAAoB;AACjD,UAAI,QAAQ,WAAW,QAAQ,KAAK;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AAAA,IACF;AACA,UAAM,WAA2B;AAAA,MAC/B,UAAU;AAAA,QACR,QAAQ,GAAG,MAAM;AAAA,QACjB,MAAM,GAAG,MAAM;AAAA,QACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAA;AAAA,MAAC;AAAA,MAEjD,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE;AAAA,QAC/B,CAAA,QAAO,EAAE,OAAO,UAAU;AAAA,MAAA;AAAA,MAE5B;AAAA,IAAA;AAEF,WAAO,EAAC,WAAW,MAAM,SAAA;AAAA,EAC3B,CAAC;AACH;"}
+{"version":3,"file":"backfill-stream.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  createReplicationSlot,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n  const db = pgClient(lc, upstreamURI, {\n    connection: {['application_name']: 'backfill-stream'},\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  const tx = new TransactionPool(lc, READONLY).run(db);\n  try {\n    const watermark = await setSnapshot(lc, upstreamURI, tx, slot);\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n    const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n\n    yield* stream(\n      lc,\n      tx,\n      backfill,\n      makeDownloadStatements(tableSpec, cols),\n      cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n      flushThresholdBytes,\n    );\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx.setDone();\n    // errors are already thrown and handled from processReadTask()\n    void tx.done().catch(() => {});\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {select, getTotalRows, getTotalBytes}: DownloadStatements,\n  colParsers: TypeParser[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {\n    status,\n  });\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n  );\n\n  const tsvParser = new TsvParser();\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: colParsers.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const text of tsvParser.parse(chunk)) {\n      row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n      if (++col === colParsers.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: colParsers.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function setSnapshot(\n  lc: LogContext,\n  upstreamURI: string,\n  tx: TransactionPool,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, tempSlot);\n\n    await tx.processReadTask(sql =>\n      sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`),\n    );\n    // Once the snapshot has been set, the replication session and slot can\n    // be closed / dropped.\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return watermark;\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${tempSlot}'`,\n    );\n    lc.error?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"names":["spec","v.parse"],"mappings":";;;;;;;;;;;;;;AAgDA,MAAM,2BAA2B,KAAK;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,IAC+B;AACrD,OAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;AAErC,QAAM,EAAC,sBAAsB,yBAAA,IAA4B;AACzD,QAAM,KAAK,SAAS,IAAI,aAAa;AAAA,IACnC,YAAY,EAAC,CAAC,kBAAkB,GAAG,kBAAA;AAAA,IACnC,CAAC,cAAc,GAAG,MAAM;AAAA;AAAA,EAAA,CACzB;AACD,QAAM,KAAK,IAAI,gBAAgB,IAAI,QAAQ,EAAE,IAAI,EAAE;AACnD,MAAI;AACF,UAAM,YAAY,MAAM,YAAY,IAAI,aAAa,IAAI,IAAI;AAC7D,UAAM,EAAC,WAAW,SAAA,IAAY,MAAM;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AAGjE,UAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,UAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;AAEpD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,uBAAuB,WAAW,IAAI;AAAA,MACtC,KAAK,IAAI,CAAA,QAAO,MAAM,cAAc,UAAU,QAAQ,GAAG,EAAE,OAAO,CAAC;AAAA,MACnE;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AASV,QACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAC7C;AACA,YAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,GAAE;AAAA,IAChE;AACA,UAAM;AAAA,EACR,UAAA;AACE,OAAG,QAAA;AAEH,SAAK,GAAG,OAAO,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7B,SAAK,GAAG,MAAM,MAAM,OAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;AAAA,EAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,QAAQ,cAAc,cAAA,GACvB,YACA,qBACqD;AACrD,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,GAAG;AAAA,IAAgB,CAAA,QAC7C,QAAQ,IAAI;AAAA,MACV,IAAI,OAA8B,YAAY;AAAA,MAC9C,IAAI,OAA+B,aAAa;AAAA,IAAA,CACjD;AAAA,EAAA;AAEH,QAAM,SAAyB;AAAA,IAC7B,MAAM;AAAA,IACN,WAAW,OAAO,KAAK,CAAC,EAAE,SAAS;AAAA,IACnC,YAAY,OAAO,MAAM,CAAC,EAAE,UAAU;AAAA,EAAA;AAGxC,MAAI,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACnD,KAAG,OAAO,sCAAsC,MAAM,KAAK,OAAO,QAAQ;AAAA,IACxE;AAAA,EAAA,CACD;AACD,QAAM,aAAa,MAAM,GAAG;AAAA,IAAgB,SAC1C,IAAI,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,EAAS;AAGpD,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,aAAa;AACjB,MAAI,YAAY;AAChB,MAAI,YAA2B,CAAA;AAC/B,MAAI,gBAAgB;AAEpB,QAAM,aAAa,MAAM;AACvB,OAAG;AAAA,MACD,WAAW,UAAU,MAAM,UAAU,aAAa,uBAChC,OAAO,IAAI,UAAU,SAAS,WAAW,UAAU;AAAA,IAAA;AAAA,EAEzE;AAGA,MAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC7D,MAAI,MAAM;AAEV,mBAAiB,QAAQ,YAAY;AACnC,UAAM,QAAQ;AACd,eAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,UAAI,GAAG,IAAI,SAAS,OAAO,OAAQ,WAAW,GAAG,EAAE,IAAI;AAEvD,UAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,kBAAU,KAAK,GAAG;AAClB,eAAO;AACP,cAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC5C,cAAM;AAAA,MACR;AAAA,IACF;AACA,qBAAiB,MAAM;AACvB,kBAAc,MAAM;AAEpB,QAAI,iBAAiB,qBAAqB;AACxC,YAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,iBAAA;AACA,kBAAY,CAAA;AACZ,sBAAgB;AAAA,IAClB;AAAA,EACF;AAGA,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,eAAA;AAAA,EACF;AAEA,QAAM,EAAC,KAAK,sBAAsB,GAAG,UAAU,OAAA;AAC/C,aAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AAC/C,KAAG;AAAA,IACD,sBAAsB,OAAO,IAAI,UAAU,SAAS,UAAU,UAAU,WAClE,OAAO;AAAA,EAAA;AAEjB;AAWA,eAAe,YACb,IACA,aACA,IACA,gBACA;AACA,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,GAAG,cAAc,OAAO,KAAK,KAAK;AACnD,MAAI;AACF,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AAE9D,UAAM,GAAG;AAAA,MAAgB,CAAA,QACvB,IAAI,OAAO,6BAA6B,QAAQ,GAAG;AAAA,IAAA;AAIrD,UAAM,mBAAmB,OAAO,0BAA0B,QAAQ,GAAG;AAErE,UAAM,YAAY,qBAAqB,GAAG;AAC1C,OAAG,OAAO,sCAAsC,GAAG,KAAK,SAAS,GAAG;AACpE,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,UAAM,mBAAmB;AAAA;AAAA,MAEvB;AAAA,8BACwB,QAAQ;AAAA,IAAA;AAElC,OAAG,QAAQ,sCAAsC,CAAC;AAClD,UAAM;AAAA,EACR,UAAA;AACE,UAAM,mBAAmB,IAAA;AAAA,EAC3B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,SAAO,GAAG,gBAAgB,OAAM,QAAO;AACrC,UAAM,EAAC,OAAA,IAAU,MAAM,mBAAmB,KAAK,YAAY;AAC3D,UAAM,OAAO,OAAO;AAAA,MAClB,CAAAA,UAAQA,MAAK,WAAW,GAAG,MAAM,UAAUA,MAAK,SAAS,GAAG,MAAM;AAAA,IAAA;AAEpE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,YAAYC,MAAQ,GAAG,MAAM,UAAU,mBAAmB;AAChE,QAAI,KAAK,cAAc,UAAU,WAAW;AAC1C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QAAI,KAAK,QAAQ,UAAU,aAAa;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QACE,CAAC;AAAA,MACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC;AAAA,MACrC,IAAI,IAAI,KAAK,sBAAsB;AAAA,IAAA,GAErC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,UAAU;AAAA,MACd,GAAG,OAAO,QAAQ,UAAU,MAAM;AAAA,MAClC,GAAG,OAAO,QAAQ,GAAG,OAAO;AAAA,IAAA;AAE9B,eAAW,CAAC,KAAK,GAAG,KAAK,SAAS;AAChC,YAAM,UAAU,KAAK,QAAQ,GAAG;AAChC,UAAI,CAAC,SAAS;AACZ,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AACA,YAAM,UAAUA,MAAQ,KAAK,oBAAoB;AACjD,UAAI,QAAQ,WAAW,QAAQ,KAAK;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AAAA,IACF;AACA,UAAM,WAA2B;AAAA,MAC/B,UAAU;AAAA,QACR,QAAQ,GAAG,MAAM;AAAA,QACjB,MAAM,GAAG,MAAM;AAAA,QACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAA;AAAA,MAAC;AAAA,MAEjD,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE;AAAA,QAC/B,CAAA,QAAO,EAAE,OAAO,UAAU;AAAA,MAAA;AAAA,MAE5B;AAAA,IAAA;AAEF,WAAO,EAAC,WAAW,MAAM,SAAA;AAAA,EAC3B,CAAC;AACH;"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAIzE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAa9D,OAAO,KAAK,EAAY,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAexE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAkB1D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,iBAiMvB;AAkGD,KAAK,eAAe,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAKF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAQ1B;AA6BD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,kBAAkB,CAgBpB"}
+{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAIzE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAa9D,OAAO,KAAK,EAAY,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAexE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAkB1D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,iBAkMvB;AAkGD,KAAK,eAAe,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAKF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAQ1B;AA6BD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,kBAAkB,CAgBpB"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -108,7 +108,9 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
     const numWorkers = platform() === "win32" ? numTables : Math.min(tableCopyWorkers, numTables);
     const copyPool = pgClient(lc, upstreamURI, {
       max: numWorkers,
-      connection: { ["application_name"]: "initial-sync-copy-worker" }
+      connection: { ["application_name"]: "initial-sync-copy-worker" },
+      ["max_lifetime"]: 120 * 60
+      // set a long (2h) limit for COPY streaming
     });
     const copiers = startTableCopyWorkers(
       lc,

package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"initial-sync.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n  createLiteIndexStatement,\n  createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n  JSON_STRINGIFIED,\n  liteValue,\n  type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {\n  pgClient,\n  type PostgresDB,\n  type PostgresTransaction,\n  type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n  addReplica,\n  dropShard,\n  getInternalShardConfig,\n  newReplicationSlot,\n  replicationSlotExpression,\n  validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n  tableCopyWorkers: number;\n  profileCopy?: boolean | undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n) {\n  if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n    throw new Error(\n      'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n    );\n  }\n  const {tableCopyWorkers, profileCopy} = syncOptions;\n  const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n  const sql = pgClient(lc, upstreamURI);\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const slotName = newReplicationSlot(shard);\n  const statusPublisher = new ReplicationStatusPublisher(tx).publish(\n    lc,\n    'Initializing',\n  );\n  try {\n    await checkUpstreamConfig(sql);\n\n    const {publications} = await ensurePublishedTables(lc, sql, shard);\n    lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n    const {database, host} = sql.options;\n    lc.info?.(`opening replication session to ${database}@${host}`);\n\n    let slot: ReplicationSlot;\n    for (let first = true; ; first = false) {\n      try {\n        slot = await createReplicationSlot(lc, replicationSession, slotName);\n        break;\n      } catch (e) {\n        if (first && e instanceof postgres.PostgresError) {\n          if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n            // Some Postgres variants (e.g. Google Cloud SQL) require that\n            // the user have the REPLICATION role in order to create a slot.\n            // Note that this must be done by the upstreamDB connection, and\n            // does not work in the replicationSession itself.\n            await sql`ALTER ROLE current_user WITH REPLICATION`;\n            lc.info?.(`Added the REPLICATION role to database user`);\n            continue;\n          }\n          if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n            const slotExpression = replicationSlotExpression(shard);\n\n            const dropped = await sql<{slot: string}[]>`\n              SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n                FROM pg_replication_slots\n                WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n            if (dropped.length) {\n              lc.warn?.(\n                `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n                e,\n              );\n              continue;\n            }\n            lc.error?.(`Unable to drop replication slots`, e);\n          }\n        }\n        throw e;\n      }\n    }\n    const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n    const initialVersion = toStateVersionString(lsn);\n\n    initReplicationState(tx, publications, initialVersion, context);\n\n    // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n    const start = performance.now();\n    // Retrieve the published schema at the consistent_point.\n    const published = await runTx(\n      sql,\n      async tx => {\n        await tx.unsafe(/* sql*/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n        return getPublicationInfo(tx, publications);\n      },\n      {mode: Mode.READONLY},\n    );\n    // Note: If this throws, initial-sync is aborted.\n    validatePublications(lc, published);\n\n    // Now that tables have been validated, kick off the copiers.\n    const {tables, indexes} = published;\n    const numTables = tables.length;\n    if (platform() === 'win32' && tableCopyWorkers < numTables) {\n      lc.warn?.(\n        `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n          `${numTables} to work around a Node/Postgres connection bug`,\n      );\n    }\n    const numWorkers =\n      platform() === 'win32'\n        ? numTables\n        : Math.min(tableCopyWorkers, numTables);\n\n    const copyPool = pgClient(lc, upstreamURI, {\n      max: numWorkers,\n      connection: {['application_name']: 'initial-sync-copy-worker'},\n    });\n    const copiers = startTableCopyWorkers(\n      lc,\n      copyPool,\n      snapshot,\n      numWorkers,\n      numTables,\n    );\n    try {\n      createLiteTables(tx, tables, initialVersion);\n      const downloads = await Promise.all(\n        tables.map(spec =>\n          copiers.processReadTask((db, lc) =>\n            getInitialDownloadState(lc, db, spec),\n          ),\n        ),\n      );\n      statusPublisher.publish(\n        lc,\n        'Initializing',\n        `Copying ${numTables} upstream tables at version ${initialVersion}`,\n        5000,\n        () => ({downloadStatus: downloads.map(({status}) => status)}),\n      );\n\n      void copyProfiler?.start();\n      const rowCounts = await Promise.all(\n        downloads.map(table =>\n          copiers.processReadTask((db, lc) =>\n            copy(lc, table, copyPool, db, tx),\n          ),\n        ),\n      );\n      void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n\n      const total = rowCounts.reduce(\n        (acc, curr) => ({\n          rows: acc.rows + curr.rows,\n          flushTime: acc.flushTime + curr.flushTime,\n        }),\n        {rows: 0, flushTime: 0},\n      );\n\n      statusPublisher.publish(\n        lc,\n        'Indexing',\n        `Creating ${indexes.length} indexes`,\n        5000,\n      );\n      const indexStart = performance.now();\n      createLiteIndices(tx, indexes);\n      const index = performance.now() - indexStart;\n      lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n      await addReplica(\n        sql,\n        shard,\n        slotName,\n        initialVersion,\n        published,\n        context,\n      );\n\n      const elapsed = performance.now() - start;\n      lc.info?.(\n        `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n          `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n      );\n    } finally {\n      copiers.setDone();\n      if (platform() === 'win32') {\n        // Workaround a Node bug in Windows in which certain COPY streams result\n        // in hanging the connection, which causes this await to never resolve.\n        void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n      } else {\n        await copyPool.end();\n      }\n    }\n  } catch (e) {\n    // If initial-sync did not succeed, make a best effort to drop the\n    // orphaned replication slot to avoid running out of slots in\n    // pathological cases that result in repeated failures.\n    lc.warn?.(`dropping replication slot ${slotName}`, e);\n    await sql`\n      SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n        WHERE slot_name = ${slotName};\n    `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n    await replicationSession.end();\n    await sql.end();\n  }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n  const {walLevel, version} = (\n    await sql<{walLevel: string; version: number}[]>`\n      SELECT current_setting('wal_level') as \"walLevel\", \n             current_setting('server_version_num') as \"version\";\n  `\n  )[0];\n\n  if (walLevel !== 'logical') {\n    throw new Error(\n      `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n    );\n  }\n  if (version < 150000) {\n    throw new Error(\n      `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n    );\n  }\n}\n\nasync function ensurePublishedTables(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  validate = true,\n): Promise<{publications: string[]}> {\n  const {database, host} = sql.options;\n  lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n  await ensureShardSchema(lc, sql, shard);\n  const {publications} = await getInternalShardConfig(sql, shard);\n\n  if (validate) {\n    let valid = false;\n    const nonInternalPublications = publications.filter(\n      p => !p.startsWith('_'),\n    );\n    const exists = await sql`\n      SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n      `.values();\n    if (exists.length !== publications.length) {\n      lc.warn?.(\n        `some configured publications [${publications}] are missing: ` +\n          `[${exists.flat()}]. resyncing`,\n      );\n    } else if (\n      !equals(new Set(shard.publications), new Set(nonInternalPublications))\n    ) {\n      lc.warn?.(\n        `requested publications [${shard.publications}] differ from previous` +\n          `publications [${nonInternalPublications}]. resyncing`,\n      );\n    } else {\n      valid = true;\n    }\n    if (!valid) {\n      await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n      return ensurePublishedTables(lc, sql, shard, false);\n    }\n  }\n  return {publications};\n}\n\nfunction startTableCopyWorkers(\n  lc: LogContext,\n  db: PostgresDB,\n  snapshot: string,\n  numWorkers: number,\n  numTables: number,\n): TransactionPool {\n  const {init} = importSnapshot(snapshot);\n  const tableCopiers = new TransactionPool(\n    lc,\n    Mode.READONLY,\n    init,\n    undefined,\n    numWorkers,\n  );\n  tableCopiers.run(db);\n\n  lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n  if (parseInt(process.versions.node) < 22) {\n    lc.warn?.(\n      `\\n\\n\\n` +\n        `Older versions of Node have a bug that results in an unresponsive\\n` +\n        `Postgres connection after running certain combinations of COPY commands.\\n` +\n        `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n        `benefit of being consistent with the Node version run in the production container image.` +\n        `\\n\\n\\n`,\n    );\n  }\n  return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  slotName: string,\n): Promise<ReplicationSlot> {\n  const slot = (\n    await session.unsafe<ReplicationSlot[]>(\n      /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n    )\n  )[0];\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nfunction createLiteTables(\n  tx: Database,\n  tables: PublishedTableSpec[],\n  initialVersion: string,\n) {\n  // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n  //       duplicating the ColumnMetadata logic.\n  const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n  for (const t of tables) {\n    tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n    const tableName = liteTableName(t);\n    for (const [colName, colSpec] of Object.entries(t.columns)) {\n      columnMetadata.insert(tableName, colName, colSpec);\n    }\n  }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n  for (const index of indices) {\n    tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n  }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n  select: string;\n  getTotalRows: string;\n  getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n  table: PublishedTableSpec,\n  cols: string[],\n): DownloadStatements {\n  const filterConditions = Object.values(table.publications)\n    .map(({rowFilter}) => rowFilter)\n    .filter(f => !!f); // remove nulls\n  const where =\n    filterConditions.length === 0\n      ? ''\n      : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n  const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n  const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n  const stmts = {\n    select: /*sql*/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n    getTotalRows: /*sql*/ `SELECT COUNT(*) AS \"totalRows\" ${fromTable}`,\n    getTotalBytes: /*sql*/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n  };\n  return stmts;\n}\n\ntype DownloadState = {\n  spec: PublishedTableSpec;\n  status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n  lc: LogContext,\n  sql: PostgresDB,\n  spec: PublishedTableSpec,\n): Promise<DownloadState> {\n  const start = performance.now();\n  const table = liteTableName(spec);\n  const columns = Object.keys(spec.columns);\n  const stmts = makeDownloadStatements(spec, columns);\n  const rowsResult = sql\n    .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n    .execute();\n  const bytesResult = sql\n    .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n    .execute();\n\n  const state: DownloadState = {\n    spec,\n    status: {\n      table,\n      columns,\n      rows: 0,\n      totalRows: Number((await rowsResult)[0].totalRows),\n      totalBytes: Number((await bytesResult)[0].totalBytes),\n    },\n  };\n  const elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n    state: state.status,\n  });\n  return state;\n}\n\nasync function copy(\n  lc: LogContext,\n  {spec: table, status}: DownloadState,\n  dbClient: PostgresDB,\n  from: PostgresTransaction,\n  to: Database,\n) {\n  const start = performance.now();\n  let flushTime = 0;\n\n  const tableName = liteTableName(table);\n  const orderedColumns = Object.entries(table.columns);\n\n  const columnNames = orderedColumns.map(([c]) => c);\n  const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n  const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n  // (?,?,?,?,?)\n  const valuesSql =\n    columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n  const insertSql = /*sql*/ `\n    INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n  const insertStmt = to.prepare(insertSql);\n  // INSERT VALUES (?,?,?,?,?),... x INSERT_BATCH_SIZE\n  const insertBatchStmt = to.prepare(\n    insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n  );\n\n  const {select} = makeDownloadStatements(table, columnNames);\n  const valuesPerRow = columnSpecs.length;\n  const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n  // Preallocate the buffer of values to reduce memory allocation churn.\n  const pendingValues: LiteValueType[] = Array.from({\n    length: MAX_BUFFERED_ROWS * valuesPerRow,\n  });\n  let pendingRows = 0;\n  let pendingSize = 0;\n\n  function flush() {\n    const start = performance.now();\n    const flushedRows = pendingRows;\n    const flushedSize = pendingSize;\n\n    let l = 0;\n    for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n      insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n    }\n    // Insert the remaining rows individually.\n    for (; pendingRows > 0; pendingRows--) {\n      insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n    }\n    for (let i = 0; i < flushedRows; i++) {\n      // Reuse the array and unreference the values to allow GC.\n      // This is faster than allocating a new array every time.\n      pendingValues[i] = undefined as unknown as LiteValueType;\n    }\n    pendingSize = 0;\n    status.rows += flushedRows;\n\n    const elapsed = performance.now() - start;\n    flushTime += elapsed;\n    lc.debug?.(\n      `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n    );\n  }\n\n  lc.info?.(`Starting copy stream of ${tableName}:`, select);\n  const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n  const parsers = columnSpecs.map(c => {\n    const pgParse = pgParsers.getTypeParser(c.typeOID);\n    return (val: string) =>\n      liteValue(\n        pgParse(val) as PostgresValueType,\n        c.dataType,\n        JSON_STRINGIFIED,\n      );\n  });\n\n  const tsvParser = new TsvParser();\n  let col = 0;\n\n  await pipeline(\n    await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n    new Writable({\n      highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n      write(\n        chunk: Buffer,\n        _encoding: string,\n        callback: (error?: Error) => void,\n      ) {\n        try {\n          for (const text of tsvParser.parse(chunk)) {\n            pendingSize += text === null ? 4 : text.length;\n            pendingValues[pendingRows * valuesPerRow + col] =\n              text === null ? null : parsers[col](text);\n\n            if (++col === parsers.length) {\n              col = 0;\n              if (\n                ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n                pendingSize >= BUFFERED_SIZE_THRESHOLD\n              ) {\n                flush();\n              }\n            }\n          }\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n\n      final: (callback: (error?: Error) => void) => {\n        try {\n          flush();\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n    }),\n  );\n\n  const elapsed = performance.now() - start;\n  lc.info?.(\n    `Finished copying ${status.rows} rows into ${tableName} ` +\n      `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n  );\n  return {rows: status.rows, flushTime};\n}\n"],"names":["slot","tx","Mode.READONLY","lc","e","start","elapsed"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,MAAI,CAAC,0BAA0B,KAAK,MAAM,KAAK,GAAG;AAChD,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AACA,QAAM,EAAC,kBAAkB,YAAA,IAAe;AACxC,QAAM,eAAe,cAAc,MAAM,YAAY,YAAY;AACjE,QAAM,MAAM,SAAS,IAAI,WAAW;AACpC,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,mBAAmB,KAAK;AACzC,QAAM,kBAAkB,IAAI,2BAA2B,EAAE,EAAE;AAAA,IACzD;AAAA,IACA;AAAA,EAAA;AAEF,MAAI;AACF,UAAM,oBAAoB,GAAG;AAE7B,UAAM,EAAC,aAAA,IAAgB,MAAM,sBAAsB,IAAI,KAAK,KAAK;AACjE,OAAG,OAAO,wCAAwC,YAAY,GAAG;AAEjE,UAAM,EAAC,UAAU,KAAA,IAAQ,IAAI;AAC7B,OAAG,OAAO,kCAAkC,QAAQ,IAAI,IAAI,EAAE;AAE9D,QAAI;AACJ,aAAS,QAAQ,QAAQ,QAAQ,OAAO;AACtC,UAAI;AACF,eAAO,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AACnE;AAAA,MACF,SAAS,GAAG;AACV,YAAI,SAAS,aAAa,SAAS,eAAe;AAChD,cAAI,EAAE,SAAS,2BAA2B;AAKxC,kBAAM;AACN,eAAG,OAAO,6CAA6C;AACvD;AAAA,UACF;AACA,cAAI,EAAE,SAAS,iCAAiC;AAC9C,kBAAM,iBAAiB,0BAA0B,KAAK;AAEtD,kBAAM,UAAU,MAAM;AAAA;AAAA;AAAA,uCAGK,cAAc;AACzC,gBAAI,QAAQ,QAAQ;AAClB,iBAAG;AAAA,gBACD,uCAAuC,QAAQ,IAAI,CAAC,EAAC,MAAAA,MAAAA,MAAUA,KAAI,CAAC;AAAA,gBACpE;AAAA,cAAA;AAEF;AAAA,YACF;AACA,eAAG,QAAQ,oCAAoC,CAAC;AAAA,UAClD;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAAA,IACF;AACA,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;AACzD,UAAM,iBAAiB,qBAAqB,GAAG;AAE/C,yBAAqB,IAAI,cAAc,gBAAgB,OAAO;AAG9D,UAAM,QAAQ,YAAY,IAAA;AAE1B,UAAM,YAAY,MAAM;AAAA,MACtB;AAAA,MACA,OAAMC,QAAM;AACV,cAAMA,IAAG;AAAA;AAAA,UAAgB,6BAA6B,QAAQ;AAAA,QAAA;AAC9D,eAAO,mBAAmBA,KAAI,YAAY;AAAA,MAC5C;AAAA,MACA,EAAC,MAAMC,SAAK;AAAA,IAAQ;AAGtB,yBAAqB,IAAI,SAAS;AAGlC,UAAM,EAAC,QAAQ,QAAA,IAAW;AAC1B,UAAM,YAAY,OAAO;AACzB,QAAI,SAAA,MAAe,WAAW,mBAAmB,WAAW;AAC1D,SAAG;AAAA,QACD,8CAA8C,gBAAgB,OACzD,SAAS;AAAA,MAAA;AAAA,IAElB;AACA,UAAM,aACJ,eAAe,UACX,YACA,KAAK,IAAI,kBAAkB,SAAS;AAE1C,UAAM,WAAW,SAAS,IAAI,aAAa;AAAA,MACzC,KAAK;AAAA,MACL,YAAY,EAAC,CAAC,kBAAkB,GAAG,2BAAA;AAAA,IAA0B,CAC9D;AACD,UAAM,UAAU;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI;AACF,uBAAiB,IAAI,QAAQ,cAAc;AAC3C,YAAM,YAAY,MAAM,QAAQ;AAAA,QAC9B,OAAO;AAAA,UAAI,UACT,QAAQ;AAAA,YAAgB,CAAC,IAAIC,QAC3B,wBAAwBA,KAAI,IAAI,IAAI;AAAA,UAAA;AAAA,QACtC;AAAA,MACF;AAEF,sBAAgB;AAAA,QACd;AAAA,QACA;AAAA,QACA,WAAW,SAAS,+BAA+B,cAAc;AAAA,QACjE;AAAA,QACA,OAAO,EAAC,gBAAgB,UAAU,IAAI,CAAC,EAAC,OAAA,MAAY,MAAM,EAAA;AAAA,MAAC;AAG7D,WAAK,cAAc,MAAA;AACnB,YAAM,YAAY,MAAM,QAAQ;AAAA,QAC9B,UAAU;AAAA,UAAI,WACZ,QAAQ;AAAA,YAAgB,CAAC,IAAIA,QAC3B,KAAKA,KAAI,OAAO,UAAU,IAAI,EAAE;AAAA,UAAA;AAAA,QAClC;AAAA,MACF;AAEF,WAAK,cAAc,eAAe,IAAI,cAAc;AAEpD,YAAM,QAAQ,UAAU;AAAA,QACtB,CAAC,KAAK,UAAU;AAAA,UACd,MAAM,IAAI,OAAO,KAAK;AAAA,UACtB,WAAW,IAAI,YAAY,KAAK;AAAA,QAAA;AAAA,QAElC,EAAC,MAAM,GAAG,WAAW,EAAA;AAAA,MAAC;AAGxB,sBAAgB;AAAA,QACd;AAAA,QACA;AAAA,QACA,YAAY,QAAQ,MAAM;AAAA,QAC1B;AAAA,MAAA;AAEF,YAAM,aAAa,YAAY,IAAA;AAC/B,wBAAkB,IAAI,OAAO;AAC7B,YAAM,QAAQ,YAAY,IAAA,IAAQ;AAClC,SAAG,OAAO,oBAAoB,MAAM,QAAQ,CAAC,CAAC,MAAM;AAEpD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAGF,YAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,SAAG;AAAA,QACD,UAAU,MAAM,KAAK,eAAA,CAAgB,YAAY,SAAS,cAAc,YAAY,UAAU,GAAG,YACpF,MAAM,UAAU,QAAQ,CAAC,CAAC,YAAY,MAAM,QAAQ,CAAC,CAAC,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,MAAA;AAAA,IAErG,UAAA;AACE,cAAQ,QAAA;AACR,UAAI,SAAA,MAAe,SAAS;AAG1B,aAAK,SAAS,MAAM,MAAM,OAAK,GAAG,OAAO,0BAA0B,CAAC,CAAC;AAAA,MACvE,OAAO;AACL,cAAM,SAAS,IAAA;AAAA,MACjB;AAAA,IACF;AAAA,EACF,SAAS,GAAG;AAIV,OAAG,OAAO,6BAA6B,QAAQ,IAAI,CAAC;AACpD,UAAM;AAAA;AAAA,4BAEkB,QAAQ;AAAA,MAC9B,MAAM,CAAAC,OAAK,GAAG,OAAO,mCAAmC,QAAQ,IAAIA,EAAC,CAAC;AACxE,UAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;AAAA,EAClE,UAAA;AACE,oBAAgB,KAAA;AAChB,UAAM,mBAAmB,IAAA;AACzB,UAAM,IAAI,IAAA;AAAA,EACZ;AACF;AAEA,eAAe,oBAAoB,KAAiB;AAClD,QAAM,EAAC,UAAU,QAAA,KACf,MAAM;AAAA;AAAA;AAAA,KAIN,CAAC;AAEH,MAAI,aAAa,WAAW;AAC1B,UAAM,IAAI;AAAA,MACR,uEAAuE,QAAQ;AAAA,IAAA;AAAA,EAEnF;AACA,MAAI,UAAU,MAAQ;AACpB,UAAM,IAAI;AAAA,MACR,sDAAsD,OAAO;AAAA,IAAA;AAAA,EAEjE;AACF;AAEA,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;AACnC,QAAM,EAAC,UAAU,KAAA,IAAQ,IAAI;AAC7B,KAAG,OAAO,oCAAoC,QAAQ,IAAI,IAAI,EAAE;AAEhE,QAAM,kBAAkB,IAAI,KAAK,KAAK;AACtC,QAAM,EAAC,aAAA,IAAgB,MAAM,uBAAuB,KAAK,KAAK;AAE9D,MAAI,UAAU;AACZ,QAAI,QAAQ;AACZ,UAAM,0BAA0B,aAAa;AAAA,MAC3C,CAAA,MAAK,CAAC,EAAE,WAAW,GAAG;AAAA,IAAA;AAExB,UAAM,SAAS,MAAM;AAAA,4DACmC,IAAI,YAAY,CAAC;AAAA,QACrE,OAAA;AACJ,QAAI,OAAO,WAAW,aAAa,QAAQ;AACzC,SAAG;AAAA,QACD,iCAAiC,YAAY,mBACvC,OAAO,MAAM;AAAA,MAAA;AAAA,IAEvB,WACE,CAAC,OAAO,IAAI,IAAI,MAAM,YAAY,GAAG,IAAI,IAAI,uBAAuB,CAAC,GACrE;AACA,SAAG;AAAA,QACD,2BAA2B,MAAM,YAAY,uCAC1B,uBAAuB;AAAA,MAAA;AAAA,IAE9C,OAAO;AACL,cAAQ;AAAA,IACV;AACA,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;AACvD,aAAO,sBAAsB,IAAI,KAAK,OAAO,KAAK;AAAA,IACpD;AAAA,EACF;AACA,SAAO,EAAC,aAAA;AACV;AAEA,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;AACjB,QAAM,EAAC,KAAA,IAAQ,eAAe,QAAQ;AACtC,QAAM,eAAe,IAAI;AAAA,IACvB;AAAA,IACAF;AAAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEF,eAAa,IAAI,EAAE;AAEnB,KAAG,OAAO,WAAW,UAAU,oBAAoB,SAAS,SAAS;AAErE,MAAI,SAAS,QAAQ,SAAS,IAAI,IAAI,IAAI;AACxC,OAAG;AAAA,MACD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAOJ;AACA,SAAO;AACT;AAaA,eAAsB,sBACpB,IACA,SACA,UAC0B;AAC1B,QAAM,QACJ,MAAM,QAAQ;AAAA;AAAA,IACJ,4BAA4B,QAAQ;AAAA,EAAA,GAE9C,CAAC;AACH,KAAG,OAAO,4BAA4B,QAAQ,IAAI,IAAI;AACtD,SAAO;AACT;AAEA,SAAS,iBACP,IACA,QACA,gBACA;AAGA,QAAM,iBAAiB,KAAK,oBAAoB,YAAY,EAAE,CAAC;AAC/D,aAAW,KAAK,QAAQ;AACtB,OAAG,KAAK,yBAAyB,kBAAkB,GAAG,cAAc,CAAC,CAAC;AACtE,UAAM,YAAY,cAAc,CAAC;AACjC,eAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,EAAE,OAAO,GAAG;AAC1D,qBAAe,OAAO,WAAW,SAAS,OAAO;AAAA,IACnD;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,aAAW,SAAS,SAAS;AAC3B,OAAG,KAAK,yBAAyB,uBAAuB,KAAK,CAAC,CAAC;AAAA,EACjE;AACF;AAMO,MAAM,oBAAoB;AAEjC,MAAM,KAAK,OAAO;AAClB,MAAM,oBAAoB;AAC1B,MAAM,0BAA0B,IAAI;AAQ7B,SAAS,uBACd,OACA,MACoB;AACpB,QAAM,mBAAmB,OAAO,OAAO,MAAM,YAAY,EACtD,IAAI,CAAC,EAAC,UAAA,MAAe,SAAS,EAC9B,OAAO,CAAA,MAAK,CAAC,CAAC,CAAC;AAClB,QAAM,QACJ,iBAAiB,WAAW,IACxB;AAAA;AAAA,IACQ,SAAS,iBAAiB,KAAK,MAAM,CAAC;AAAA;AACpD,QAAM;AAAA;AAAA,IAAoB,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,KAAK;AAAA;AAC7E,QAAM,aAAa,IAAI,KAAK,IAAI,CAAA,QAAO,+BAA+B,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,KAAK,CAAC;AAClG,QAAM,QAAQ;AAAA,IACZ;AAAA;AAAA,MAAgB,UAAU,KAAK,IAAI,EAAE,EAAE,KAAK,GAAG,CAAC,IAAI,SAAS;AAAA;AAAA,IAC7D;AAAA;AAAA,MAAsB,kCAAkC,SAAS;AAAA;AAAA,IACjE;AAAA;AAAA,MAAuB,UAAU,UAAU,oBAAoB,SAAS;AAAA;AAAA,EAAA;AAE1E,SAAO;AACT;AAOA,eAAe,wBACb,IACA,KACA,MACwB;AACxB,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,QAAQ,cAAc,IAAI;AAChC,QAAM,UAAU,OAAO,KAAK,KAAK,OAAO;AACxC,QAAM,QAAQ,uBAAuB,MAAM,OAAO;AAClD,QAAM,aAAa,IAChB,OAA8B,MAAM,YAAY,EAChD,QAAA;AACH,QAAM,cAAc,IACjB,OAA+B,MAAM,aAAa,EAClD,QAAA;AAEH,QAAM,QAAuB;AAAA,IAC3B;AAAA,IACA,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA,MAAM;AAAA,MACN,WAAW,QAAQ,MAAM,YAAY,CAAC,EAAE,SAAS;AAAA,MACjD,YAAY,QAAQ,MAAM,aAAa,CAAC,EAAE,UAAU;AAAA,IAAA;AAAA,EACtD;AAEF,QAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,KAAG,OAAO,uCAAuC,KAAK,KAAK,OAAO,QAAQ;AAAA,IACxE,OAAO,MAAM;AAAA,EAAA,CACd;AACD,SAAO;AACT;AAEA,eAAe,KACb,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA;AACA,QAAM,QAAQ,YAAY,IAAA;AAC1B,MAAI,YAAY;AAEhB,QAAM,YAAY,cAAc,KAAK;AACrC,QAAM,iBAAiB,OAAO,QAAQ,MAAM,OAAO;AAEnD,QAAM,cAAc,eAAe,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACjD,QAAM,cAAc,eAAe,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,IAAI;AAC9D,QAAM,mBAAmB,YAAY,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG;AAG7D,QAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,CAAC,CAAC,OAAO;AACzE,QAAM;AAAA;AAAA,IAAoB;AAAA,mBACT,SAAS,MAAM,gBAAgB,YAAY,SAAS;AAAA;AACrE,QAAM,aAAa,GAAG,QAAQ,SAAS;AAEvC,QAAM,kBAAkB,GAAG;AAAA,IACzB,YAAY,IAAI,SAAS,GAAG,OAAO,oBAAoB,CAAC;AAAA,EAAA;AAG1D,QAAM,EAAC,OAAA,IAAU,uBAAuB,OAAO,WAAW;AAC1D,QAAM,eAAe,YAAY;AACjC,QAAM,iBAAiB,eAAe;AAGtC,QAAM,gBAAiC,MAAM,KAAK;AAAA,IAChD,QAAQ,oBAAoB;AAAA,EAAA,CAC7B;AACD,MAAI,cAAc;AAClB,MAAI,cAAc;AAElB,WAAS,QAAQ;AACf,UAAMG,SAAQ,YAAY,IAAA;AAC1B,UAAM,cAAc;AACpB,UAAM,cAAc;AAEpB,QAAI,IAAI;AACR,WAAO,cAAc,mBAAmB,eAAe,mBAAmB;AACxE,sBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,cAAe,CAAC;AAAA,IACnE;AAEA,WAAO,cAAc,GAAG,eAAe;AACrC,iBAAW,IAAI,cAAc,MAAM,GAAI,KAAK,YAAa,CAAC;AAAA,IAC5D;AACA,aAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AAGpC,oBAAc,CAAC,IAAI;AAAA,IACrB;AACA,kBAAc;AACd,WAAO,QAAQ;AAEf,UAAMC,WAAU,YAAY,IAAA,IAAQD;AACpC,iBAAaC;AACb,OAAG;AAAA,MACD,WAAW,WAAW,IAAI,SAAS,UAAU,WAAW,cAAcA,SAAQ,QAAQ,CAAC,CAAC;AAAA,IAAA;AAAA,EAE5F;AAEA,KAAG,OAAO,2BAA2B,SAAS,KAAK,MAAM;AACzD,QAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK;AAC3E,QAAM,UAAU,YAAY,IAAI,CAAA,MAAK;AACnC,UAAM,UAAU,UAAU,cAAc,EAAE,OAAO;AACjD,WAAO,CAAC,QACN;AAAA,MACE,QAAQ,GAAG;AAAA,MACX,EAAE;AAAA,MACF;AAAA,IAAA;AAAA,EAEN,CAAC;AAED,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,MAAM;AAEV,QAAM;AAAA,IACJ,MAAM,KAAK,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,IAChD,IAAI,SAAS;AAAA,MACX,eAAe;AAAA,MAEf,MACE,OACA,WACA,UACA;AACA,YAAI;AACF,qBAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,2BAAe,SAAS,OAAO,IAAI,KAAK;AACxC,0BAAc,cAAc,eAAe,GAAG,IAC5C,SAAS,OAAO,OAAO,QAAQ,GAAG,EAAE,IAAI;AAE1C,gBAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,oBAAM;AACN,kBACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,yBACf;AACA,sBAAA;AAAA,cACF;AAAA,YACF;AAAA,UACF;AACA,mBAAA;AAAA,QACF,SAAS,GAAG;AACV,mBAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAAA,MAEA,OAAO,CAAC,aAAsC;AAC5C,YAAI;AACF,gBAAA;AACA,mBAAA;AAAA,QACF,SAAS,GAAG;AACV,mBAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAAA,IAAA,CACD;AAAA,EAAA;AAGH,QAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,KAAG;AAAA,IACD,oBAAoB,OAAO,IAAI,cAAc,SAAS,YACzC,UAAU,QAAQ,CAAC,CAAC,gBAAgB,QAAQ,QAAQ,CAAC,CAAC;AAAA,EAAA;AAErE,SAAO,EAAC,MAAM,OAAO,MAAM,UAAA;AAC7B;"}
+{"version":3,"file":"initial-sync.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n  createLiteIndexStatement,\n  createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n  JSON_STRINGIFIED,\n  liteValue,\n  type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {\n  pgClient,\n  type PostgresDB,\n  type PostgresTransaction,\n  type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n  addReplica,\n  dropShard,\n  getInternalShardConfig,\n  newReplicationSlot,\n  replicationSlotExpression,\n  validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n  tableCopyWorkers: number;\n  profileCopy?: boolean | undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n) {\n  if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n    throw new Error(\n      'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n    );\n  }\n  const {tableCopyWorkers, profileCopy} = syncOptions;\n  const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n  const sql = pgClient(lc, upstreamURI);\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const slotName = newReplicationSlot(shard);\n  const statusPublisher = new ReplicationStatusPublisher(tx).publish(\n    lc,\n    'Initializing',\n  );\n  try {\n    await checkUpstreamConfig(sql);\n\n    const {publications} = await ensurePublishedTables(lc, sql, shard);\n    lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n    const {database, host} = sql.options;\n    lc.info?.(`opening replication session to ${database}@${host}`);\n\n    let slot: ReplicationSlot;\n    for (let first = true; ; first = false) {\n      try {\n        slot = await createReplicationSlot(lc, replicationSession, slotName);\n        break;\n      } catch (e) {\n        if (first && e instanceof postgres.PostgresError) {\n          if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n            // Some Postgres variants (e.g. Google Cloud SQL) require that\n            // the user have the REPLICATION role in order to create a slot.\n            // Note that this must be done by the upstreamDB connection, and\n            // does not work in the replicationSession itself.\n            await sql`ALTER ROLE current_user WITH REPLICATION`;\n            lc.info?.(`Added the REPLICATION role to database user`);\n            continue;\n          }\n          if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n            const slotExpression = replicationSlotExpression(shard);\n\n            const dropped = await sql<{slot: string}[]>`\n              SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n                FROM pg_replication_slots\n                WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n            if (dropped.length) {\n              lc.warn?.(\n                `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n                e,\n              );\n              continue;\n            }\n            lc.error?.(`Unable to drop replication slots`, e);\n          }\n        }\n        throw e;\n      }\n    }\n    const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n    const initialVersion = toStateVersionString(lsn);\n\n    initReplicationState(tx, publications, initialVersion, context);\n\n    // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n    const start = performance.now();\n    // Retrieve the published schema at the consistent_point.\n    const published = await runTx(\n      sql,\n      async tx => {\n        await tx.unsafe(/* sql*/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n        return getPublicationInfo(tx, publications);\n      },\n      {mode: Mode.READONLY},\n    );\n    // Note: If this throws, initial-sync is aborted.\n    validatePublications(lc, published);\n\n    // Now that tables have been validated, kick off the copiers.\n    const {tables, indexes} = published;\n    const numTables = tables.length;\n    if (platform() === 'win32' && tableCopyWorkers < numTables) {\n      lc.warn?.(\n        `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n          `${numTables} to work around a Node/Postgres connection bug`,\n      );\n    }\n    const numWorkers =\n      platform() === 'win32'\n        ? numTables\n        : Math.min(tableCopyWorkers, numTables);\n\n    const copyPool = pgClient(lc, upstreamURI, {\n      max: numWorkers,\n      connection: {['application_name']: 'initial-sync-copy-worker'},\n      ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n    });\n    const copiers = startTableCopyWorkers(\n      lc,\n      copyPool,\n      snapshot,\n      numWorkers,\n      numTables,\n    );\n    try {\n      createLiteTables(tx, tables, initialVersion);\n      const downloads = await Promise.all(\n        tables.map(spec =>\n          copiers.processReadTask((db, lc) =>\n            getInitialDownloadState(lc, db, spec),\n          ),\n        ),\n      );\n      statusPublisher.publish(\n        lc,\n        'Initializing',\n        `Copying ${numTables} upstream tables at version ${initialVersion}`,\n        5000,\n        () => ({downloadStatus: downloads.map(({status}) => status)}),\n      );\n\n      void copyProfiler?.start();\n      const rowCounts = await Promise.all(\n        downloads.map(table =>\n          copiers.processReadTask((db, lc) =>\n            copy(lc, table, copyPool, db, tx),\n          ),\n        ),\n      );\n      void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n\n      const total = rowCounts.reduce(\n        (acc, curr) => ({\n          rows: acc.rows + curr.rows,\n          flushTime: acc.flushTime + curr.flushTime,\n        }),\n        {rows: 0, flushTime: 0},\n      );\n\n      statusPublisher.publish(\n        lc,\n        'Indexing',\n        `Creating ${indexes.length} indexes`,\n        5000,\n      );\n      const indexStart = performance.now();\n      createLiteIndices(tx, indexes);\n      const index = performance.now() - indexStart;\n      lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n      await addReplica(\n        sql,\n        shard,\n        slotName,\n        initialVersion,\n        published,\n        context,\n      );\n\n      const elapsed = performance.now() - start;\n      lc.info?.(\n        `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n          `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n      );\n    } finally {\n      copiers.setDone();\n      if (platform() === 'win32') {\n        // Workaround a Node bug in Windows in which certain COPY streams result\n        // in hanging the connection, which causes this await to never resolve.\n        void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n      } else {\n        await copyPool.end();\n      }\n    }\n  } catch (e) {\n    // If initial-sync did not succeed, make a best effort to drop the\n    // orphaned replication slot to avoid running out of slots in\n    // pathological cases that result in repeated failures.\n    lc.warn?.(`dropping replication slot ${slotName}`, e);\n    await sql`\n      SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n        WHERE slot_name = ${slotName};\n    `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n    await replicationSession.end();\n    await sql.end();\n  }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n  const {walLevel, version} = (\n    await sql<{walLevel: string; version: number}[]>`\n      SELECT current_setting('wal_level') as \"walLevel\", \n             current_setting('server_version_num') as \"version\";\n  `\n  )[0];\n\n  if (walLevel !== 'logical') {\n    throw new Error(\n      `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n    );\n  }\n  if (version < 150000) {\n    throw new Error(\n      `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n    );\n  }\n}\n\nasync function ensurePublishedTables(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  validate = true,\n): Promise<{publications: string[]}> {\n  const {database, host} = sql.options;\n  lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n  await ensureShardSchema(lc, sql, shard);\n  const {publications} = await getInternalShardConfig(sql, shard);\n\n  if (validate) {\n    let valid = false;\n    const nonInternalPublications = publications.filter(\n      p => !p.startsWith('_'),\n    );\n    const exists = await sql`\n      SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n      `.values();\n    if (exists.length !== publications.length) {\n      lc.warn?.(\n        `some configured publications [${publications}] are missing: ` +\n          `[${exists.flat()}]. resyncing`,\n      );\n    } else if (\n      !equals(new Set(shard.publications), new Set(nonInternalPublications))\n    ) {\n      lc.warn?.(\n        `requested publications [${shard.publications}] differ from previous` +\n          `publications [${nonInternalPublications}]. resyncing`,\n      );\n    } else {\n      valid = true;\n    }\n    if (!valid) {\n      await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n      return ensurePublishedTables(lc, sql, shard, false);\n    }\n  }\n  return {publications};\n}\n\nfunction startTableCopyWorkers(\n  lc: LogContext,\n  db: PostgresDB,\n  snapshot: string,\n  numWorkers: number,\n  numTables: number,\n): TransactionPool {\n  const {init} = importSnapshot(snapshot);\n  const tableCopiers = new TransactionPool(\n    lc,\n    Mode.READONLY,\n    init,\n    undefined,\n    numWorkers,\n  );\n  tableCopiers.run(db);\n\n  lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n  if (parseInt(process.versions.node) < 22) {\n    lc.warn?.(\n      `\\n\\n\\n` +\n        `Older versions of Node have a bug that results in an unresponsive\\n` +\n        `Postgres connection after running certain combinations of COPY commands.\\n` +\n        `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n        `benefit of being consistent with the Node version run in the production container image.` +\n        `\\n\\n\\n`,\n    );\n  }\n  return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  slotName: string,\n): Promise<ReplicationSlot> {\n  const slot = (\n    await session.unsafe<ReplicationSlot[]>(\n      /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n    )\n  )[0];\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nfunction createLiteTables(\n  tx: Database,\n  tables: PublishedTableSpec[],\n  initialVersion: string,\n) {\n  // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n  //       duplicating the ColumnMetadata logic.\n  const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n  for (const t of tables) {\n    tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n    const tableName = liteTableName(t);\n    for (const [colName, colSpec] of Object.entries(t.columns)) {\n      columnMetadata.insert(tableName, colName, colSpec);\n    }\n  }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n  for (const index of indices) {\n    tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n  }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n  select: string;\n  getTotalRows: string;\n  getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n  table: PublishedTableSpec,\n  cols: string[],\n): DownloadStatements {\n  const filterConditions = Object.values(table.publications)\n    .map(({rowFilter}) => rowFilter)\n    .filter(f => !!f); // remove nulls\n  const where =\n    filterConditions.length === 0\n      ? ''\n      : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n  const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n  const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n  const stmts = {\n    select: /*sql*/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n    getTotalRows: /*sql*/ `SELECT COUNT(*) AS \"totalRows\" ${fromTable}`,\n    getTotalBytes: /*sql*/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n  };\n  return stmts;\n}\n\ntype DownloadState = {\n  spec: PublishedTableSpec;\n  status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n  lc: LogContext,\n  sql: PostgresDB,\n  spec: PublishedTableSpec,\n): Promise<DownloadState> {\n  const start = performance.now();\n  const table = liteTableName(spec);\n  const columns = Object.keys(spec.columns);\n  const stmts = makeDownloadStatements(spec, columns);\n  const rowsResult = sql\n    .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n    .execute();\n  const bytesResult = sql\n    .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n    .execute();\n\n  const state: DownloadState = {\n    spec,\n    status: {\n      table,\n      columns,\n      rows: 0,\n      totalRows: Number((await rowsResult)[0].totalRows),\n      totalBytes: Number((await bytesResult)[0].totalBytes),\n    },\n  };\n  const elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n    state: state.status,\n  });\n  return state;\n}\n\nasync function copy(\n  lc: LogContext,\n  {spec: table, status}: DownloadState,\n  dbClient: PostgresDB,\n  from: PostgresTransaction,\n  to: Database,\n) {\n  const start = performance.now();\n  let flushTime = 0;\n\n  const tableName = liteTableName(table);\n  const orderedColumns = Object.entries(table.columns);\n\n  const columnNames = orderedColumns.map(([c]) => c);\n  const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n  const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n  // (?,?,?,?,?)\n  const valuesSql =\n    columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n  const insertSql = /*sql*/ `\n    INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n  const insertStmt = to.prepare(insertSql);\n  // INSERT VALUES (?,?,?,?,?),... x INSERT_BATCH_SIZE\n  const insertBatchStmt = to.prepare(\n    insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n  );\n\n  const {select} = makeDownloadStatements(table, columnNames);\n  const valuesPerRow = columnSpecs.length;\n  const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n  // Preallocate the buffer of values to reduce memory allocation churn.\n  const pendingValues: LiteValueType[] = Array.from({\n    length: MAX_BUFFERED_ROWS * valuesPerRow,\n  });\n  let pendingRows = 0;\n  let pendingSize = 0;\n\n  function flush() {\n    const start = performance.now();\n    const flushedRows = pendingRows;\n    const flushedSize = pendingSize;\n\n    let l = 0;\n    for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n      insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n    }\n    // Insert the remaining rows individually.\n    for (; pendingRows > 0; pendingRows--) {\n      insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n    }\n    for (let i = 0; i < flushedRows; i++) {\n      // Reuse the array and unreference the values to allow GC.\n      // This is faster than allocating a new array every time.\n      pendingValues[i] = undefined as unknown as LiteValueType;\n    }\n    pendingSize = 0;\n    status.rows += flushedRows;\n\n    const elapsed = performance.now() - start;\n    flushTime += elapsed;\n    lc.debug?.(\n      `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n    );\n  }\n\n  lc.info?.(`Starting copy stream of ${tableName}:`, select);\n  const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n  const parsers = columnSpecs.map(c => {\n    const pgParse = pgParsers.getTypeParser(c.typeOID);\n    return (val: string) =>\n      liteValue(\n        pgParse(val) as PostgresValueType,\n        c.dataType,\n        JSON_STRINGIFIED,\n      );\n  });\n\n  const tsvParser = new TsvParser();\n  let col = 0;\n\n  await pipeline(\n    await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n    new Writable({\n      highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n      write(\n        chunk: Buffer,\n        _encoding: string,\n        callback: (error?: Error) => void,\n      ) {\n        try {\n          for (const text of tsvParser.parse(chunk)) {\n            pendingSize += text === null ? 4 : text.length;\n            pendingValues[pendingRows * valuesPerRow + col] =\n              text === null ? null : parsers[col](text);\n\n            if (++col === parsers.length) {\n              col = 0;\n              if (\n                ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n                pendingSize >= BUFFERED_SIZE_THRESHOLD\n              ) {\n                flush();\n              }\n            }\n          }\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n\n      final: (callback: (error?: Error) => void) => {\n        try {\n          flush();\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n    }),\n  );\n\n  const elapsed = performance.now() - start;\n  lc.info?.(\n    `Finished copying ${status.rows} rows into ${tableName} ` +\n      `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n  );\n  return {rows: status.rows, flushTime};\n}\n"],"names":["slot","tx","Mode.READONLY","lc","e","start","elapsed"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,MAAI,CAAC,0BAA0B,KAAK,MAAM,KAAK,GAAG;AAChD,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AACA,QAAM,EAAC,kBAAkB,YAAA,IAAe;AACxC,QAAM,eAAe,cAAc,MAAM,YAAY,YAAY;AACjE,QAAM,MAAM,SAAS,IAAI,WAAW;AACpC,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,mBAAmB,KAAK;AACzC,QAAM,kBAAkB,IAAI,2BAA2B,EAAE,EAAE;AAAA,IACzD;AAAA,IACA;AAAA,EAAA;AAEF,MAAI;AACF,UAAM,oBAAoB,GAAG;AAE7B,UAAM,EAAC,aAAA,IAAgB,MAAM,sBAAsB,IAAI,KAAK,KAAK;AACjE,OAAG,OAAO,wCAAwC,YAAY,GAAG;AAEjE,UAAM,EAAC,UAAU,KAAA,IAAQ,IAAI;AAC7B,OAAG,OAAO,kCAAkC,QAAQ,IAAI,IAAI,EAAE;AAE9D,QAAI;AACJ,aAAS,QAAQ,QAAQ,QAAQ,OAAO;AACtC,UAAI;AACF,eAAO,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AACnE;AAAA,MACF,SAAS,GAAG;AACV,YAAI,SAAS,aAAa,SAAS,eAAe;AAChD,cAAI,EAAE,SAAS,2BAA2B;AAKxC,kBAAM;AACN,eAAG,OAAO,6CAA6C;AACvD;AAAA,UACF;AACA,cAAI,EAAE,SAAS,iCAAiC;AAC9C,kBAAM,iBAAiB,0BAA0B,KAAK;AAEtD,kBAAM,UAAU,MAAM;AAAA;AAAA;AAAA,uCAGK,cAAc;AACzC,gBAAI,QAAQ,QAAQ;AAClB,iBAAG;AAAA,gBACD,uCAAuC,QAAQ,IAAI,CAAC,EAAC,MAAAA,MAAAA,MAAUA,KAAI,CAAC;AAAA,gBACpE;AAAA,cAAA;AAEF;AAAA,YACF;AACA,eAAG,QAAQ,oCAAoC,CAAC;AAAA,UAClD;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAAA,IACF;AACA,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;AACzD,UAAM,iBAAiB,qBAAqB,GAAG;AAE/C,yBAAqB,IAAI,cAAc,gBAAgB,OAAO;AAG9D,UAAM,QAAQ,YAAY,IAAA;AAE1B,UAAM,YAAY,MAAM;AAAA,MACtB;AAAA,MACA,OAAMC,QAAM;AACV,cAAMA,IAAG;AAAA;AAAA,UAAgB,6BAA6B,QAAQ;AAAA,QAAA;AAC9D,eAAO,mBAAmBA,KAAI,YAAY;AAAA,MAC5C;AAAA,MACA,EAAC,MAAMC,SAAK;AAAA,IAAQ;AAGtB,yBAAqB,IAAI,SAAS;AAGlC,UAAM,EAAC,QAAQ,QAAA,IAAW;AAC1B,UAAM,YAAY,OAAO;AACzB,QAAI,SAAA,MAAe,WAAW,mBAAmB,WAAW;AAC1D,SAAG;AAAA,QACD,8CAA8C,gBAAgB,OACzD,SAAS;AAAA,MAAA;AAAA,IAElB;AACA,UAAM,aACJ,eAAe,UACX,YACA,KAAK,IAAI,kBAAkB,SAAS;AAE1C,UAAM,WAAW,SAAS,IAAI,aAAa;AAAA,MACzC,KAAK;AAAA,MACL,YAAY,EAAC,CAAC,kBAAkB,GAAG,2BAAA;AAAA,MACnC,CAAC,cAAc,GAAG,MAAM;AAAA;AAAA,IAAA,CACzB;AACD,UAAM,UAAU;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI;AACF,uBAAiB,IAAI,QAAQ,cAAc;AAC3C,YAAM,YAAY,MAAM,QAAQ;AAAA,QAC9B,OAAO;AAAA,UAAI,UACT,QAAQ;AAAA,YAAgB,CAAC,IAAIC,QAC3B,wBAAwBA,KAAI,IAAI,IAAI;AAAA,UAAA;AAAA,QACtC;AAAA,MACF;AAEF,sBAAgB;AAAA,QACd;AAAA,QACA;AAAA,QACA,WAAW,SAAS,+BAA+B,cAAc;AAAA,QACjE;AAAA,QACA,OAAO,EAAC,gBAAgB,UAAU,IAAI,CAAC,EAAC,OAAA,MAAY,MAAM,EAAA;AAAA,MAAC;AAG7D,WAAK,cAAc,MAAA;AACnB,YAAM,YAAY,MAAM,QAAQ;AAAA,QAC9B,UAAU;AAAA,UAAI,WACZ,QAAQ;AAAA,YAAgB,CAAC,IAAIA,QAC3B,KAAKA,KAAI,OAAO,UAAU,IAAI,EAAE;AAAA,UAAA;AAAA,QAClC;AAAA,MACF;AAEF,WAAK,cAAc,eAAe,IAAI,cAAc;AAEpD,YAAM,QAAQ,UAAU;AAAA,QACtB,CAAC,KAAK,UAAU;AAAA,UACd,MAAM,IAAI,OAAO,KAAK;AAAA,UACtB,WAAW,IAAI,YAAY,KAAK;AAAA,QAAA;AAAA,QAElC,EAAC,MAAM,GAAG,WAAW,EAAA;AAAA,MAAC;AAGxB,sBAAgB;AAAA,QACd;AAAA,QACA;AAAA,QACA,YAAY,QAAQ,MAAM;AAAA,QAC1B;AAAA,MAAA;AAEF,YAAM,aAAa,YAAY,IAAA;AAC/B,wBAAkB,IAAI,OAAO;AAC7B,YAAM,QAAQ,YAAY,IAAA,IAAQ;AAClC,SAAG,OAAO,oBAAoB,MAAM,QAAQ,CAAC,CAAC,MAAM;AAEpD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAGF,YAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,SAAG;AAAA,QACD,UAAU,MAAM,KAAK,eAAA,CAAgB,YAAY,SAAS,cAAc,YAAY,UAAU,GAAG,YACpF,MAAM,UAAU,QAAQ,CAAC,CAAC,YAAY,MAAM,QAAQ,CAAC,CAAC,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,MAAA;AAAA,IAErG,UAAA;AACE,cAAQ,QAAA;AACR,UAAI,SAAA,MAAe,SAAS;AAG1B,aAAK,SAAS,MAAM,MAAM,OAAK,GAAG,OAAO,0BAA0B,CAAC,CAAC;AAAA,MACvE,OAAO;AACL,cAAM,SAAS,IAAA;AAAA,MACjB;AAAA,IACF;AAAA,EACF,SAAS,GAAG;AAIV,OAAG,OAAO,6BAA6B,QAAQ,IAAI,CAAC;AACpD,UAAM;AAAA;AAAA,4BAEkB,QAAQ;AAAA,MAC9B,MAAM,CAAAC,OAAK,GAAG,OAAO,mCAAmC,QAAQ,IAAIA,EAAC,CAAC;AACxE,UAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;AAAA,EAClE,UAAA;AACE,oBAAgB,KAAA;AAChB,UAAM,mBAAmB,IAAA;AACzB,UAAM,IAAI,IAAA;AAAA,EACZ;AACF;AAEA,eAAe,oBAAoB,KAAiB;AAClD,QAAM,EAAC,UAAU,QAAA,KACf,MAAM;AAAA;AAAA;AAAA,KAIN,CAAC;AAEH,MAAI,aAAa,WAAW;AAC1B,UAAM,IAAI;AAAA,MACR,uEAAuE,QAAQ;AAAA,IAAA;AAAA,EAEnF;AACA,MAAI,UAAU,MAAQ;AACpB,UAAM,IAAI;AAAA,MACR,sDAAsD,OAAO;AAAA,IAAA;AAAA,EAEjE;AACF;AAEA,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;AACnC,QAAM,EAAC,UAAU,KAAA,IAAQ,IAAI;AAC7B,KAAG,OAAO,oCAAoC,QAAQ,IAAI,IAAI,EAAE;AAEhE,QAAM,kBAAkB,IAAI,KAAK,KAAK;AACtC,QAAM,EAAC,aAAA,IAAgB,MAAM,uBAAuB,KAAK,KAAK;AAE9D,MAAI,UAAU;AACZ,QAAI,QAAQ;AACZ,UAAM,0BAA0B,aAAa;AAAA,MAC3C,CAAA,MAAK,CAAC,EAAE,WAAW,GAAG;AAAA,IAAA;AAExB,UAAM,SAAS,MAAM;AAAA,4DACmC,IAAI,YAAY,CAAC;AAAA,QACrE,OAAA;AACJ,QAAI,OAAO,WAAW,aAAa,QAAQ;AACzC,SAAG;AAAA,QACD,iCAAiC,YAAY,mBACvC,OAAO,MAAM;AAAA,MAAA;AAAA,IAEvB,WACE,CAAC,OAAO,IAAI,IAAI,MAAM,YAAY,GAAG,IAAI,IAAI,uBAAuB,CAAC,GACrE;AACA,SAAG;AAAA,QACD,2BAA2B,MAAM,YAAY,uCAC1B,uBAAuB;AAAA,MAAA;AAAA,IAE9C,OAAO;AACL,cAAQ;AAAA,IACV;AACA,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;AACvD,aAAO,sBAAsB,IAAI,KAAK,OAAO,KAAK;AAAA,IACpD;AAAA,EACF;AACA,SAAO,EAAC,aAAA;AACV;AAEA,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;AACjB,QAAM,EAAC,KAAA,IAAQ,eAAe,QAAQ;AACtC,QAAM,eAAe,IAAI;AAAA,IACvB;AAAA,IACAF;AAAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEF,eAAa,IAAI,EAAE;AAEnB,KAAG,OAAO,WAAW,UAAU,oBAAoB,SAAS,SAAS;AAErE,MAAI,SAAS,QAAQ,SAAS,IAAI,IAAI,IAAI;AACxC,OAAG;AAAA,MACD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAOJ;AACA,SAAO;AACT;AAaA,eAAsB,sBACpB,IACA,SACA,UAC0B;AAC1B,QAAM,QACJ,MAAM,QAAQ;AAAA;AAAA,IACJ,4BAA4B,QAAQ;AAAA,EAAA,GAE9C,CAAC;AACH,KAAG,OAAO,4BAA4B,QAAQ,IAAI,IAAI;AACtD,SAAO;AACT;AAEA,SAAS,iBACP,IACA,QACA,gBACA;AAGA,QAAM,iBAAiB,KAAK,oBAAoB,YAAY,EAAE,CAAC;AAC/D,aAAW,KAAK,QAAQ;AACtB,OAAG,KAAK,yBAAyB,kBAAkB,GAAG,cAAc,CAAC,CAAC;AACtE,UAAM,YAAY,cAAc,CAAC;AACjC,eAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,EAAE,OAAO,GAAG;AAC1D,qBAAe,OAAO,WAAW,SAAS,OAAO;AAAA,IACnD;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,aAAW,SAAS,SAAS;AAC3B,OAAG,KAAK,yBAAyB,uBAAuB,KAAK,CAAC,CAAC;AAAA,EACjE;AACF;AAMO,MAAM,oBAAoB;AAEjC,MAAM,KAAK,OAAO;AAClB,MAAM,oBAAoB;AAC1B,MAAM,0BAA0B,IAAI;AAQ7B,SAAS,uBACd,OACA,MACoB;AACpB,QAAM,mBAAmB,OAAO,OAAO,MAAM,YAAY,EACtD,IAAI,CAAC,EAAC,UAAA,MAAe,SAAS,EAC9B,OAAO,CAAA,MAAK,CAAC,CAAC,CAAC;AAClB,QAAM,QACJ,iBAAiB,WAAW,IACxB;AAAA;AAAA,IACQ,SAAS,iBAAiB,KAAK,MAAM,CAAC;AAAA;AACpD,QAAM;AAAA;AAAA,IAAoB,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,KAAK;AAAA;AAC7E,QAAM,aAAa,IAAI,KAAK,IAAI,CAAA,QAAO,+BAA+B,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,KAAK,CAAC;AAClG,QAAM,QAAQ;AAAA,IACZ;AAAA;AAAA,MAAgB,UAAU,KAAK,IAAI,EAAE,EAAE,KAAK,GAAG,CAAC,IAAI,SAAS;AAAA;AAAA,IAC7D;AAAA;AAAA,MAAsB,kCAAkC,SAAS;AAAA;AAAA,IACjE;AAAA;AAAA,MAAuB,UAAU,UAAU,oBAAoB,SAAS;AAAA;AAAA,EAAA;AAE1E,SAAO;AACT;AAOA,eAAe,wBACb,IACA,KACA,MACwB;AACxB,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,QAAQ,cAAc,IAAI;AAChC,QAAM,UAAU,OAAO,KAAK,KAAK,OAAO;AACxC,QAAM,QAAQ,uBAAuB,MAAM,OAAO;AAClD,QAAM,aAAa,IAChB,OAA8B,MAAM,YAAY,EAChD,QAAA;AACH,QAAM,cAAc,IACjB,OAA+B,MAAM,aAAa,EAClD,QAAA;AAEH,QAAM,QAAuB;AAAA,IAC3B;AAAA,IACA,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA,MAAM;AAAA,MACN,WAAW,QAAQ,MAAM,YAAY,CAAC,EAAE,SAAS;AAAA,MACjD,YAAY,QAAQ,MAAM,aAAa,CAAC,EAAE,UAAU;AAAA,IAAA;AAAA,EACtD;AAEF,QAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,KAAG,OAAO,uCAAuC,KAAK,KAAK,OAAO,QAAQ;AAAA,IACxE,OAAO,MAAM;AAAA,EAAA,CACd;AACD,SAAO;AACT;AAEA,eAAe,KACb,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA;AACA,QAAM,QAAQ,YAAY,IAAA;AAC1B,MAAI,YAAY;AAEhB,QAAM,YAAY,cAAc,KAAK;AACrC,QAAM,iBAAiB,OAAO,QAAQ,MAAM,OAAO;AAEnD,QAAM,cAAc,eAAe,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACjD,QAAM,cAAc,eAAe,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,IAAI;AAC9D,QAAM,mBAAmB,YAAY,IAAI,CAAA,MAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG;AAG7D,QAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,CAAC,CAAC,OAAO;AACzE,QAAM;AAAA;AAAA,IAAoB;AAAA,mBACT,SAAS,MAAM,gBAAgB,YAAY,SAAS;AAAA;AACrE,QAAM,aAAa,GAAG,QAAQ,SAAS;AAEvC,QAAM,kBAAkB,GAAG;AAAA,IACzB,YAAY,IAAI,SAAS,GAAG,OAAO,oBAAoB,CAAC;AAAA,EAAA;AAG1D,QAAM,EAAC,OAAA,IAAU,uBAAuB,OAAO,WAAW;AAC1D,QAAM,eAAe,YAAY;AACjC,QAAM,iBAAiB,eAAe;AAGtC,QAAM,gBAAiC,MAAM,KAAK;AAAA,IAChD,QAAQ,oBAAoB;AAAA,EAAA,CAC7B;AACD,MAAI,cAAc;AAClB,MAAI,cAAc;AAElB,WAAS,QAAQ;AACf,UAAMG,SAAQ,YAAY,IAAA;AAC1B,UAAM,cAAc;AACpB,UAAM,cAAc;AAEpB,QAAI,IAAI;AACR,WAAO,cAAc,mBAAmB,eAAe,mBAAmB;AACxE,sBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,cAAe,CAAC;AAAA,IACnE;AAEA,WAAO,cAAc,GAAG,eAAe;AACrC,iBAAW,IAAI,cAAc,MAAM,GAAI,KAAK,YAAa,CAAC;AAAA,IAC5D;AACA,aAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AAGpC,oBAAc,CAAC,IAAI;AAAA,IACrB;AACA,kBAAc;AACd,WAAO,QAAQ;AAEf,UAAMC,WAAU,YAAY,IAAA,IAAQD;AACpC,iBAAaC;AACb,OAAG;AAAA,MACD,WAAW,WAAW,IAAI,SAAS,UAAU,WAAW,cAAcA,SAAQ,QAAQ,CAAC,CAAC;AAAA,IAAA;AAAA,EAE5F;AAEA,KAAG,OAAO,2BAA2B,SAAS,KAAK,MAAM;AACzD,QAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK;AAC3E,QAAM,UAAU,YAAY,IAAI,CAAA,MAAK;AACnC,UAAM,UAAU,UAAU,cAAc,EAAE,OAAO;AACjD,WAAO,CAAC,QACN;AAAA,MACE,QAAQ,GAAG;AAAA,MACX,EAAE;AAAA,MACF;AAAA,IAAA;AAAA,EAEN,CAAC;AAED,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,MAAM;AAEV,QAAM;AAAA,IACJ,MAAM,KAAK,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,IAChD,IAAI,SAAS;AAAA,MACX,eAAe;AAAA,MAEf,MACE,OACA,WACA,UACA;AACA,YAAI;AACF,qBAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,2BAAe,SAAS,OAAO,IAAI,KAAK;AACxC,0BAAc,cAAc,eAAe,GAAG,IAC5C,SAAS,OAAO,OAAO,QAAQ,GAAG,EAAE,IAAI;AAE1C,gBAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,oBAAM;AACN,kBACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,yBACf;AACA,sBAAA;AAAA,cACF;AAAA,YACF;AAAA,UACF;AACA,mBAAA;AAAA,QACF,SAAS,GAAG;AACV,mBAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAAA,MAEA,OAAO,CAAC,aAAsC;AAC5C,YAAI;AACF,gBAAA;AACA,mBAAA;AAAA,QACF,SAAS,GAAG;AACV,mBAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAAA,IAAA,CACD;AAAA,EAAA;AAGH,QAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,KAAG;AAAA,IACD,oBAAoB,OAAO,IAAI,cAAc,SAAS,YACzC,UAAU,QAAQ,CAAC,CAAC,gBAAgB,QAAQ,QAAQ,CAAC,CAAC;AAAA,EAAA;AAErE,SAAO,EAAC,MAAM,OAAO,MAAM,UAAA;AAC7B;"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AAEzD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAyBhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AAKzD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAyBhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,11 +1,13 @@
 import { resolver } from "@rocicorp/resolver";
 import { getDefaultHighWaterMark } from "node:stream";
 import { unreachable } from "../../../../shared/src/asserts.js";
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { publishCriticalEvent } from "../../observability/events.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { min } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
 import "../change-source/protocol/current/downstream.js";
-import { publishReplicationError } from "../replicator/replication-status.js";
+import { replicationStatusError, publishReplicationError } from "../replicator/replication-status.js";
 import { RunningState, UnrecoverableError, DEFAULT_MAX_RETRY_DELAY_MS } from "../running-state.js";
 import "./change-streamer.js";
 import { WrongReplicaVersion } from "./error-type-enum.js";
@@ -167,7 +169,11 @@ class ChangeStreamerImpl {
       }
       await Promise.all([
         this.#storer.stop(),
-        this.#state.backoff(this.#lc, err)
+        this.#state.backoff(this.#lc, err),
+        this.#state.retryDelay > 5e3 ? publishCriticalEvent(
+          this.#lc,
+          replicationStatusError(this.#lc, "Replicating", err)
+        ) : promiseVoid
       ]);
     }
     this.#lc.info?.("ChangeStreamer stopped");

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    backPressureLimitHeapProportion,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          unflushedBytes += this.#storer.store([watermark, change]);\n          const sent = this.#forwarder.forward([watermark, change]);\n          if (unflushedBytes >= flushBytesThreshold) {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await sent;\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        await this.#forwarder.forward([\n          watermark,\n          ['rollback', {tag: 'rollback'}],\n        ]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n      ]);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;AAgDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAInB,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,QAAQ,MAAM,MAAM,OAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAEtD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,4BAAkB,KAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACxD,gBAAM,OAAO,KAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AACxD,cAAI,kBAAkB,qBAAqB;AAQzC,kBAAM;AACN,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAGA,YAAM,QAAQ,IAAI;AAAA,QAChB,KAAK,QAAQ,KAAA;AAAA,QACb,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,MAAA,CAClC;AAAA,IACH;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}
+{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    backPressureLimitHeapProportion,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          unflushedBytes += this.#storer.store([watermark, change]);\n          const sent = this.#forwarder.forward([watermark, change]);\n          if (unflushedBytes >= flushBytesThreshold) {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await sent;\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        await this.#forwarder.forward([\n          watermark,\n          ['rollback', {tag: 'rollback'}],\n        ]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > 5000\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;;;AAqDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAInB,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,QAAQ,MAAM,MAAM,OAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAEtD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,4BAAkB,KAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACxD,gBAAM,OAAO,KAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AACxD,cAAI,kBAAkB,qBAAqB;AAQzC,kBAAM;AACN,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAGA,YAAM,QAAQ,IAAI;AAAA,QAChB,KAAK,QAAQ,KAAA;AAAA,QACb,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,QACjC,KAAK,OAAO,aAAa,MACrB;AAAA,UACE,KAAK;AAAA,UACL,uBAAuB,KAAK,KAAK,eAAe,GAAG;AAAA,QAAA,IAErD;AAAA,MAAA,CACL;AAAA,IACH;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/replicator/replication-status.js

@@ -37,7 +37,7 @@ class ReplicationStatusPublisher {
   }
   async publishAndThrowError(lc, stage, e) {
     this.stop();
-    const event = replicationStatusError(lc, stage, "ERROR", this.#db);
+    const event = replicationStatusError(lc, stage, e, this.#db);
     await this.#publish(lc, event);
     throw e;
   }

package/out/zero-cache/src/services/replicator/replication-status.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.js","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #db: Database;\n  readonly #publish: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(db: Database, publishFn = publishCriticalEvent) {\n    this.#db = db;\n    this.#publish = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    this.stop();\n    const event = replicationStatusEvent(\n      lc,\n      this.#db,\n      stage,\n      'OK',\n      description,\n      now,\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    void this.#publish(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return this;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = replicationStatusError(lc, stage, 'ERROR', this.#db);\n    await this.#publish(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n    table,\n    unique,\n    columns: Object.entries(columns)\n      .sort(byKeys)\n      .map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"names":["spec"],"mappings":";;;AAmBA,MAAM,SAAS,CAAC,GAAsB,MACpC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI;AAEhC,MAAM,2BAA2B;AAAA,EAC7B;AAAA,EACA;AAAA,EACT;AAAA,EAEA,YAAY,IAAc,YAAY,sBAAsB;AAC1D,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,MAAM,oBAAI,QACJ;AACN,SAAK,KAAA;AACL,UAAM,QAAQ;AAAA,MACZ;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,MAAM,OAAO;AACf,YAAM,QAAQ;AAAA,QACZ,GAAG,MAAM;AAAA,QACT,GAAG,aAAA;AAAA,MAAa;AAAA,IAEpB;AACA,SAAK,KAAK,SAAS,IAAI,KAAK;AAE5B,QAAI,UAAU;AACZ,WAAK,SAAS;AAAA,QACZ,MAAM,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU;AAAA,QAC/D;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBACJ,IACA,OACA,GACgB;AAChB,SAAK,KAAA;AACL,UAAM,QAAQ,uBAAuB,IAAI,OAAO,SAAS,KAAK,GAAG;AACjE,UAAM,KAAK,SAAS,IAAI,KAAK;AAC7B,UAAM;AAAA,EACR;AAAA,EAEA,OAAa;AACX,kBAAc,KAAK,MAAM;AACzB,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,MAAM,oBAAI,QACV;AACA,QAAM,QAAgC;AAAA,IACpC,MAAM;AAAA,IACN,WAAW;AAAA,IACX,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM,IAAI,YAAA;AAAA,EAAY;AAExB,QAAM,qBAAqB,IAAI,KAAK;AACtC;AAEO,SAAS,uBACd,IACA,OACA,GACA,IACA,MAAM,oBAAI,QACV;AACA,QAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;AAC3E,QAAM,eAAe,iBAAiB,CAAC;AACvC,SAAO;AACT;AAGO,SAAS,uBACd,IACA,IACA,OACA,QACA,aACA,MAAM,oBAAI,QACc;AACxB,QAAM,QAAQ,YAAY,IAAA;AAC1B,MAAI;AACF,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,IAAI,YAAA;AAAA,MACV,OAAO;AAAA,QACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAA;AAAA,QACvC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAA;AAAA,QACzC,aAAa,KAAK,eAAe,EAAE,IAAI;AAAA,MAAA;AAAA,IACzC;AAAA,EAEJ,SAAS,GAAG;AACV,OAAG,OAAO,gDAAgD,CAAC;AAC3D,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,IAAI,YAAA;AAAA,MACV,OAAO;AAAA,QACL,QAAQ,CAAA;AAAA,QACR,SAAS,CAAA;AAAA,QACT,aAAa;AAAA,MAAA;AAAA,IACf;AAAA,EAEJ,UAAA;AACE,UAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,OAAG,QAAQ,0CAA0C,OAAO,MAAM;AAAA,EACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;AAC5D,QAAM,iCAAiB,IAAA;AACvB,QAAM,eAAe;AAAA,IACnB,uBAAA;AAAA;AAAA,IACA;AAAA,IACA,EAAC,2BAA2B,MAAA;AAAA,wBACxB,IAAA;AAAA,IACJ;AAAA,EAAA;AAGF,SAAO,CAAC,GAAG,WAAW,QAAA,CAAS,EAAE,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO;AAAA,IACpE;AAAA,IACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,IAAI,CAAC,CAAC,QAAQA,KAAI,OAAO;AAAA,MACxB;AAAA,MACA,cAAcA,MAAK,SAAS,MAAM,GAAG,EAAE,CAAC;AAAA,MACxC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,MAAM,GAAG,QAAQ;AAAA,IAAA,EAC9D;AAAA,EAAA,EACJ;AACJ;AAEA,SAAS,qBAAqB,IAAiC;AAC7D,SAAO,YAAY,EAAE,EAAE,IAAI,CAAC,EAAC,WAAW,OAAO,SAAS,cAAa;AAAA,IACnE;AAAA,IACA;AAAA,IACA,SAAS,OAAO,QAAQ,OAAO,EAC5B,KAAK,MAAM,EACX,IAAI,CAAC,CAAC,QAAQ,GAAG,OAAO,EAAC,QAAQ,MAAK;AAAA,EAAA,EACzC;AACJ;AAEA,SAAS,eAAe,IAAc;AACpC,QAAM,CAAC,EAAC,YAAY,UAAA,CAAU,IAAI,GAAG;AAAA,IACnC;AAAA,EAAA;AAEF,QAAM,CAAC,EAAC,WAAW,SAAA,CAAS,IAAI,GAAG,OAA4B,WAAW;AAC1E,SAAO,YAAY;AACrB;"}
+{"version":3,"file":"replication-status.js","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #db: Database;\n  readonly #publish: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(db: Database, publishFn = publishCriticalEvent) {\n    this.#db = db;\n    this.#publish = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    this.stop();\n    const event = replicationStatusEvent(\n      lc,\n      this.#db,\n      stage,\n      'OK',\n      description,\n      now,\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    void this.#publish(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return this;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = replicationStatusError(lc, stage, e, this.#db);\n    await this.#publish(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n    table,\n    unique,\n    columns: Object.entries(columns)\n      .sort(byKeys)\n      .map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"names":["spec"],"mappings":";;;AAmBA,MAAM,SAAS,CAAC,GAAsB,MACpC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI;AAEhC,MAAM,2BAA2B;AAAA,EAC7B;AAAA,EACA;AAAA,EACT;AAAA,EAEA,YAAY,IAAc,YAAY,sBAAsB;AAC1D,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,MAAM,oBAAI,QACJ;AACN,SAAK,KAAA;AACL,UAAM,QAAQ;AAAA,MACZ;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,MAAM,OAAO;AACf,YAAM,QAAQ;AAAA,QACZ,GAAG,MAAM;AAAA,QACT,GAAG,aAAA;AAAA,MAAa;AAAA,IAEpB;AACA,SAAK,KAAK,SAAS,IAAI,KAAK;AAE5B,QAAI,UAAU;AACZ,WAAK,SAAS;AAAA,QACZ,MAAM,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU;AAAA,QAC/D;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBACJ,IACA,OACA,GACgB;AAChB,SAAK,KAAA;AACL,UAAM,QAAQ,uBAAuB,IAAI,OAAO,GAAG,KAAK,GAAG;AAC3D,UAAM,KAAK,SAAS,IAAI,KAAK;AAC7B,UAAM;AAAA,EACR;AAAA,EAEA,OAAa;AACX,kBAAc,KAAK,MAAM;AACzB,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,MAAM,oBAAI,QACV;AACA,QAAM,QAAgC;AAAA,IACpC,MAAM;AAAA,IACN,WAAW;AAAA,IACX,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM,IAAI,YAAA;AAAA,EAAY;AAExB,QAAM,qBAAqB,IAAI,KAAK;AACtC;AAEO,SAAS,uBACd,IACA,OACA,GACA,IACA,MAAM,oBAAI,QACV;AACA,QAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;AAC3E,QAAM,eAAe,iBAAiB,CAAC;AACvC,SAAO;AACT;AAGO,SAAS,uBACd,IACA,IACA,OACA,QACA,aACA,MAAM,oBAAI,QACc;AACxB,QAAM,QAAQ,YAAY,IAAA;AAC1B,MAAI;AACF,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,IAAI,YAAA;AAAA,MACV,OAAO;AAAA,QACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAA;AAAA,QACvC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAA;AAAA,QACzC,aAAa,KAAK,eAAe,EAAE,IAAI;AAAA,MAAA;AAAA,IACzC;AAAA,EAEJ,SAAS,GAAG;AACV,OAAG,OAAO,gDAAgD,CAAC;AAC3D,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,IAAI,YAAA;AAAA,MACV,OAAO;AAAA,QACL,QAAQ,CAAA;AAAA,QACR,SAAS,CAAA;AAAA,QACT,aAAa;AAAA,MAAA;AAAA,IACf;AAAA,EAEJ,UAAA;AACE,UAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,OAAG,QAAQ,0CAA0C,OAAO,MAAM;AAAA,EACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;AAC5D,QAAM,iCAAiB,IAAA;AACvB,QAAM,eAAe;AAAA,IACnB,uBAAA;AAAA;AAAA,IACA;AAAA,IACA,EAAC,2BAA2B,MAAA;AAAA,wBACxB,IAAA;AAAA,IACJ;AAAA,EAAA;AAGF,SAAO,CAAC,GAAG,WAAW,QAAA,CAAS,EAAE,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO;AAAA,IACpE;AAAA,IACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,IAAI,CAAC,CAAC,QAAQA,KAAI,OAAO;AAAA,MACxB;AAAA,MACA,cAAcA,MAAK,SAAS,MAAM,GAAG,EAAE,CAAC;AAAA,MACxC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,MAAM,GAAG,QAAQ;AAAA,IAAA,EAC9D;AAAA,EAAA,EACJ;AACJ;AAEA,SAAS,qBAAqB,IAAiC;AAC7D,SAAO,YAAY,EAAE,EAAE,IAAI,CAAC,EAAC,WAAW,OAAO,SAAS,cAAa;AAAA,IACnE;AAAA,IACA;AAAA,IACA,SAAS,OAAO,QAAQ,OAAO,EAC5B,KAAK,MAAM,EACX,IAAI,CAAC,CAAC,QAAQ,GAAG,OAAO,EAAC,QAAQ,MAAK;AAAA,EAAA,EACzC;AACJ;AAEA,SAAS,eAAe,IAAc;AACpC,QAAM,CAAC,EAAC,YAAY,UAAA,CAAU,IAAI,GAAG;AAAA,IACnC;AAAA,EAAA;AAEF,QAAM,CAAC,EAAC,WAAW,SAAA,CAAS,IAAI,GAAG,OAA4B,WAAW;AAC1E,SAAO,YAAY;AACrB;"}

package/out/zero-cache/src/services/running-state.d.ts

@@ -16,6 +16,7 @@ export declare class RunningState {
     #private;
     constructor(serviceName: string, retryConfig?: RetryConfig, setTimeoutFn?: typeof setTimeout, sleeper?: typeof sleepWithAbort);
     get signal(): AbortSignal;
+    get retryDelay(): number;
     /**
      * Returns `true` until {@link stop()} has been called.
      *

package/out/zero-cache/src/services/running-state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAG5D,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B,IAAI,MAAM,IAAI,WAAW,CAExB;IAED;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1D;;;;OAIG;IACH,YAAY;CAGb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}
+{"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAG5D,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,UAAU,WAEb;IAED;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB1D;;;;OAIG;IACH,YAAY;CAGb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}

package/out/zero-cache/src/services/running-state.js

@@ -42,6 +42,9 @@ class RunningState {
   get signal() {
     return this.#controller.signal;
   }
+  get retryDelay() {
+    return this.#retryDelay;
+  }
   /**
    * Returns `true` until {@link stop()} has been called.
    *
@@ -106,6 +109,7 @@ class RunningState {
     const delay = this.#retryDelay;
     this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);
     if (err instanceof AbortError || err instanceof UnrecoverableError) {
+      this.resetBackoff();
       this.stop(lc, err);
     } else if (this.shouldRun()) {
       const log = delay < 1e3 ? "info" : delay < 6500 ? "warn" : "error";

package/out/zero-cache/src/services/running-state.js.map

@@ -1 +1 @@
-{"version":3,"file":"running-state.js","sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n  initialRetryDelay?: number;\n  maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n  cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/**\n * Facilitates lifecycle control with exponential backoff.\n */\nexport class RunningState {\n  readonly #serviceName: string;\n  readonly #controller: AbortController;\n  readonly #sleep: typeof sleepWithAbort;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #stopped: Promise<void>;\n\n  readonly #initialRetryDelay: number;\n  readonly #maxRetryDelay: number;\n  readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n  #retryDelay: number;\n\n  constructor(\n    serviceName: string,\n    retryConfig?: RetryConfig,\n    setTimeoutFn = setTimeout,\n    sleeper = sleepWithAbort,\n  ) {\n    const {\n      initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n      maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n    } = retryConfig ?? {};\n\n    this.#serviceName = serviceName;\n    this.#initialRetryDelay = initialRetryDelay;\n    this.#maxRetryDelay = maxRetryDelay;\n    this.#retryDelay = initialRetryDelay;\n\n    this.#controller = new AbortController();\n    this.#sleep = sleeper;\n    this.#setTimeout = setTimeoutFn;\n\n    const {promise, resolve} = resolver();\n    this.#stopped = promise;\n    this.#controller.signal.addEventListener(\n      'abort',\n      () => {\n        resolve();\n        for (const timeout of this.#pendingTimeouts) {\n          clearTimeout(timeout);\n        }\n        this.#pendingTimeouts.clear();\n      },\n      {once: true},\n    );\n  }\n\n  get signal(): AbortSignal {\n    return this.#controller.signal;\n  }\n\n  /**\n   * Returns `true` until {@link stop()} has been called.\n   *\n   * This is usually called as part of the service's main loop\n   * conditional to determine if the next iteration should execute.\n   */\n  shouldRun(): boolean {\n    return !this.#controller.signal.aborted;\n  }\n\n  /**\n   * Registers a Cancelable object to be invoked when {@link stop()} is called.\n   * Returns a method to unregister the object.\n   */\n  cancelOnStop(c: Cancelable): UnregisterFn {\n    const onStop = () => c.cancel();\n    this.#controller.signal.addEventListener('abort', onStop, {once: true});\n    return () => this.#controller.signal.removeEventListener('abort', onStop);\n  }\n\n  /**\n   * Sets a Timeout that is automatically cancelled if the service is cancelled.\n   */\n  setTimeout<TArgs extends unknown[]>(\n    fn: (...args: TArgs) => void,\n    timeoutMs: number,\n    ...args: TArgs\n  ) {\n    const timeout = this.#setTimeout(() => {\n      clearTimeout(timeout);\n      this.#pendingTimeouts.delete(timeout);\n      return fn(...args);\n    }, timeoutMs);\n\n    this.#pendingTimeouts.add(timeout);\n  }\n\n  /**\n   * Returns a promise that resolves after `ms` milliseconds or when\n   * the service is stopped.\n   */\n  async sleep(ms: number): Promise<void> {\n    await Promise.race(this.#sleep(ms, this.#controller.signal));\n  }\n\n  /**\n   * Called to stop the service. After this is called, {@link shouldRun()}\n   * will return `false` and the {@link stopped()} Promise will be resolved.\n   */\n  stop(lc: LogContext, err?: unknown): void {\n    if (this.shouldRun()) {\n      const log = !err || err instanceof AbortError ? 'info' : 'error';\n      lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n      this.#controller.abort();\n    }\n  }\n\n  /**\n   * Returns a Promise that resolves when {@link stop()} is called.\n   */\n  stopped(): Promise<void> {\n    return this.#stopped;\n  }\n\n  /**\n   * Call in response to an error or unexpected termination in the main\n   * loop of the service. The returned Promise will resolve after an\n   * exponential delay, or once {@link stop()} is called.\n   *\n   * If the supplied `err` is an `AbortError`, the service will shut down.\n   */\n  async backoff(lc: LogContext, err: unknown): Promise<void> {\n    const delay = this.#retryDelay;\n    this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);\n\n    if (err instanceof AbortError || err instanceof UnrecoverableError) {\n      this.stop(lc, err);\n    } else if (this.shouldRun()) {\n      // Use delay-based log level: higher delay means more retries\n      const log: 'info' | 'warn' | 'error' =\n        delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n      lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n      await this.sleep(delay);\n    }\n  }\n\n  /**\n   * When using {@link backoff()}, this method should be called when the\n   * implementation receives a healthy signal (e.g. a successful\n   * response). This resets the delay used in {@link backoff()}.\n   */\n  resetBackoff() {\n    this.#retryDelay = this.#initialRetryDelay;\n  }\n}\n\n/**\n * Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n  readonly name = 'UnrecoverableError';\n}\n"],"names":[],"mappings":";;;AAKA,MAAM,iCAAiC;AAChC,MAAM,6BAA6B;AAgBnC,MAAM,aAAa;AAAA,EACf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,uCAAuB,IAAA;AAAA,EAChC;AAAA,EAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;AACA,UAAM;AAAA,MACJ,oBAAoB;AAAA,MACpB,gBAAgB;AAAA,IAAA,IACd,eAAe,CAAA;AAEnB,SAAK,eAAe;AACpB,SAAK,qBAAqB;AAC1B,SAAK,iBAAiB;AACtB,SAAK,cAAc;AAEnB,SAAK,cAAc,IAAI,gBAAA;AACvB,SAAK,SAAS;AACd,SAAK,cAAc;AAEnB,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,WAAW;AAChB,SAAK,YAAY,OAAO;AAAA,MACtB;AAAA,MACA,MAAM;AACJ,gBAAA;AACA,mBAAW,WAAW,KAAK,kBAAkB;AAC3C,uBAAa,OAAO;AAAA,QACtB;AACA,aAAK,iBAAiB,MAAA;AAAA,MACxB;AAAA,MACA,EAAC,MAAM,KAAA;AAAA,IAAI;AAAA,EAEf;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAqB;AACnB,WAAO,CAAC,KAAK,YAAY,OAAO;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,GAA6B;AACxC,UAAM,SAAS,MAAM,EAAE,OAAA;AACvB,SAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK;AACtE,WAAO,MAAM,KAAK,YAAY,OAAO,oBAAoB,SAAS,MAAM;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,WACE,IACA,cACG,MACH;AACA,UAAM,UAAU,KAAK,YAAY,MAAM;AACrC,mBAAa,OAAO;AACpB,WAAK,iBAAiB,OAAO,OAAO;AACpC,aAAO,GAAG,GAAG,IAAI;AAAA,IACnB,GAAG,SAAS;AAEZ,SAAK,iBAAiB,IAAI,OAAO;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,MAAM,IAA2B;AACrC,UAAM,QAAQ,KAAK,KAAK,OAAO,IAAI,KAAK,YAAY,MAAM,CAAC;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,KAAK,IAAgB,KAAqB;AACxC,QAAI,KAAK,aAAa;AACpB,YAAM,MAAM,CAAC,OAAO,eAAe,aAAa,SAAS;AACzD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,IAAI,OAAO,EAAE;AACpD,WAAK,YAAY,MAAA;AAAA,IACnB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAyB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,QAAQ,IAAgB,KAA6B;AACzD,UAAM,QAAQ,KAAK;AACnB,SAAK,cAAc,KAAK,IAAI,QAAQ,GAAG,KAAK,cAAc;AAE1D,QAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,WAAK,KAAK,IAAI,GAAG;AAAA,IACnB,WAAW,KAAK,aAAa;AAE3B,YAAM,MACJ,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS;AAElD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,OAAO,KAAK,OAAO,GAAG;AAC7D,YAAM,KAAK,MAAM,KAAK;AAAA,IACxB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,eAAe;AACb,SAAK,cAAc,KAAK;AAAA,EAC1B;AACF;AAMO,MAAM,2BAA2B,MAAM;AAAA,EACnC,OAAO;AAClB;"}
+{"version":3,"file":"running-state.js","sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n  initialRetryDelay?: number;\n  maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n  cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/**\n * Facilitates lifecycle control with exponential backoff.\n */\nexport class RunningState {\n  readonly #serviceName: string;\n  readonly #controller: AbortController;\n  readonly #sleep: typeof sleepWithAbort;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #stopped: Promise<void>;\n\n  readonly #initialRetryDelay: number;\n  readonly #maxRetryDelay: number;\n  readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n  #retryDelay: number;\n\n  constructor(\n    serviceName: string,\n    retryConfig?: RetryConfig,\n    setTimeoutFn = setTimeout,\n    sleeper = sleepWithAbort,\n  ) {\n    const {\n      initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n      maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n    } = retryConfig ?? {};\n\n    this.#serviceName = serviceName;\n    this.#initialRetryDelay = initialRetryDelay;\n    this.#maxRetryDelay = maxRetryDelay;\n    this.#retryDelay = initialRetryDelay;\n\n    this.#controller = new AbortController();\n    this.#sleep = sleeper;\n    this.#setTimeout = setTimeoutFn;\n\n    const {promise, resolve} = resolver();\n    this.#stopped = promise;\n    this.#controller.signal.addEventListener(\n      'abort',\n      () => {\n        resolve();\n        for (const timeout of this.#pendingTimeouts) {\n          clearTimeout(timeout);\n        }\n        this.#pendingTimeouts.clear();\n      },\n      {once: true},\n    );\n  }\n\n  get signal(): AbortSignal {\n    return this.#controller.signal;\n  }\n\n  get retryDelay() {\n    return this.#retryDelay;\n  }\n\n  /**\n   * Returns `true` until {@link stop()} has been called.\n   *\n   * This is usually called as part of the service's main loop\n   * conditional to determine if the next iteration should execute.\n   */\n  shouldRun(): boolean {\n    return !this.#controller.signal.aborted;\n  }\n\n  /**\n   * Registers a Cancelable object to be invoked when {@link stop()} is called.\n   * Returns a method to unregister the object.\n   */\n  cancelOnStop(c: Cancelable): UnregisterFn {\n    const onStop = () => c.cancel();\n    this.#controller.signal.addEventListener('abort', onStop, {once: true});\n    return () => this.#controller.signal.removeEventListener('abort', onStop);\n  }\n\n  /**\n   * Sets a Timeout that is automatically cancelled if the service is cancelled.\n   */\n  setTimeout<TArgs extends unknown[]>(\n    fn: (...args: TArgs) => void,\n    timeoutMs: number,\n    ...args: TArgs\n  ) {\n    const timeout = this.#setTimeout(() => {\n      clearTimeout(timeout);\n      this.#pendingTimeouts.delete(timeout);\n      return fn(...args);\n    }, timeoutMs);\n\n    this.#pendingTimeouts.add(timeout);\n  }\n\n  /**\n   * Returns a promise that resolves after `ms` milliseconds or when\n   * the service is stopped.\n   */\n  async sleep(ms: number): Promise<void> {\n    await Promise.race(this.#sleep(ms, this.#controller.signal));\n  }\n\n  /**\n   * Called to stop the service. After this is called, {@link shouldRun()}\n   * will return `false` and the {@link stopped()} Promise will be resolved.\n   */\n  stop(lc: LogContext, err?: unknown): void {\n    if (this.shouldRun()) {\n      const log = !err || err instanceof AbortError ? 'info' : 'error';\n      lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n      this.#controller.abort();\n    }\n  }\n\n  /**\n   * Returns a Promise that resolves when {@link stop()} is called.\n   */\n  stopped(): Promise<void> {\n    return this.#stopped;\n  }\n\n  /**\n   * Call in response to an error or unexpected termination in the main\n   * loop of the service. The returned Promise will resolve after an\n   * exponential delay, or once {@link stop()} is called.\n   *\n   * If the supplied `err` is an `AbortError`, the service will shut down.\n   */\n  async backoff(lc: LogContext, err: unknown): Promise<void> {\n    const delay = this.#retryDelay;\n    this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);\n\n    if (err instanceof AbortError || err instanceof UnrecoverableError) {\n      this.resetBackoff();\n      this.stop(lc, err);\n    } else if (this.shouldRun()) {\n      // Use delay-based log level: higher delay means more retries\n      const log: 'info' | 'warn' | 'error' =\n        delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n      lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n      await this.sleep(delay);\n    }\n  }\n\n  /**\n   * When using {@link backoff()}, this method should be called when the\n   * implementation receives a healthy signal (e.g. a successful\n   * response). This resets the delay used in {@link backoff()}.\n   */\n  resetBackoff() {\n    this.#retryDelay = this.#initialRetryDelay;\n  }\n}\n\n/**\n * Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n  readonly name = 'UnrecoverableError';\n}\n"],"names":[],"mappings":";;;AAKA,MAAM,iCAAiC;AAChC,MAAM,6BAA6B;AAgBnC,MAAM,aAAa;AAAA,EACf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,uCAAuB,IAAA;AAAA,EAChC;AAAA,EAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;AACA,UAAM;AAAA,MACJ,oBAAoB;AAAA,MACpB,gBAAgB;AAAA,IAAA,IACd,eAAe,CAAA;AAEnB,SAAK,eAAe;AACpB,SAAK,qBAAqB;AAC1B,SAAK,iBAAiB;AACtB,SAAK,cAAc;AAEnB,SAAK,cAAc,IAAI,gBAAA;AACvB,SAAK,SAAS;AACd,SAAK,cAAc;AAEnB,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,WAAW;AAChB,SAAK,YAAY,OAAO;AAAA,MACtB;AAAA,MACA,MAAM;AACJ,gBAAA;AACA,mBAAW,WAAW,KAAK,kBAAkB;AAC3C,uBAAa,OAAO;AAAA,QACtB;AACA,aAAK,iBAAiB,MAAA;AAAA,MACxB;AAAA,MACA,EAAC,MAAM,KAAA;AAAA,IAAI;AAAA,EAEf;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAqB;AACnB,WAAO,CAAC,KAAK,YAAY,OAAO;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,GAA6B;AACxC,UAAM,SAAS,MAAM,EAAE,OAAA;AACvB,SAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK;AACtE,WAAO,MAAM,KAAK,YAAY,OAAO,oBAAoB,SAAS,MAAM;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,WACE,IACA,cACG,MACH;AACA,UAAM,UAAU,KAAK,YAAY,MAAM;AACrC,mBAAa,OAAO;AACpB,WAAK,iBAAiB,OAAO,OAAO;AACpC,aAAO,GAAG,GAAG,IAAI;AAAA,IACnB,GAAG,SAAS;AAEZ,SAAK,iBAAiB,IAAI,OAAO;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,MAAM,IAA2B;AACrC,UAAM,QAAQ,KAAK,KAAK,OAAO,IAAI,KAAK,YAAY,MAAM,CAAC;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,KAAK,IAAgB,KAAqB;AACxC,QAAI,KAAK,aAAa;AACpB,YAAM,MAAM,CAAC,OAAO,eAAe,aAAa,SAAS;AACzD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,IAAI,OAAO,EAAE;AACpD,WAAK,YAAY,MAAA;AAAA,IACnB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAyB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,QAAQ,IAAgB,KAA6B;AACzD,UAAM,QAAQ,KAAK;AACnB,SAAK,cAAc,KAAK,IAAI,QAAQ,GAAG,KAAK,cAAc;AAE1D,QAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,WAAK,aAAA;AACL,WAAK,KAAK,IAAI,GAAG;AAAA,IACnB,WAAW,KAAK,aAAa;AAE3B,YAAM,MACJ,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS;AAElD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,OAAO,KAAK,OAAO,GAAG;AAC7D,YAAM,KAAK,MAAM,KAAK;AAAA,IACxB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,eAAe;AACb,SAAK,cAAc,KAAK;AAAA,EAC1B;AACF;AAMO,MAAM,2BAA2B,MAAM;AAAA,EACnC,OAAO;AAClB;"}

package/out/zero-client/src/client/version.js

@@ -1,4 +1,4 @@
-const version = "0.26.1-canary.4";
+const version = "0.26.1-canary.6";
 export {
   version
 };

package/out/zero-server/src/process-mutations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"process-mutations.d.ts","sourceRoot":"","sources":["../../../../zero-server/src/process-mutations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAG3D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAC5D,OAAO,KAAK,CAAC,MAAM,4BAA4B,CAAC;AAWhD,OAAO,EAIL,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EAGnB,KAAK,gBAAgB,EAErB,KAAK,YAAY,EAClB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAEjF,OAAO,KAAK,EAAC,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,aAAa,CAAC;AAGtE,MAAM,WAAW,wBAAwB;IACvC,sBAAsB,EAAE,MAAM,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC,CAAC;IACzE,mBAAmB,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,qBAAqB,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE;AAED,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC;IACzB,WAAW,EAAE,CAAC,CAAC,EACb,QAAQ,EAAE,CACR,EAAE,EAAE,CAAC,EACL,gBAAgB,EAAE,wBAAwB,KACvC,YAAY,CAAC,CAAC,CAAC,EACpB,gBAAgB,CAAC,EAAE,wBAAwB,KACxC,OAAO,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAChF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEtD,MAAM,MAAM,UAAU,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CACtE,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC,KACtB,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B,MAAM,MAAM,kBAAkB,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAC1E,CACE,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC,EAC7B,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,iBAAiB,GAAG,SAAS,KACvC,OAAO,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,MAAM,MAAM,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI;IAClE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACxB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B,CAAC;AAqBF;;GAEG;AACH,eAAO,MAAM,qBAAqB,4BAAsB,CAAC;AAEzD,wBAAgB,mBAAmB,CACjC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,EAE7C,UAAU,EAAE,CAAC,EACb,EAAE,EAAE,CACF,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EACvB,QAAQ,EAAE,cAAc,KACrB,OAAO,CAAC,gBAAgB,CAAC,EAC9B,WAAW,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACrD,IAAI,EAAE,iBAAiB,EACvB,QAAQ,CAAC,EAAE,QAAQ,GAClB,OAAO,CAAC,YAAY,CAAC,CAAC;AAEzB,wBAAgB,mBAAmB,CACjC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,EAE7C,UAAU,EAAE,CAAC,EACb,EAAE,EAAE,CACF,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EACvB,QAAQ,EAAE,cAAc,KACrB,OAAO,CAAC,gBAAgB,CAAC,EAC9B,OAAO,EAAE,OAAO,EAChB,QAAQ,CAAC,EAAE,QAAQ,GAClB,OAAO,CAAC,YAAY,CAAC,CAAC;AAyYzB,qBAAa,kBAAmB,SAAQ,KAAK;gBAEzC,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,EAC1B,cAAc,EAAE,MAAM,GAAG,MAAM;CAMlC;AAmBD,yCAAyC;AACzC,wBAAgB,WAAW,CAEzB,QAAQ,EAAE,kBAAkB,GAAG,iBAAiB,CAAC,GAAG,CAAC,EACrD,IAAI,EAAE,MAAM,GAEX,iBAAiB,CAAC,GAAG,CAAC,CAaxB"}
+{"version":3,"file":"process-mutations.d.ts","sourceRoot":"","sources":["../../../../zero-server/src/process-mutations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAG3D,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAC5D,OAAO,KAAK,CAAC,MAAM,4BAA4B,CAAC;AAWhD,OAAO,EAIL,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EAGnB,KAAK,gBAAgB,EAErB,KAAK,YAAY,EAClB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,0CAA0C,CAAC;AAEjF,OAAO,KAAK,EAAC,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,aAAa,CAAC;AAGtE,MAAM,WAAW,wBAAwB;IACvC,sBAAsB,EAAE,MAAM,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAC,CAAC;IACzE,mBAAmB,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,qBAAqB,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE;AAED,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC;IACzB,WAAW,EAAE,CAAC,CAAC,EACb,QAAQ,EAAE,CACR,EAAE,EAAE,CAAC,EACL,gBAAgB,EAAE,wBAAwB,KACvC,YAAY,CAAC,CAAC,CAAC,EACpB,gBAAgB,CAAC,EAAE,wBAAwB,KACxC,OAAO,CAAC,CAAC,CAAC,CAAC;CACjB;AAED,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAChF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEtD,MAAM,MAAM,UAAU,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CACtE,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC,KACtB,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B,MAAM,MAAM,kBAAkB,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAC1E,CACE,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC,EAC7B,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,iBAAiB,GAAG,SAAS,KACvC,OAAO,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,MAAM,MAAM,CAAC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI;IAClE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACxB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B,CAAC;AAqBF;;GAEG;AACH,eAAO,MAAM,qBAAqB,4BAAsB,CAAC;AAEzD,wBAAgB,mBAAmB,CACjC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,EAE7C,UAAU,EAAE,CAAC,EACb,EAAE,EAAE,CACF,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EACvB,QAAQ,EAAE,cAAc,KACrB,OAAO,CAAC,gBAAgB,CAAC,EAC9B,WAAW,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACrD,IAAI,EAAE,iBAAiB,EACvB,QAAQ,CAAC,EAAE,QAAQ,GAClB,OAAO,CAAC,YAAY,CAAC,CAAC;AAEzB,wBAAgB,mBAAmB,CACjC,CAAC,SAAS,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,EAE7C,UAAU,EAAE,CAAC,EACb,EAAE,EAAE,CACF,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EACvB,QAAQ,EAAE,cAAc,KACrB,OAAO,CAAC,gBAAgB,CAAC,EAC9B,OAAO,EAAE,OAAO,EAChB,QAAQ,CAAC,EAAE,QAAQ,GAClB,OAAO,CAAC,YAAY,CAAC,CAAC;AAwYzB,qBAAa,kBAAmB,SAAQ,KAAK;gBAEzC,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,EAC1B,cAAc,EAAE,MAAM,GAAG,MAAM;CAMlC;AAmBD,yCAAyC;AACzC,wBAAgB,WAAW,CAEzB,QAAQ,EAAE,kBAAkB,GAAG,iBAAiB,CAAC,GAAG,CAAC,EACrD,IAAI,EAAE,MAAM,GAEX,iBAAiB,CAAC,GAAG,CAAC,CAaxB"}

package/out/zero-server/src/process-mutations.js

@@ -130,10 +130,7 @@ async function handleMutateRequest(dbProvider, cb, queryStringOrRequest, bodyOrL
       let mutationPhase = "preTransaction";
       const transactProxy = async (innerCb) => {
         mutationPhase = "transactionPending";
-        const result = await transactor.transact(
-          m,
-          (tx, name, args) => applicationErrorWrapper(() => innerCb(tx, name, args))
-        );
+        const result = await transactor.transact(m, innerCb);
         mutationPhase = "postCommit";
         return result;
       };
@@ -222,19 +219,20 @@ class Transactor {
             }
           };
         }
-        if (isApplicationError(error)) {
-          appError = error;
-          this.#lc.warn?.(
-            `Application error processing mutation ${mutation.id} for client ${mutation.clientID}`,
-            appError
+        if (appError !== void 0) {
+          this.#lc.error?.(
+            `Retry also failed for mutation ${mutation.id} for client ${mutation.clientID}`,
+            error
           );
-          continue;
+          throw error;
         }
-        this.#lc.error?.(
-          `Unexpected error processing mutation ${mutation.id} for client ${mutation.clientID}`,
-          error
+        const originalError = error instanceof DatabaseTransactionError ? error.cause ?? error : error;
+        appError = wrapWithApplicationError(originalError);
+        this.#lc.warn?.(
+          `Error processing mutation ${mutation.id} for client ${mutation.clientID}, retrying without mutator`,
+          appError
         );
-        throw error;
+        continue;
       }
     }
   };
@@ -262,11 +260,7 @@ class Transactor {
             this.#lc.debug?.(
               `Executing mutator '${mutation.name}' (id=${mutation.id})`
             );
-            try {
-              await cb(dbTx, mutation.name, mutation.args[0]);
-            } catch (appError2) {
-              throw wrapWithApplicationError(appError2);
-            }
+            await cb(dbTx, mutation.name, mutation.args[0]);
           } else {
             const mutationResult = makeAppErrorResponse(mutation, appError);
             await transactionHooks.writeMutationResult(mutationResult);

package/out/zero-server/src/process-mutations.js.map

@@ -1 +1 @@
-{"version":3,"file":"process-mutations.js","sources":["../../../../zero-server/src/process-mutations.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {getErrorDetails, getErrorMessage} from '../../shared/src/error.ts';\nimport type {ReadonlyJSONValue} from '../../shared/src/json.ts';\nimport {promiseVoid} from '../../shared/src/resolved-promises.ts';\nimport type {MaybePromise} from '../../shared/src/types.ts';\nimport * as v from '../../shared/src/valita.ts';\nimport {MutationAlreadyProcessedError} from '../../zero-cache/src/services/mutagen/error.ts';\nimport type {ApplicationError} from '../../zero-protocol/src/application-error.ts';\nimport {\n  isApplicationError,\n  wrapWithApplicationError,\n} from '../../zero-protocol/src/application-error.ts';\nimport {ErrorKind} from '../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../zero-protocol/src/error-reason.ts';\nimport type {PushFailedBody} from '../../zero-protocol/src/error.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  cleanupResultsArgSchema,\n  pushBodySchema,\n  pushParamsSchema,\n  type CleanupResultsArg,\n  type CustomMutation,\n  type Mutation,\n  type MutationID,\n  type MutationResponse,\n  type PushBody,\n  type PushResponse,\n} from '../../zero-protocol/src/push.ts';\nimport type {AnyMutatorRegistry} from '../../zql/src/mutate/mutator-registry.ts';\nimport {isMutator} from '../../zql/src/mutate/mutator.ts';\nimport type {CustomMutatorDefs, CustomMutatorImpl} from './custom.ts';\nimport {createLogContext} from './logging.ts';\n\nexport interface TransactionProviderHooks {\n  updateClientMutationID: () => Promise<{lastMutationID: number | bigint}>;\n  writeMutationResult: (result: MutationResponse) => Promise<void>;\n  deleteMutationResults: (args: CleanupResultsArg) => Promise<void>;\n}\n\nexport interface TransactionProviderInput {\n  upstreamSchema: string;\n  clientGroupID: string;\n  clientID: string;\n  mutationID: number;\n}\n\n/**\n * Defines the abstract interface for a database that PushProcessor can execute\n * transactions against.\n */\nexport interface Database<T> {\n  transaction: <R>(\n    callback: (\n      tx: T,\n      transactionHooks: TransactionProviderHooks,\n    ) => MaybePromise<R>,\n    transactionInput?: TransactionProviderInput,\n  ) => Promise<R>;\n}\n\nexport type ExtractTransactionType<D> = D extends Database<infer T> ? T : never;\nexport type Params = v.Infer<typeof pushParamsSchema>;\n\nexport type TransactFn<D extends Database<ExtractTransactionType<D>>> = (\n  cb: TransactFnCallback<D>,\n) => Promise<MutationResponse>;\n\nexport type TransactFnCallback<D extends Database<ExtractTransactionType<D>>> =\n  (\n    tx: ExtractTransactionType<D>,\n    mutatorName: string,\n    mutatorArgs: ReadonlyJSONValue | undefined,\n  ) => Promise<void>;\n\nexport type Parsed<D extends Database<ExtractTransactionType<D>>> = {\n  transact: TransactFn<D>;\n  mutations: CustomMutation[];\n};\n\ntype MutationPhase = 'preTransaction' | 'transactionPending' | 'postCommit';\n\nconst applicationErrorWrapper = async <T>(fn: () => Promise<T>): Promise<T> => {\n  try {\n    return await fn();\n  } catch (error) {\n    if (\n      error instanceof DatabaseTransactionError ||\n      error instanceof OutOfOrderMutation ||\n      error instanceof MutationAlreadyProcessedError ||\n      isApplicationError(error)\n    ) {\n      throw error;\n    }\n\n    throw wrapWithApplicationError(error);\n  }\n};\n\n/**\n * @deprecated Use {@linkcode handleMutateRequest} instead.\n */\nexport const handleMutationRequest = handleMutateRequest;\n\nexport function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  queryString: URLSearchParams | Record<string, string>,\n  body: ReadonlyJSONValue,\n  logLevel?: LogLevel,\n): Promise<PushResponse>;\n\nexport function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  request: Request,\n  logLevel?: LogLevel,\n): Promise<PushResponse>;\n\nexport async function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  queryStringOrRequest: Request | URLSearchParams | Record<string, string>,\n  bodyOrLogLevel?: ReadonlyJSONValue | LogLevel,\n  logLevel?: LogLevel,\n): Promise<PushResponse> {\n  // Parse overload arguments\n  const isRequestOverload = queryStringOrRequest instanceof Request;\n\n  let request: Request | undefined;\n  let queryString: URLSearchParams | Record<string, string>;\n  let jsonBody: unknown;\n\n  let lc: LogContext;\n\n  if (isRequestOverload) {\n    request = queryStringOrRequest;\n    const level = (bodyOrLogLevel as LogLevel | undefined) ?? 'info';\n\n    // Create log context early, before extracting JSON from Request\n    lc = createLogContext(level).withContext('PushProcessor');\n\n    const url = new URL(request.url);\n    queryString = url.searchParams;\n\n    try {\n      jsonBody = await request.json();\n    } catch (error) {\n      lc.error?.('Failed to parse push body', error);\n      const message = `Failed to parse push body: ${getErrorMessage(error)}`;\n      const details = getErrorDetails(error);\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.Server,\n        reason: ErrorReason.Parse,\n        message,\n        mutationIDs: [],\n        ...(details ? {details} : {}),\n      } as const satisfies PushFailedBody;\n    }\n  } else {\n    queryString = queryStringOrRequest;\n    jsonBody = bodyOrLogLevel;\n    const level = logLevel ?? 'info';\n    lc = createLogContext(level).withContext('PushProcessor');\n  }\n\n  let mutationIDs: MutationID[] = [];\n\n  let pushBody: PushBody;\n  try {\n    pushBody = v.parse(jsonBody, pushBodySchema);\n    mutationIDs = pushBody.mutations.map(m => ({\n      id: m.id,\n      clientID: m.clientID,\n    }));\n  } catch (error) {\n    lc.error?.('Failed to parse push body', error);\n    const message = `Failed to parse push body: ${getErrorMessage(error)}`;\n    const details = getErrorDetails(error);\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.Parse,\n      message,\n      mutationIDs,\n      ...(details ? {details} : {}),\n    } as const satisfies PushFailedBody;\n  }\n\n  let queryParams: Params;\n  try {\n    const queryStringObj =\n      queryString instanceof URLSearchParams\n        ? Object.fromEntries(queryString)\n        : queryString;\n    queryParams = v.parse(queryStringObj, pushParamsSchema, 'passthrough');\n  } catch (error) {\n    lc.error?.('Failed to parse push query parameters', error);\n    const message = `Failed to parse push query parameters: ${getErrorMessage(error)}`;\n    const details = getErrorDetails(error);\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.Parse,\n      message,\n      mutationIDs,\n      ...(details ? {details} : {}),\n    } as const satisfies PushFailedBody;\n  }\n\n  if (pushBody.pushVersion !== 1) {\n    const response = {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.UnsupportedPushVersion,\n      mutationIDs,\n      message: `Unsupported push version: ${pushBody.pushVersion}`,\n    } as const satisfies PushFailedBody;\n    return response;\n  }\n\n  const responses: MutationResponse[] = [];\n  let processedCount = 0;\n\n  try {\n    const transactor = new Transactor(dbProvider, pushBody, queryParams, lc);\n\n    // Each mutation goes through three phases:\n    //   1. Pre-transaction: user logic that runs before `transact` is called. If\n    //      this throws we still advance LMID and persist the failure result.\n    //   2. Transaction: the callback passed to `transact`, which can be retried\n    //      if it fails with an ApplicationError.\n    //   3. Post-commit: any logic that runs after `transact` resolves. Failures\n    //      here are logged but the mutation remains committed.\n    for (const m of pushBody.mutations) {\n      // Handle internal mutations (like cleanup) directly without user dispatch\n      if (m.type === 'custom' && m.name === CLEANUP_RESULTS_MUTATION_NAME) {\n        lc.debug?.(\n          `Processing internal mutation '${m.name}' (clientID=${m.clientID})`,\n        );\n        try {\n          await processCleanupResultsMutation(dbProvider, m, queryParams, lc);\n          // No response added - this is fire-and-forget\n          processedCount++;\n        } catch (error) {\n          lc.warn?.(\n            `Failed to process cleanup mutation for client ${m.clientID}`,\n            error,\n          );\n          // Don't fail the whole push for cleanup errors\n          processedCount++;\n        }\n        continue;\n      }\n\n      assert(m.type === 'custom', 'Expected custom mutation');\n      lc.debug?.(\n        `Processing mutation '${m.name}' (id=${m.id}, clientID=${m.clientID})`,\n      );\n\n      let mutationPhase: MutationPhase = 'preTransaction';\n\n      const transactProxy: TransactFn<D> = async innerCb => {\n        mutationPhase = 'transactionPending';\n        const result = await transactor.transact(m, (tx, name, args) =>\n          applicationErrorWrapper(() => innerCb(tx, name, args)),\n        );\n        mutationPhase = 'postCommit';\n        return result;\n      };\n\n      try {\n        const res = await applicationErrorWrapper(() => cb(transactProxy, m));\n        responses.push(res);\n        lc.debug?.(`Mutation '${m.name}' (id=${m.id}) completed successfully`);\n\n        processedCount++;\n      } catch (error) {\n        if (!isApplicationError(error)) {\n          throw error;\n        }\n\n        if (mutationPhase === 'preTransaction') {\n          // Pre-transaction\n          await transactor.persistPreTransactionFailure(m, error);\n        } else if (mutationPhase === 'postCommit') {\n          // Post-commit\n          lc.error?.(\n            `Post-commit mutation handler failed for mutation ${m.id} for client ${m.clientID}`,\n            error,\n          );\n        }\n\n        lc.warn?.(\n          `Application error processing mutation ${m.id} for client ${m.clientID}`,\n          error,\n        );\n        responses.push(makeAppErrorResponse(m, error));\n\n        processedCount++;\n      }\n    }\n\n    return {\n      mutations: responses,\n    };\n  } catch (error) {\n    lc.error?.('Failed to process push request', error);\n    // only include mutationIDs for mutations that were not processed\n    const unprocessedMutationIDs = mutationIDs.slice(processedCount);\n\n    const message = getErrorMessage(error);\n    const details = getErrorDetails(error);\n\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason:\n        error instanceof OutOfOrderMutation\n          ? ErrorReason.OutOfOrderMutation\n          : error instanceof DatabaseTransactionError\n            ? ErrorReason.Database\n            : ErrorReason.Internal,\n      message,\n      mutationIDs: unprocessedMutationIDs,\n      ...(details ? {details} : {}),\n    };\n  }\n}\n\nclass Transactor<D extends Database<ExtractTransactionType<D>>> {\n  readonly #dbProvider: D;\n  readonly #req: PushBody;\n  readonly #params: Params;\n  readonly #lc: LogContext;\n\n  constructor(dbProvider: D, req: PushBody, params: Params, lc: LogContext) {\n    this.#dbProvider = dbProvider;\n    this.#req = req;\n    this.#params = params;\n    this.#lc = lc;\n  }\n\n  transact = async (\n    mutation: CustomMutation,\n    cb: TransactFnCallback<D>,\n  ): Promise<MutationResponse> => {\n    let appError: ApplicationError | undefined = undefined;\n    for (;;) {\n      try {\n        const ret = await this.#transactImpl(mutation, cb, appError);\n        if (appError !== undefined) {\n          this.#lc.warn?.(\n            `Mutation ${mutation.id} for client ${mutation.clientID} was retried after an error`,\n            appError,\n          );\n          return makeAppErrorResponse(mutation, appError);\n        }\n\n        return ret;\n      } catch (error) {\n        if (error instanceof OutOfOrderMutation) {\n          this.#lc.error?.(error);\n          throw error;\n        }\n\n        if (error instanceof MutationAlreadyProcessedError) {\n          this.#lc.warn?.(error);\n          return {\n            id: {\n              clientID: mutation.clientID,\n              id: mutation.id,\n            },\n            result: {\n              error: 'alreadyProcessed',\n              details: error.message,\n            },\n          };\n        }\n\n        if (isApplicationError(error)) {\n          appError = error;\n          this.#lc.warn?.(\n            `Application error processing mutation ${mutation.id} for client ${mutation.clientID}`,\n            appError,\n          );\n          continue;\n        }\n\n        this.#lc.error?.(\n          `Unexpected error processing mutation ${mutation.id} for client ${mutation.clientID}`,\n          error,\n        );\n\n        throw error;\n      }\n    }\n  };\n\n  async persistPreTransactionFailure(\n    mutation: CustomMutation,\n    appError: ApplicationError<ReadonlyJSONValue | undefined>,\n  ): Promise<MutationResponse> {\n    // User-land code threw before calling `transact`. We still need to bump the\n    // LMID for this mutation and persist the error so that the client knows it failed.\n    const ret = await this.#transactImpl(\n      mutation,\n      // noop callback since there's no transaction to execute\n      () => promiseVoid,\n      appError,\n    );\n    return ret;\n  }\n\n  async #transactImpl(\n    mutation: CustomMutation,\n    cb: TransactFnCallback<D>,\n    appError: ApplicationError | undefined,\n  ): Promise<MutationResponse> {\n    let transactionPhase: DatabaseTransactionPhase = 'open';\n\n    try {\n      const ret = await this.#dbProvider.transaction(\n        async (dbTx, transactionHooks) => {\n          // update the transaction phase to 'execute' after the transaction is opened\n          transactionPhase = 'execute';\n\n          await this.#checkAndIncrementLastMutationID(\n            transactionHooks,\n            mutation.clientID,\n            mutation.id,\n          );\n\n          if (appError === undefined) {\n            this.#lc.debug?.(\n              `Executing mutator '${mutation.name}' (id=${mutation.id})`,\n            );\n            try {\n              await cb(dbTx, mutation.name, mutation.args[0]);\n            } catch (appError) {\n              throw wrapWithApplicationError(appError);\n            }\n          } else {\n            const mutationResult = makeAppErrorResponse(mutation, appError);\n            await transactionHooks.writeMutationResult(mutationResult);\n          }\n\n          return {\n            id: {\n              clientID: mutation.clientID,\n              id: mutation.id,\n            },\n            result: {},\n          };\n        },\n        this.#getTransactionInput(mutation),\n      );\n\n      return ret;\n    } catch (error) {\n      if (\n        isApplicationError(error) ||\n        error instanceof OutOfOrderMutation ||\n        error instanceof MutationAlreadyProcessedError\n      ) {\n        throw error;\n      }\n\n      throw new DatabaseTransactionError(transactionPhase, {cause: error});\n    }\n  }\n\n  #getTransactionInput(mutation: CustomMutation): TransactionProviderInput {\n    return {\n      upstreamSchema: this.#params.schema,\n      clientGroupID: this.#req.clientGroupID,\n      clientID: mutation.clientID,\n      mutationID: mutation.id,\n    };\n  }\n\n  async #checkAndIncrementLastMutationID(\n    transactionHooks: TransactionProviderHooks,\n    clientID: string,\n    receivedMutationID: number,\n  ) {\n    const {lastMutationID} = await transactionHooks.updateClientMutationID();\n\n    if (receivedMutationID < lastMutationID) {\n      throw new MutationAlreadyProcessedError(\n        clientID,\n        receivedMutationID,\n        lastMutationID,\n      );\n    } else if (receivedMutationID > lastMutationID) {\n      throw new OutOfOrderMutation(\n        clientID,\n        receivedMutationID,\n        lastMutationID,\n      );\n    }\n  }\n}\n\nexport class OutOfOrderMutation extends Error {\n  constructor(\n    clientID: string,\n    receivedMutationID: number,\n    lastMutationID: number | bigint,\n  ) {\n    super(\n      `Client ${clientID} sent mutation ID ${receivedMutationID} but expected ${lastMutationID}`,\n    );\n  }\n}\n\nfunction makeAppErrorResponse(\n  m: Mutation,\n  error: ApplicationError<ReadonlyJSONValue | undefined>,\n): MutationResponse {\n  return {\n    id: {\n      clientID: m.clientID,\n      id: m.id,\n    },\n    result: {\n      error: 'app',\n      message: error.message,\n      ...(error.details ? {details: error.details} : {}),\n    },\n  };\n}\n\n/** @deprecated Use getMutator instead */\nexport function getMutation(\n  // oxlint-disable-next-line no-explicit-any\n  mutators: AnyMutatorRegistry | CustomMutatorDefs<any>,\n  name: string,\n  // oxlint-disable-next-line no-explicit-any\n): CustomMutatorImpl<any> {\n  const path = name.split(/\\.|\\|/);\n  const mutator = getObjectAtPath(mutators, path);\n  assert(typeof mutator === 'function', `could not find mutator ${name}`);\n\n  if (isMutator(mutator)) {\n    // mutator needs to be called with {tx, args, ctx}\n    // CustomMutatorImpl is called with (tx, args, ctx)\n    return (tx, args, ctx) => mutator.fn({args, ctx, tx});\n  }\n\n  // oxlint-disable-next-line no-explicit-any\n  return mutator as CustomMutatorImpl<any>;\n}\n\nfunction getObjectAtPath(\n  obj: Record<string, unknown>,\n  path: string[],\n): unknown {\n  let current: unknown = obj;\n  for (const part of path) {\n    if (typeof current !== 'object' || current === null || !(part in current)) {\n      return undefined;\n    }\n    current = (current as Record<string, unknown>)[part];\n  }\n  return current;\n}\n\n/**\n * Processes internal cleanup mutation that deletes acknowledged mutation results\n * from the upstream database. This runs without LMID tracking since it's an\n * internal operation.\n */\nasync function processCleanupResultsMutation<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  mutation: CustomMutation,\n  queryParams: Params,\n  lc: LogContext,\n): Promise<void> {\n  const parseResult = v.test(mutation.args[0], cleanupResultsArgSchema);\n  if (!parseResult.ok) {\n    lc.warn?.('Cleanup mutation has invalid args', parseResult.error);\n    return;\n  }\n  const args: CleanupResultsArg = parseResult.value;\n\n  // Determine clientID for transaction input based on cleanup type\n  // Note: legacy format without type field is treated as single\n  const clientID =\n    'type' in args && args.type === 'bulk' ? args.clientIDs[0] : args.clientID;\n\n  // Run in a transaction, using the hook for DB-specific operation.\n  // Note: only upstreamSchema is used by deleteMutationResults; the other\n  // fields are required by the interface but ignored for this operation.\n  await dbProvider.transaction(\n    async (_, hooks) => {\n      await hooks.deleteMutationResults(args);\n    },\n    {\n      upstreamSchema: queryParams.schema,\n      clientGroupID: args.clientGroupID,\n      clientID,\n      mutationID: 0,\n    },\n  );\n}\n\ntype DatabaseTransactionPhase = 'open' | 'execute';\nclass DatabaseTransactionError extends Error {\n  constructor(phase: DatabaseTransactionPhase, options?: ErrorOptions) {\n    super(\n      phase === 'open'\n        ? `Failed to open database transaction: ${getErrorMessage(options?.cause)}`\n        : `Database transaction failed after opening: ${getErrorMessage(options?.cause)}`,\n      options,\n    );\n    this.name = 'DatabaseTransactionError';\n  }\n}\n"],"names":["ErrorKind.PushFailed","ErrorOrigin.Server","ErrorReason.Parse","v.parse","ErrorReason.UnsupportedPushVersion","ErrorReason.OutOfOrderMutation","ErrorReason.Database","ErrorReason.Internal","appError","v.test"],"mappings":";;;;;;;;;;;;AAmFA,MAAM,0BAA0B,OAAU,OAAqC;AAC7E,MAAI;AACF,WAAO,MAAM,GAAA;AAAA,EACf,SAAS,OAAO;AACd,QACE,iBAAiB,4BACjB,iBAAiB,sBACjB,iBAAiB,iCACjB,mBAAmB,KAAK,GACxB;AACA,YAAM;AAAA,IACR;AAEA,UAAM,yBAAyB,KAAK;AAAA,EACtC;AACF;AAKO,MAAM,wBAAwB;AA2BrC,eAAsB,oBAGpB,YACA,IAIA,sBACA,gBACA,UACuB;AAEvB,QAAM,oBAAoB,gCAAgC;AAE1D,MAAI;AACJ,MAAI;AACJ,MAAI;AAEJ,MAAI;AAEJ,MAAI,mBAAmB;AACrB,cAAU;AACV,UAAM,QAAS,kBAA2C;AAG1D,SAAK,iBAAiB,KAAK,EAAE,YAAY,eAAe;AAExD,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,kBAAc,IAAI;AAElB,QAAI;AACF,iBAAW,MAAM,QAAQ,KAAA;AAAA,IAC3B,SAAS,OAAO;AACd,SAAG,QAAQ,6BAA6B,KAAK;AAC7C,YAAM,UAAU,8BAA8B,gBAAgB,KAAK,CAAC;AACpE,YAAM,UAAU,gBAAgB,KAAK;AACrC,aAAO;AAAA,QACL,MAAMA;AAAAA,QACN,QAAQC;AAAAA,QACR,QAAQC;AAAAA,QACR;AAAA,QACA,aAAa,CAAA;AAAA,QACb,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,MAAC;AAAA,IAE/B;AAAA,EACF,OAAO;AACL,kBAAc;AACd,eAAW;AACX,UAAM,QAAQ,YAAY;AAC1B,SAAK,iBAAiB,KAAK,EAAE,YAAY,eAAe;AAAA,EAC1D;AAEA,MAAI,cAA4B,CAAA;AAEhC,MAAI;AACJ,MAAI;AACF,eAAWC,MAAQ,UAAU,cAAc;AAC3C,kBAAc,SAAS,UAAU,IAAI,CAAA,OAAM;AAAA,MACzC,IAAI,EAAE;AAAA,MACN,UAAU,EAAE;AAAA,IAAA,EACZ;AAAA,EACJ,SAAS,OAAO;AACd,OAAG,QAAQ,6BAA6B,KAAK;AAC7C,UAAM,UAAU,8BAA8B,gBAAgB,KAAK,CAAC;AACpE,UAAM,UAAU,gBAAgB,KAAK;AACrC,WAAO;AAAA,MACL,MAAMH;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQC;AAAAA,MACR;AAAA,MACA;AAAA,MACA,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,iBACJ,uBAAuB,kBACnB,OAAO,YAAY,WAAW,IAC9B;AACN,kBAAcC,MAAQ,gBAAgB,kBAAkB,aAAa;AAAA,EACvE,SAAS,OAAO;AACd,OAAG,QAAQ,yCAAyC,KAAK;AACzD,UAAM,UAAU,0CAA0C,gBAAgB,KAAK,CAAC;AAChF,UAAM,UAAU,gBAAgB,KAAK;AACrC,WAAO;AAAA,MACL,MAAMH;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQC;AAAAA,MACR;AAAA,MACA;AAAA,MACA,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AAEA,MAAI,SAAS,gBAAgB,GAAG;AAC9B,UAAM,WAAW;AAAA,MACf,MAAMF;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQG;AAAAA,MACR;AAAA,MACA,SAAS,6BAA6B,SAAS,WAAW;AAAA,IAAA;AAE5D,WAAO;AAAA,EACT;AAEA,QAAM,YAAgC,CAAA;AACtC,MAAI,iBAAiB;AAErB,MAAI;AACF,UAAM,aAAa,IAAI,WAAW,YAAY,UAAU,aAAa,EAAE;AASvE,eAAW,KAAK,SAAS,WAAW;AAElC,UAAI,EAAE,SAAS,YAAY,EAAE,SAAS,+BAA+B;AACnE,WAAG;AAAA,UACD,iCAAiC,EAAE,IAAI,eAAe,EAAE,QAAQ;AAAA,QAAA;AAElE,YAAI;AACF,gBAAM,8BAA8B,YAAY,GAAG,aAAa,EAAE;AAElE;AAAA,QACF,SAAS,OAAO;AACd,aAAG;AAAA,YACD,iDAAiD,EAAE,QAAQ;AAAA,YAC3D;AAAA,UAAA;AAGF;AAAA,QACF;AACA;AAAA,MACF;AAEA,aAAO,EAAE,SAAS,UAAU,0BAA0B;AACtD,SAAG;AAAA,QACD,wBAAwB,EAAE,IAAI,SAAS,EAAE,EAAE,cAAc,EAAE,QAAQ;AAAA,MAAA;AAGrE,UAAI,gBAA+B;AAEnC,YAAM,gBAA+B,OAAM,YAAW;AACpD,wBAAgB;AAChB,cAAM,SAAS,MAAM,WAAW;AAAA,UAAS;AAAA,UAAG,CAAC,IAAI,MAAM,SACrD,wBAAwB,MAAM,QAAQ,IAAI,MAAM,IAAI,CAAC;AAAA,QAAA;AAEvD,wBAAgB;AAChB,eAAO;AAAA,MACT;AAEA,UAAI;AACF,cAAM,MAAM,MAAM,wBAAwB,MAAM,GAAG,eAAe,CAAC,CAAC;AACpE,kBAAU,KAAK,GAAG;AAClB,WAAG,QAAQ,aAAa,EAAE,IAAI,SAAS,EAAE,EAAE,0BAA0B;AAErE;AAAA,MACF,SAAS,OAAO;AACd,YAAI,CAAC,mBAAmB,KAAK,GAAG;AAC9B,gBAAM;AAAA,QACR;AAEA,YAAI,kBAAkB,kBAAkB;AAEtC,gBAAM,WAAW,6BAA6B,GAAG,KAAK;AAAA,QACxD,WAAW,kBAAkB,cAAc;AAEzC,aAAG;AAAA,YACD,oDAAoD,EAAE,EAAE,eAAe,EAAE,QAAQ;AAAA,YACjF;AAAA,UAAA;AAAA,QAEJ;AAEA,WAAG;AAAA,UACD,yCAAyC,EAAE,EAAE,eAAe,EAAE,QAAQ;AAAA,UACtE;AAAA,QAAA;AAEF,kBAAU,KAAK,qBAAqB,GAAG,KAAK,CAAC;AAE7C;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,IAAA;AAAA,EAEf,SAAS,OAAO;AACd,OAAG,QAAQ,kCAAkC,KAAK;AAElD,UAAM,yBAAyB,YAAY,MAAM,cAAc;AAE/D,UAAM,UAAU,gBAAgB,KAAK;AACrC,UAAM,UAAU,gBAAgB,KAAK;AAErC,WAAO;AAAA,MACL,MAAMJ;AAAAA,MACN,QAAQC;AAAAA,MACR,QACE,iBAAiB,qBACbI,uBACA,iBAAiB,2BACfC,WACAC;AAAAA,MACR;AAAA,MACA,aAAa;AAAA,MACb,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AACF;AAEA,MAAM,WAA0D;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,YAAe,KAAe,QAAgB,IAAgB;AACxE,SAAK,cAAc;AACnB,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,WAAW,OACT,UACA,OAC8B;AAC9B,QAAI,WAAyC;AAC7C,eAAS;AACP,UAAI;AACF,cAAM,MAAM,MAAM,KAAK,cAAc,UAAU,IAAI,QAAQ;AAC3D,YAAI,aAAa,QAAW;AAC1B,eAAK,IAAI;AAAA,YACP,YAAY,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,YACvD;AAAA,UAAA;AAEF,iBAAO,qBAAqB,UAAU,QAAQ;AAAA,QAChD;AAEA,eAAO;AAAA,MACT,SAAS,OAAO;AACd,YAAI,iBAAiB,oBAAoB;AACvC,eAAK,IAAI,QAAQ,KAAK;AACtB,gBAAM;AAAA,QACR;AAEA,YAAI,iBAAiB,+BAA+B;AAClD,eAAK,IAAI,OAAO,KAAK;AACrB,iBAAO;AAAA,YACL,IAAI;AAAA,cACF,UAAU,SAAS;AAAA,cACnB,IAAI,SAAS;AAAA,YAAA;AAAA,YAEf,QAAQ;AAAA,cACN,OAAO;AAAA,cACP,SAAS,MAAM;AAAA,YAAA;AAAA,UACjB;AAAA,QAEJ;AAEA,YAAI,mBAAmB,KAAK,GAAG;AAC7B,qBAAW;AACX,eAAK,IAAI;AAAA,YACP,yCAAyC,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,YACpF;AAAA,UAAA;AAEF;AAAA,QACF;AAEA,aAAK,IAAI;AAAA,UACP,wCAAwC,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,UACnF;AAAA,QAAA;AAGF,cAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,6BACJ,UACA,UAC2B;AAG3B,UAAM,MAAM,MAAM,KAAK;AAAA,MACrB;AAAA;AAAA,MAEA,MAAM;AAAA,MACN;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cACJ,UACA,IACA,UAC2B;AAC3B,QAAI,mBAA6C;AAEjD,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,YAAY;AAAA,QACjC,OAAO,MAAM,qBAAqB;AAEhC,6BAAmB;AAEnB,gBAAM,KAAK;AAAA,YACT;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,UAAA;AAGX,cAAI,aAAa,QAAW;AAC1B,iBAAK,IAAI;AAAA,cACP,sBAAsB,SAAS,IAAI,SAAS,SAAS,EAAE;AAAA,YAAA;AAEzD,gBAAI;AACF,oBAAM,GAAG,MAAM,SAAS,MAAM,SAAS,KAAK,CAAC,CAAC;AAAA,YAChD,SAASC,WAAU;AACjB,oBAAM,yBAAyBA,SAAQ;AAAA,YACzC;AAAA,UACF,OAAO;AACL,kBAAM,iBAAiB,qBAAqB,UAAU,QAAQ;AAC9D,kBAAM,iBAAiB,oBAAoB,cAAc;AAAA,UAC3D;AAEA,iBAAO;AAAA,YACL,IAAI;AAAA,cACF,UAAU,SAAS;AAAA,cACnB,IAAI,SAAS;AAAA,YAAA;AAAA,YAEf,QAAQ,CAAA;AAAA,UAAC;AAAA,QAEb;AAAA,QACA,KAAK,qBAAqB,QAAQ;AAAA,MAAA;AAGpC,aAAO;AAAA,IACT,SAAS,OAAO;AACd,UACE,mBAAmB,KAAK,KACxB,iBAAiB,sBACjB,iBAAiB,+BACjB;AACA,cAAM;AAAA,MACR;AAEA,YAAM,IAAI,yBAAyB,kBAAkB,EAAC,OAAO,OAAM;AAAA,IACrE;AAAA,EACF;AAAA,EAEA,qBAAqB,UAAoD;AACvE,WAAO;AAAA,MACL,gBAAgB,KAAK,QAAQ;AAAA,MAC7B,eAAe,KAAK,KAAK;AAAA,MACzB,UAAU,SAAS;AAAA,MACnB,YAAY,SAAS;AAAA,IAAA;AAAA,EAEzB;AAAA,EAEA,MAAM,iCACJ,kBACA,UACA,oBACA;AACA,UAAM,EAAC,eAAA,IAAkB,MAAM,iBAAiB,uBAAA;AAEhD,QAAI,qBAAqB,gBAAgB;AACvC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ,WAAW,qBAAqB,gBAAgB;AAC9C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,2BAA2B,MAAM;AAAA,EAC5C,YACE,UACA,oBACA,gBACA;AACA;AAAA,MACE,UAAU,QAAQ,qBAAqB,kBAAkB,iBAAiB,cAAc;AAAA,IAAA;AAAA,EAE5F;AACF;AAEA,SAAS,qBACP,GACA,OACkB;AAClB,SAAO;AAAA,IACL,IAAI;AAAA,MACF,UAAU,EAAE;AAAA,MACZ,IAAI,EAAE;AAAA,IAAA;AAAA,IAER,QAAQ;AAAA,MACN,OAAO;AAAA,MACP,SAAS,MAAM;AAAA,MACf,GAAI,MAAM,UAAU,EAAC,SAAS,MAAM,QAAA,IAAW,CAAA;AAAA,IAAC;AAAA,EAClD;AAEJ;AAGO,SAAS,YAEd,UACA,MAEwB;AACxB,QAAM,OAAO,KAAK,MAAM,OAAO;AAC/B,QAAM,UAAU,gBAAgB,UAAU,IAAI;AAC9C,SAAO,OAAO,YAAY,YAAY,0BAA0B,IAAI,EAAE;AAEtE,MAAI,UAAU,OAAO,GAAG;AAGtB,WAAO,CAAC,IAAI,MAAM,QAAQ,QAAQ,GAAG,EAAC,MAAM,KAAK,IAAG;AAAA,EACtD;AAGA,SAAO;AACT;AAEA,SAAS,gBACP,KACA,MACS;AACT,MAAI,UAAmB;AACvB,aAAW,QAAQ,MAAM;AACvB,QAAI,OAAO,YAAY,YAAY,YAAY,QAAQ,EAAE,QAAQ,UAAU;AACzE,aAAO;AAAA,IACT;AACA,cAAW,QAAoC,IAAI;AAAA,EACrD;AACA,SAAO;AACT;AAOA,eAAe,8BAGb,YACA,UACA,aACA,IACe;AACf,QAAM,cAAcC,KAAO,SAAS,KAAK,CAAC,GAAG,uBAAuB;AACpE,MAAI,CAAC,YAAY,IAAI;AACnB,OAAG,OAAO,qCAAqC,YAAY,KAAK;AAChE;AAAA,EACF;AACA,QAAM,OAA0B,YAAY;AAI5C,QAAM,WACJ,UAAU,QAAQ,KAAK,SAAS,SAAS,KAAK,UAAU,CAAC,IAAI,KAAK;AAKpE,QAAM,WAAW;AAAA,IACf,OAAO,GAAG,UAAU;AAClB,YAAM,MAAM,sBAAsB,IAAI;AAAA,IACxC;AAAA,IACA;AAAA,MACE,gBAAgB,YAAY;AAAA,MAC5B,eAAe,KAAK;AAAA,MACpB;AAAA,MACA,YAAY;AAAA,IAAA;AAAA,EACd;AAEJ;AAGA,MAAM,iCAAiC,MAAM;AAAA,EAC3C,YAAY,OAAiC,SAAwB;AACnE;AAAA,MACE,UAAU,SACN,wCAAwC,gBAAgB,SAAS,KAAK,CAAC,KACvE,8CAA8C,gBAAgB,SAAS,KAAK,CAAC;AAAA,MACjF;AAAA,IAAA;AAEF,SAAK,OAAO;AAAA,EACd;AACF;"}
+{"version":3,"file":"process-mutations.js","sources":["../../../../zero-server/src/process-mutations.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {assert} from '../../shared/src/asserts.ts';\nimport {getErrorDetails, getErrorMessage} from '../../shared/src/error.ts';\nimport type {ReadonlyJSONValue} from '../../shared/src/json.ts';\nimport {promiseVoid} from '../../shared/src/resolved-promises.ts';\nimport type {MaybePromise} from '../../shared/src/types.ts';\nimport * as v from '../../shared/src/valita.ts';\nimport {MutationAlreadyProcessedError} from '../../zero-cache/src/services/mutagen/error.ts';\nimport type {ApplicationError} from '../../zero-protocol/src/application-error.ts';\nimport {\n  isApplicationError,\n  wrapWithApplicationError,\n} from '../../zero-protocol/src/application-error.ts';\nimport {ErrorKind} from '../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../zero-protocol/src/error-reason.ts';\nimport type {PushFailedBody} from '../../zero-protocol/src/error.ts';\nimport {\n  CLEANUP_RESULTS_MUTATION_NAME,\n  cleanupResultsArgSchema,\n  pushBodySchema,\n  pushParamsSchema,\n  type CleanupResultsArg,\n  type CustomMutation,\n  type Mutation,\n  type MutationID,\n  type MutationResponse,\n  type PushBody,\n  type PushResponse,\n} from '../../zero-protocol/src/push.ts';\nimport type {AnyMutatorRegistry} from '../../zql/src/mutate/mutator-registry.ts';\nimport {isMutator} from '../../zql/src/mutate/mutator.ts';\nimport type {CustomMutatorDefs, CustomMutatorImpl} from './custom.ts';\nimport {createLogContext} from './logging.ts';\n\nexport interface TransactionProviderHooks {\n  updateClientMutationID: () => Promise<{lastMutationID: number | bigint}>;\n  writeMutationResult: (result: MutationResponse) => Promise<void>;\n  deleteMutationResults: (args: CleanupResultsArg) => Promise<void>;\n}\n\nexport interface TransactionProviderInput {\n  upstreamSchema: string;\n  clientGroupID: string;\n  clientID: string;\n  mutationID: number;\n}\n\n/**\n * Defines the abstract interface for a database that PushProcessor can execute\n * transactions against.\n */\nexport interface Database<T> {\n  transaction: <R>(\n    callback: (\n      tx: T,\n      transactionHooks: TransactionProviderHooks,\n    ) => MaybePromise<R>,\n    transactionInput?: TransactionProviderInput,\n  ) => Promise<R>;\n}\n\nexport type ExtractTransactionType<D> = D extends Database<infer T> ? T : never;\nexport type Params = v.Infer<typeof pushParamsSchema>;\n\nexport type TransactFn<D extends Database<ExtractTransactionType<D>>> = (\n  cb: TransactFnCallback<D>,\n) => Promise<MutationResponse>;\n\nexport type TransactFnCallback<D extends Database<ExtractTransactionType<D>>> =\n  (\n    tx: ExtractTransactionType<D>,\n    mutatorName: string,\n    mutatorArgs: ReadonlyJSONValue | undefined,\n  ) => Promise<void>;\n\nexport type Parsed<D extends Database<ExtractTransactionType<D>>> = {\n  transact: TransactFn<D>;\n  mutations: CustomMutation[];\n};\n\ntype MutationPhase = 'preTransaction' | 'transactionPending' | 'postCommit';\n\nconst applicationErrorWrapper = async <T>(fn: () => Promise<T>): Promise<T> => {\n  try {\n    return await fn();\n  } catch (error) {\n    if (\n      error instanceof DatabaseTransactionError ||\n      error instanceof OutOfOrderMutation ||\n      error instanceof MutationAlreadyProcessedError ||\n      isApplicationError(error)\n    ) {\n      throw error;\n    }\n\n    throw wrapWithApplicationError(error);\n  }\n};\n\n/**\n * @deprecated Use {@linkcode handleMutateRequest} instead.\n */\nexport const handleMutationRequest = handleMutateRequest;\n\nexport function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  queryString: URLSearchParams | Record<string, string>,\n  body: ReadonlyJSONValue,\n  logLevel?: LogLevel,\n): Promise<PushResponse>;\n\nexport function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  request: Request,\n  logLevel?: LogLevel,\n): Promise<PushResponse>;\n\nexport async function handleMutateRequest<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  cb: (\n    transact: TransactFn<D>,\n    mutation: CustomMutation,\n  ) => Promise<MutationResponse>,\n  queryStringOrRequest: Request | URLSearchParams | Record<string, string>,\n  bodyOrLogLevel?: ReadonlyJSONValue | LogLevel,\n  logLevel?: LogLevel,\n): Promise<PushResponse> {\n  // Parse overload arguments\n  const isRequestOverload = queryStringOrRequest instanceof Request;\n\n  let request: Request | undefined;\n  let queryString: URLSearchParams | Record<string, string>;\n  let jsonBody: unknown;\n\n  let lc: LogContext;\n\n  if (isRequestOverload) {\n    request = queryStringOrRequest;\n    const level = (bodyOrLogLevel as LogLevel | undefined) ?? 'info';\n\n    // Create log context early, before extracting JSON from Request\n    lc = createLogContext(level).withContext('PushProcessor');\n\n    const url = new URL(request.url);\n    queryString = url.searchParams;\n\n    try {\n      jsonBody = await request.json();\n    } catch (error) {\n      lc.error?.('Failed to parse push body', error);\n      const message = `Failed to parse push body: ${getErrorMessage(error)}`;\n      const details = getErrorDetails(error);\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.Server,\n        reason: ErrorReason.Parse,\n        message,\n        mutationIDs: [],\n        ...(details ? {details} : {}),\n      } as const satisfies PushFailedBody;\n    }\n  } else {\n    queryString = queryStringOrRequest;\n    jsonBody = bodyOrLogLevel;\n    const level = logLevel ?? 'info';\n    lc = createLogContext(level).withContext('PushProcessor');\n  }\n\n  let mutationIDs: MutationID[] = [];\n\n  let pushBody: PushBody;\n  try {\n    pushBody = v.parse(jsonBody, pushBodySchema);\n    mutationIDs = pushBody.mutations.map(m => ({\n      id: m.id,\n      clientID: m.clientID,\n    }));\n  } catch (error) {\n    lc.error?.('Failed to parse push body', error);\n    const message = `Failed to parse push body: ${getErrorMessage(error)}`;\n    const details = getErrorDetails(error);\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.Parse,\n      message,\n      mutationIDs,\n      ...(details ? {details} : {}),\n    } as const satisfies PushFailedBody;\n  }\n\n  let queryParams: Params;\n  try {\n    const queryStringObj =\n      queryString instanceof URLSearchParams\n        ? Object.fromEntries(queryString)\n        : queryString;\n    queryParams = v.parse(queryStringObj, pushParamsSchema, 'passthrough');\n  } catch (error) {\n    lc.error?.('Failed to parse push query parameters', error);\n    const message = `Failed to parse push query parameters: ${getErrorMessage(error)}`;\n    const details = getErrorDetails(error);\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.Parse,\n      message,\n      mutationIDs,\n      ...(details ? {details} : {}),\n    } as const satisfies PushFailedBody;\n  }\n\n  if (pushBody.pushVersion !== 1) {\n    const response = {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason: ErrorReason.UnsupportedPushVersion,\n      mutationIDs,\n      message: `Unsupported push version: ${pushBody.pushVersion}`,\n    } as const satisfies PushFailedBody;\n    return response;\n  }\n\n  const responses: MutationResponse[] = [];\n  let processedCount = 0;\n\n  try {\n    const transactor = new Transactor(dbProvider, pushBody, queryParams, lc);\n\n    // Each mutation goes through three phases:\n    //   1. Pre-transaction: user logic that runs before `transact` is called. If\n    //      this throws we still advance LMID and persist the failure result.\n    //   2. Transaction: the callback passed to `transact`, which can be retried\n    //      if it fails with an ApplicationError.\n    //   3. Post-commit: any logic that runs after `transact` resolves. Failures\n    //      here are logged but the mutation remains committed.\n    for (const m of pushBody.mutations) {\n      // Handle internal mutations (like cleanup) directly without user dispatch\n      if (m.type === 'custom' && m.name === CLEANUP_RESULTS_MUTATION_NAME) {\n        lc.debug?.(\n          `Processing internal mutation '${m.name}' (clientID=${m.clientID})`,\n        );\n        try {\n          await processCleanupResultsMutation(dbProvider, m, queryParams, lc);\n          // No response added - this is fire-and-forget\n          processedCount++;\n        } catch (error) {\n          lc.warn?.(\n            `Failed to process cleanup mutation for client ${m.clientID}`,\n            error,\n          );\n          // Don't fail the whole push for cleanup errors\n          processedCount++;\n        }\n        continue;\n      }\n\n      assert(m.type === 'custom', 'Expected custom mutation');\n      lc.debug?.(\n        `Processing mutation '${m.name}' (id=${m.id}, clientID=${m.clientID})`,\n      );\n\n      let mutationPhase: MutationPhase = 'preTransaction';\n\n      const transactProxy: TransactFn<D> = async innerCb => {\n        mutationPhase = 'transactionPending';\n        const result = await transactor.transact(m, innerCb);\n        mutationPhase = 'postCommit';\n        return result;\n      };\n\n      try {\n        const res = await applicationErrorWrapper(() => cb(transactProxy, m));\n        responses.push(res);\n        lc.debug?.(`Mutation '${m.name}' (id=${m.id}) completed successfully`);\n\n        processedCount++;\n      } catch (error) {\n        if (!isApplicationError(error)) {\n          throw error;\n        }\n\n        if (mutationPhase === 'preTransaction') {\n          // Pre-transaction\n          await transactor.persistPreTransactionFailure(m, error);\n        } else if (mutationPhase === 'postCommit') {\n          // Post-commit\n          lc.error?.(\n            `Post-commit mutation handler failed for mutation ${m.id} for client ${m.clientID}`,\n            error,\n          );\n        }\n\n        lc.warn?.(\n          `Application error processing mutation ${m.id} for client ${m.clientID}`,\n          error,\n        );\n        responses.push(makeAppErrorResponse(m, error));\n\n        processedCount++;\n      }\n    }\n\n    return {\n      mutations: responses,\n    };\n  } catch (error) {\n    lc.error?.('Failed to process push request', error);\n    // only include mutationIDs for mutations that were not processed\n    const unprocessedMutationIDs = mutationIDs.slice(processedCount);\n\n    const message = getErrorMessage(error);\n    const details = getErrorDetails(error);\n\n    return {\n      kind: ErrorKind.PushFailed,\n      origin: ErrorOrigin.Server,\n      reason:\n        error instanceof OutOfOrderMutation\n          ? ErrorReason.OutOfOrderMutation\n          : error instanceof DatabaseTransactionError\n            ? ErrorReason.Database\n            : ErrorReason.Internal,\n      message,\n      mutationIDs: unprocessedMutationIDs,\n      ...(details ? {details} : {}),\n    };\n  }\n}\n\nclass Transactor<D extends Database<ExtractTransactionType<D>>> {\n  readonly #dbProvider: D;\n  readonly #req: PushBody;\n  readonly #params: Params;\n  readonly #lc: LogContext;\n\n  constructor(dbProvider: D, req: PushBody, params: Params, lc: LogContext) {\n    this.#dbProvider = dbProvider;\n    this.#req = req;\n    this.#params = params;\n    this.#lc = lc;\n  }\n\n  transact = async (\n    mutation: CustomMutation,\n    cb: TransactFnCallback<D>,\n  ): Promise<MutationResponse> => {\n    let appError: ApplicationError | undefined = undefined;\n    for (;;) {\n      try {\n        const ret = await this.#transactImpl(mutation, cb, appError);\n        if (appError !== undefined) {\n          this.#lc.warn?.(\n            `Mutation ${mutation.id} for client ${mutation.clientID} was retried after an error`,\n            appError,\n          );\n          return makeAppErrorResponse(mutation, appError);\n        }\n\n        return ret;\n      } catch (error) {\n        if (error instanceof OutOfOrderMutation) {\n          this.#lc.error?.(error);\n          throw error;\n        }\n\n        if (error instanceof MutationAlreadyProcessedError) {\n          this.#lc.warn?.(error);\n          return {\n            id: {\n              clientID: mutation.clientID,\n              id: mutation.id,\n            },\n            result: {\n              error: 'alreadyProcessed',\n              details: error.message,\n            },\n          };\n        }\n\n        if (appError !== undefined) {\n          // Retry also failed → internal error, cannot skip mutation\n          this.#lc.error?.(\n            `Retry also failed for mutation ${mutation.id} for client ${mutation.clientID}`,\n            error,\n          );\n          throw error;\n        }\n\n        // First attempt failed → store error and retry without mutator\n        const originalError =\n          error instanceof DatabaseTransactionError\n            ? (error.cause ?? error)\n            : error;\n        appError = wrapWithApplicationError(originalError);\n        this.#lc.warn?.(\n          `Error processing mutation ${mutation.id} for client ${mutation.clientID}, retrying without mutator`,\n          appError,\n        );\n        continue;\n      }\n    }\n  };\n\n  async persistPreTransactionFailure(\n    mutation: CustomMutation,\n    appError: ApplicationError<ReadonlyJSONValue | undefined>,\n  ): Promise<MutationResponse> {\n    // User-land code threw before calling `transact`. We still need to bump the\n    // LMID for this mutation and persist the error so that the client knows it failed.\n    const ret = await this.#transactImpl(\n      mutation,\n      // noop callback since there's no transaction to execute\n      () => promiseVoid,\n      appError,\n    );\n    return ret;\n  }\n\n  async #transactImpl(\n    mutation: CustomMutation,\n    cb: TransactFnCallback<D>,\n    appError: ApplicationError | undefined,\n  ): Promise<MutationResponse> {\n    let transactionPhase: DatabaseTransactionPhase = 'open';\n\n    try {\n      const ret = await this.#dbProvider.transaction(\n        async (dbTx, transactionHooks) => {\n          // update the transaction phase to 'execute' after the transaction is opened\n          transactionPhase = 'execute';\n\n          await this.#checkAndIncrementLastMutationID(\n            transactionHooks,\n            mutation.clientID,\n            mutation.id,\n          );\n\n          if (appError === undefined) {\n            this.#lc.debug?.(\n              `Executing mutator '${mutation.name}' (id=${mutation.id})`,\n            );\n            await cb(dbTx, mutation.name, mutation.args[0]);\n          } else {\n            const mutationResult = makeAppErrorResponse(mutation, appError);\n            await transactionHooks.writeMutationResult(mutationResult);\n          }\n\n          return {\n            id: {\n              clientID: mutation.clientID,\n              id: mutation.id,\n            },\n            result: {},\n          };\n        },\n        this.#getTransactionInput(mutation),\n      );\n\n      return ret;\n    } catch (error) {\n      if (\n        isApplicationError(error) ||\n        error instanceof OutOfOrderMutation ||\n        error instanceof MutationAlreadyProcessedError\n      ) {\n        throw error;\n      }\n\n      throw new DatabaseTransactionError(transactionPhase, {cause: error});\n    }\n  }\n\n  #getTransactionInput(mutation: CustomMutation): TransactionProviderInput {\n    return {\n      upstreamSchema: this.#params.schema,\n      clientGroupID: this.#req.clientGroupID,\n      clientID: mutation.clientID,\n      mutationID: mutation.id,\n    };\n  }\n\n  async #checkAndIncrementLastMutationID(\n    transactionHooks: TransactionProviderHooks,\n    clientID: string,\n    receivedMutationID: number,\n  ) {\n    const {lastMutationID} = await transactionHooks.updateClientMutationID();\n\n    if (receivedMutationID < lastMutationID) {\n      throw new MutationAlreadyProcessedError(\n        clientID,\n        receivedMutationID,\n        lastMutationID,\n      );\n    } else if (receivedMutationID > lastMutationID) {\n      throw new OutOfOrderMutation(\n        clientID,\n        receivedMutationID,\n        lastMutationID,\n      );\n    }\n  }\n}\n\nexport class OutOfOrderMutation extends Error {\n  constructor(\n    clientID: string,\n    receivedMutationID: number,\n    lastMutationID: number | bigint,\n  ) {\n    super(\n      `Client ${clientID} sent mutation ID ${receivedMutationID} but expected ${lastMutationID}`,\n    );\n  }\n}\n\nfunction makeAppErrorResponse(\n  m: Mutation,\n  error: ApplicationError<ReadonlyJSONValue | undefined>,\n): MutationResponse {\n  return {\n    id: {\n      clientID: m.clientID,\n      id: m.id,\n    },\n    result: {\n      error: 'app',\n      message: error.message,\n      ...(error.details ? {details: error.details} : {}),\n    },\n  };\n}\n\n/** @deprecated Use getMutator instead */\nexport function getMutation(\n  // oxlint-disable-next-line no-explicit-any\n  mutators: AnyMutatorRegistry | CustomMutatorDefs<any>,\n  name: string,\n  // oxlint-disable-next-line no-explicit-any\n): CustomMutatorImpl<any> {\n  const path = name.split(/\\.|\\|/);\n  const mutator = getObjectAtPath(mutators, path);\n  assert(typeof mutator === 'function', `could not find mutator ${name}`);\n\n  if (isMutator(mutator)) {\n    // mutator needs to be called with {tx, args, ctx}\n    // CustomMutatorImpl is called with (tx, args, ctx)\n    return (tx, args, ctx) => mutator.fn({args, ctx, tx});\n  }\n\n  // oxlint-disable-next-line no-explicit-any\n  return mutator as CustomMutatorImpl<any>;\n}\n\nfunction getObjectAtPath(\n  obj: Record<string, unknown>,\n  path: string[],\n): unknown {\n  let current: unknown = obj;\n  for (const part of path) {\n    if (typeof current !== 'object' || current === null || !(part in current)) {\n      return undefined;\n    }\n    current = (current as Record<string, unknown>)[part];\n  }\n  return current;\n}\n\n/**\n * Processes internal cleanup mutation that deletes acknowledged mutation results\n * from the upstream database. This runs without LMID tracking since it's an\n * internal operation.\n */\nasync function processCleanupResultsMutation<\n  D extends Database<ExtractTransactionType<D>>,\n>(\n  dbProvider: D,\n  mutation: CustomMutation,\n  queryParams: Params,\n  lc: LogContext,\n): Promise<void> {\n  const parseResult = v.test(mutation.args[0], cleanupResultsArgSchema);\n  if (!parseResult.ok) {\n    lc.warn?.('Cleanup mutation has invalid args', parseResult.error);\n    return;\n  }\n  const args: CleanupResultsArg = parseResult.value;\n\n  // Determine clientID for transaction input based on cleanup type\n  // Note: legacy format without type field is treated as single\n  const clientID =\n    'type' in args && args.type === 'bulk' ? args.clientIDs[0] : args.clientID;\n\n  // Run in a transaction, using the hook for DB-specific operation.\n  // Note: only upstreamSchema is used by deleteMutationResults; the other\n  // fields are required by the interface but ignored for this operation.\n  await dbProvider.transaction(\n    async (_, hooks) => {\n      await hooks.deleteMutationResults(args);\n    },\n    {\n      upstreamSchema: queryParams.schema,\n      clientGroupID: args.clientGroupID,\n      clientID,\n      mutationID: 0,\n    },\n  );\n}\n\ntype DatabaseTransactionPhase = 'open' | 'execute';\nclass DatabaseTransactionError extends Error {\n  constructor(phase: DatabaseTransactionPhase, options?: ErrorOptions) {\n    super(\n      phase === 'open'\n        ? `Failed to open database transaction: ${getErrorMessage(options?.cause)}`\n        : `Database transaction failed after opening: ${getErrorMessage(options?.cause)}`,\n      options,\n    );\n    this.name = 'DatabaseTransactionError';\n  }\n}\n"],"names":["ErrorKind.PushFailed","ErrorOrigin.Server","ErrorReason.Parse","v.parse","ErrorReason.UnsupportedPushVersion","ErrorReason.OutOfOrderMutation","ErrorReason.Database","ErrorReason.Internal","v.test"],"mappings":";;;;;;;;;;;;AAmFA,MAAM,0BAA0B,OAAU,OAAqC;AAC7E,MAAI;AACF,WAAO,MAAM,GAAA;AAAA,EACf,SAAS,OAAO;AACd,QACE,iBAAiB,4BACjB,iBAAiB,sBACjB,iBAAiB,iCACjB,mBAAmB,KAAK,GACxB;AACA,YAAM;AAAA,IACR;AAEA,UAAM,yBAAyB,KAAK;AAAA,EACtC;AACF;AAKO,MAAM,wBAAwB;AA2BrC,eAAsB,oBAGpB,YACA,IAIA,sBACA,gBACA,UACuB;AAEvB,QAAM,oBAAoB,gCAAgC;AAE1D,MAAI;AACJ,MAAI;AACJ,MAAI;AAEJ,MAAI;AAEJ,MAAI,mBAAmB;AACrB,cAAU;AACV,UAAM,QAAS,kBAA2C;AAG1D,SAAK,iBAAiB,KAAK,EAAE,YAAY,eAAe;AAExD,UAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,kBAAc,IAAI;AAElB,QAAI;AACF,iBAAW,MAAM,QAAQ,KAAA;AAAA,IAC3B,SAAS,OAAO;AACd,SAAG,QAAQ,6BAA6B,KAAK;AAC7C,YAAM,UAAU,8BAA8B,gBAAgB,KAAK,CAAC;AACpE,YAAM,UAAU,gBAAgB,KAAK;AACrC,aAAO;AAAA,QACL,MAAMA;AAAAA,QACN,QAAQC;AAAAA,QACR,QAAQC;AAAAA,QACR;AAAA,QACA,aAAa,CAAA;AAAA,QACb,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,MAAC;AAAA,IAE/B;AAAA,EACF,OAAO;AACL,kBAAc;AACd,eAAW;AACX,UAAM,QAAQ,YAAY;AAC1B,SAAK,iBAAiB,KAAK,EAAE,YAAY,eAAe;AAAA,EAC1D;AAEA,MAAI,cAA4B,CAAA;AAEhC,MAAI;AACJ,MAAI;AACF,eAAWC,MAAQ,UAAU,cAAc;AAC3C,kBAAc,SAAS,UAAU,IAAI,CAAA,OAAM;AAAA,MACzC,IAAI,EAAE;AAAA,MACN,UAAU,EAAE;AAAA,IAAA,EACZ;AAAA,EACJ,SAAS,OAAO;AACd,OAAG,QAAQ,6BAA6B,KAAK;AAC7C,UAAM,UAAU,8BAA8B,gBAAgB,KAAK,CAAC;AACpE,UAAM,UAAU,gBAAgB,KAAK;AACrC,WAAO;AAAA,MACL,MAAMH;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQC;AAAAA,MACR;AAAA,MACA;AAAA,MACA,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,iBACJ,uBAAuB,kBACnB,OAAO,YAAY,WAAW,IAC9B;AACN,kBAAcC,MAAQ,gBAAgB,kBAAkB,aAAa;AAAA,EACvE,SAAS,OAAO;AACd,OAAG,QAAQ,yCAAyC,KAAK;AACzD,UAAM,UAAU,0CAA0C,gBAAgB,KAAK,CAAC;AAChF,UAAM,UAAU,gBAAgB,KAAK;AACrC,WAAO;AAAA,MACL,MAAMH;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQC;AAAAA,MACR;AAAA,MACA;AAAA,MACA,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AAEA,MAAI,SAAS,gBAAgB,GAAG;AAC9B,UAAM,WAAW;AAAA,MACf,MAAMF;AAAAA,MACN,QAAQC;AAAAA,MACR,QAAQG;AAAAA,MACR;AAAA,MACA,SAAS,6BAA6B,SAAS,WAAW;AAAA,IAAA;AAE5D,WAAO;AAAA,EACT;AAEA,QAAM,YAAgC,CAAA;AACtC,MAAI,iBAAiB;AAErB,MAAI;AACF,UAAM,aAAa,IAAI,WAAW,YAAY,UAAU,aAAa,EAAE;AASvE,eAAW,KAAK,SAAS,WAAW;AAElC,UAAI,EAAE,SAAS,YAAY,EAAE,SAAS,+BAA+B;AACnE,WAAG;AAAA,UACD,iCAAiC,EAAE,IAAI,eAAe,EAAE,QAAQ;AAAA,QAAA;AAElE,YAAI;AACF,gBAAM,8BAA8B,YAAY,GAAG,aAAa,EAAE;AAElE;AAAA,QACF,SAAS,OAAO;AACd,aAAG;AAAA,YACD,iDAAiD,EAAE,QAAQ;AAAA,YAC3D;AAAA,UAAA;AAGF;AAAA,QACF;AACA;AAAA,MACF;AAEA,aAAO,EAAE,SAAS,UAAU,0BAA0B;AACtD,SAAG;AAAA,QACD,wBAAwB,EAAE,IAAI,SAAS,EAAE,EAAE,cAAc,EAAE,QAAQ;AAAA,MAAA;AAGrE,UAAI,gBAA+B;AAEnC,YAAM,gBAA+B,OAAM,YAAW;AACpD,wBAAgB;AAChB,cAAM,SAAS,MAAM,WAAW,SAAS,GAAG,OAAO;AACnD,wBAAgB;AAChB,eAAO;AAAA,MACT;AAEA,UAAI;AACF,cAAM,MAAM,MAAM,wBAAwB,MAAM,GAAG,eAAe,CAAC,CAAC;AACpE,kBAAU,KAAK,GAAG;AAClB,WAAG,QAAQ,aAAa,EAAE,IAAI,SAAS,EAAE,EAAE,0BAA0B;AAErE;AAAA,MACF,SAAS,OAAO;AACd,YAAI,CAAC,mBAAmB,KAAK,GAAG;AAC9B,gBAAM;AAAA,QACR;AAEA,YAAI,kBAAkB,kBAAkB;AAEtC,gBAAM,WAAW,6BAA6B,GAAG,KAAK;AAAA,QACxD,WAAW,kBAAkB,cAAc;AAEzC,aAAG;AAAA,YACD,oDAAoD,EAAE,EAAE,eAAe,EAAE,QAAQ;AAAA,YACjF;AAAA,UAAA;AAAA,QAEJ;AAEA,WAAG;AAAA,UACD,yCAAyC,EAAE,EAAE,eAAe,EAAE,QAAQ;AAAA,UACtE;AAAA,QAAA;AAEF,kBAAU,KAAK,qBAAqB,GAAG,KAAK,CAAC;AAE7C;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,IAAA;AAAA,EAEf,SAAS,OAAO;AACd,OAAG,QAAQ,kCAAkC,KAAK;AAElD,UAAM,yBAAyB,YAAY,MAAM,cAAc;AAE/D,UAAM,UAAU,gBAAgB,KAAK;AACrC,UAAM,UAAU,gBAAgB,KAAK;AAErC,WAAO;AAAA,MACL,MAAMJ;AAAAA,MACN,QAAQC;AAAAA,MACR,QACE,iBAAiB,qBACbI,uBACA,iBAAiB,2BACfC,WACAC;AAAAA,MACR;AAAA,MACA,aAAa;AAAA,MACb,GAAI,UAAU,EAAC,YAAW,CAAA;AAAA,IAAC;AAAA,EAE/B;AACF;AAEA,MAAM,WAA0D;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,YAAe,KAAe,QAAgB,IAAgB;AACxE,SAAK,cAAc;AACnB,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,WAAW,OACT,UACA,OAC8B;AAC9B,QAAI,WAAyC;AAC7C,eAAS;AACP,UAAI;AACF,cAAM,MAAM,MAAM,KAAK,cAAc,UAAU,IAAI,QAAQ;AAC3D,YAAI,aAAa,QAAW;AAC1B,eAAK,IAAI;AAAA,YACP,YAAY,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,YACvD;AAAA,UAAA;AAEF,iBAAO,qBAAqB,UAAU,QAAQ;AAAA,QAChD;AAEA,eAAO;AAAA,MACT,SAAS,OAAO;AACd,YAAI,iBAAiB,oBAAoB;AACvC,eAAK,IAAI,QAAQ,KAAK;AACtB,gBAAM;AAAA,QACR;AAEA,YAAI,iBAAiB,+BAA+B;AAClD,eAAK,IAAI,OAAO,KAAK;AACrB,iBAAO;AAAA,YACL,IAAI;AAAA,cACF,UAAU,SAAS;AAAA,cACnB,IAAI,SAAS;AAAA,YAAA;AAAA,YAEf,QAAQ;AAAA,cACN,OAAO;AAAA,cACP,SAAS,MAAM;AAAA,YAAA;AAAA,UACjB;AAAA,QAEJ;AAEA,YAAI,aAAa,QAAW;AAE1B,eAAK,IAAI;AAAA,YACP,kCAAkC,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,YAC7E;AAAA,UAAA;AAEF,gBAAM;AAAA,QACR;AAGA,cAAM,gBACJ,iBAAiB,2BACZ,MAAM,SAAS,QAChB;AACN,mBAAW,yBAAyB,aAAa;AACjD,aAAK,IAAI;AAAA,UACP,6BAA6B,SAAS,EAAE,eAAe,SAAS,QAAQ;AAAA,UACxE;AAAA,QAAA;AAEF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,6BACJ,UACA,UAC2B;AAG3B,UAAM,MAAM,MAAM,KAAK;AAAA,MACrB;AAAA;AAAA,MAEA,MAAM;AAAA,MACN;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cACJ,UACA,IACA,UAC2B;AAC3B,QAAI,mBAA6C;AAEjD,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,YAAY;AAAA,QACjC,OAAO,MAAM,qBAAqB;AAEhC,6BAAmB;AAEnB,gBAAM,KAAK;AAAA,YACT;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,UAAA;AAGX,cAAI,aAAa,QAAW;AAC1B,iBAAK,IAAI;AAAA,cACP,sBAAsB,SAAS,IAAI,SAAS,SAAS,EAAE;AAAA,YAAA;AAEzD,kBAAM,GAAG,MAAM,SAAS,MAAM,SAAS,KAAK,CAAC,CAAC;AAAA,UAChD,OAAO;AACL,kBAAM,iBAAiB,qBAAqB,UAAU,QAAQ;AAC9D,kBAAM,iBAAiB,oBAAoB,cAAc;AAAA,UAC3D;AAEA,iBAAO;AAAA,YACL,IAAI;AAAA,cACF,UAAU,SAAS;AAAA,cACnB,IAAI,SAAS;AAAA,YAAA;AAAA,YAEf,QAAQ,CAAA;AAAA,UAAC;AAAA,QAEb;AAAA,QACA,KAAK,qBAAqB,QAAQ;AAAA,MAAA;AAGpC,aAAO;AAAA,IACT,SAAS,OAAO;AACd,UACE,mBAAmB,KAAK,KACxB,iBAAiB,sBACjB,iBAAiB,+BACjB;AACA,cAAM;AAAA,MACR;AAEA,YAAM,IAAI,yBAAyB,kBAAkB,EAAC,OAAO,OAAM;AAAA,IACrE;AAAA,EACF;AAAA,EAEA,qBAAqB,UAAoD;AACvE,WAAO;AAAA,MACL,gBAAgB,KAAK,QAAQ;AAAA,MAC7B,eAAe,KAAK,KAAK;AAAA,MACzB,UAAU,SAAS;AAAA,MACnB,YAAY,SAAS;AAAA,IAAA;AAAA,EAEzB;AAAA,EAEA,MAAM,iCACJ,kBACA,UACA,oBACA;AACA,UAAM,EAAC,eAAA,IAAkB,MAAM,iBAAiB,uBAAA;AAEhD,QAAI,qBAAqB,gBAAgB;AACvC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ,WAAW,qBAAqB,gBAAgB;AAC9C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AACF;AAEO,MAAM,2BAA2B,MAAM;AAAA,EAC5C,YACE,UACA,oBACA,gBACA;AACA;AAAA,MACE,UAAU,QAAQ,qBAAqB,kBAAkB,iBAAiB,cAAc;AAAA,IAAA;AAAA,EAE5F;AACF;AAEA,SAAS,qBACP,GACA,OACkB;AAClB,SAAO;AAAA,IACL,IAAI;AAAA,MACF,UAAU,EAAE;AAAA,MACZ,IAAI,EAAE;AAAA,IAAA;AAAA,IAER,QAAQ;AAAA,MACN,OAAO;AAAA,MACP,SAAS,MAAM;AAAA,MACf,GAAI,MAAM,UAAU,EAAC,SAAS,MAAM,QAAA,IAAW,CAAA;AAAA,IAAC;AAAA,EAClD;AAEJ;AAGO,SAAS,YAEd,UACA,MAEwB;AACxB,QAAM,OAAO,KAAK,MAAM,OAAO;AAC/B,QAAM,UAAU,gBAAgB,UAAU,IAAI;AAC9C,SAAO,OAAO,YAAY,YAAY,0BAA0B,IAAI,EAAE;AAEtE,MAAI,UAAU,OAAO,GAAG;AAGtB,WAAO,CAAC,IAAI,MAAM,QAAQ,QAAQ,GAAG,EAAC,MAAM,KAAK,IAAG;AAAA,EACtD;AAGA,SAAO;AACT;AAEA,SAAS,gBACP,KACA,MACS;AACT,MAAI,UAAmB;AACvB,aAAW,QAAQ,MAAM;AACvB,QAAI,OAAO,YAAY,YAAY,YAAY,QAAQ,EAAE,QAAQ,UAAU;AACzE,aAAO;AAAA,IACT;AACA,cAAW,QAAoC,IAAI;AAAA,EACrD;AACA,SAAO;AACT;AAOA,eAAe,8BAGb,YACA,UACA,aACA,IACe;AACf,QAAM,cAAcC,KAAO,SAAS,KAAK,CAAC,GAAG,uBAAuB;AACpE,MAAI,CAAC,YAAY,IAAI;AACnB,OAAG,OAAO,qCAAqC,YAAY,KAAK;AAChE;AAAA,EACF;AACA,QAAM,OAA0B,YAAY;AAI5C,QAAM,WACJ,UAAU,QAAQ,KAAK,SAAS,SAAS,KAAK,UAAU,CAAC,IAAI,KAAK;AAKpE,QAAM,WAAW;AAAA,IACf,OAAO,GAAG,UAAU;AAClB,YAAM,MAAM,sBAAsB,IAAI;AAAA,IACxC;AAAA,IACA;AAAA,MACE,gBAAgB,YAAY;AAAA,MAC5B,eAAe,KAAK;AAAA,MACpB;AAAA,MACA,YAAY;AAAA,IAAA;AAAA,EACd;AAEJ;AAGA,MAAM,iCAAiC,MAAM;AAAA,EAC3C,YAAY,OAAiC,SAAwB;AACnE;AAAA,MACE,UAAU,SACN,wCAAwC,gBAAgB,SAAS,KAAK,CAAC,KACvE,8CAA8C,gBAAgB,SAAS,KAAK,CAAC;AAAA,MACjF;AAAA,IAAA;AAEF,SAAK,OAAO;AAAA,EACd;AACF;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "0.26.1-canary.4",
+  "version": "0.26.1-canary.6",
   "description": "Zero is a web framework for serverless web development.",
   "author": "Rocicorp, Inc.",
   "repository": {