npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.11 → 1.10.0-canary.13 - Mend

@rocicorp/zero 1.10.0-canary.11 → 1.10.0-canary.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (53) hide show

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n\n // Drain the COPY stream from within a single read task so that the\n // TransactionPool worker holds the transaction for the entire duration of\n // the COPY, rather than incorrectly considering the worker \"idle\" and\n // attempting to send keepalives on it.\n //\n // Chunks are bridged out to this generator via a Queue, with a corresponding\n // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n // necessary because it is not possible to \"yield\" from within the read task.\n const chunks = new Queue<Buffer \| 'done'>();\n const acks = new Queue<void>();\n\n const copyDone = tx.processReadTask(async sql => {\n const readable = await sql.unsafe(copyCommand).readable();\n for await (const chunk of readable) {\n chunks.enqueue(chunk as Buffer);\n await acks.dequeue(); // wait for this generator to consume the chunk\n }\n chunks.enqueue('done');\n });\n copyDone.catch(e => chunks.enqueueRejection(e));\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for (;;) {\n const chunk = await chunks.dequeue();\n if (chunk === 'done') {\n break;\n }\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n\n // Signal the read task to pull the next chunk (one-chunk backpressure).\n acks.enqueue();\n }\n\n // Surface any error from the COPY read task (and confirm clean completion).\n await copyDone;\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported.dequeue();\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GAC8B;CACrD,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB,GAClE,iBAAiB,KACpB,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACqD;CAGrD,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAC/C,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACtD;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;EAAM;EACtD;EACA,WAAW;CACb;CAEA,MAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;CAAM;CACrD,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
1	+ {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n SchemaIncompatibilityError,\n type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n // The COPY is a single stream that must outlive the entire table download,\n // so allow a very long (24h) connection lifetime.\n ['max_lifetime']: 24 * 60 * 60,\n // A backfill COPY is a background process that can be preempted by\n // upstream replication changes, potentially requiring it to sit idle for\n // long stretches. Use TCP keepalive to detect dead connections instead\n // of the inactivity watchdog, which could kill connnections that are\n // taking a back seat for upstream replication changes.\n liveness: 'keepalive',\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n\n // Drain the COPY stream from within a single read task so that the\n // TransactionPool worker holds the transaction for the entire duration of\n // the COPY, rather than incorrectly considering the worker \"idle\" and\n // attempting to send keepalives on it.\n //\n // Chunks are bridged out to this generator via a Queue, with a corresponding\n // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n // necessary because it is not possible to \"yield\" from within the read task.\n const chunks = new Queue<Buffer \| 'done'>();\n const acks = new Queue<void>();\n\n const copyDone = tx.processReadTask(async sql => {\n const readable = await sql.unsafe(copyCommand).readable();\n for await (const chunk of readable) {\n chunks.enqueue(chunk as Buffer);\n await acks.dequeue(); // wait for this generator to consume the chunk\n }\n chunks.enqueue('done');\n });\n copyDone.catch(e => chunks.enqueueRejection(e));\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for (;;) {\n const chunk = await chunks.dequeue();\n if (chunk === 'done') {\n break;\n }\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {\n message: {tag: 'backfill', ...backfill, rowValues, status},\n byteSize: bufferedBytes,\n };\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n\n // Signal the read task to pull the next chunk (one-chunk backpressure).\n acks.enqueue();\n }\n\n // Surface any error from the COPY read task (and confirm clean completion).\n await copyDone;\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {\n message: {tag: 'backfill', ...backfill, rowValues, status},\n byteSize: bufferedBytes,\n };\n totalMsgs++;\n logFlushed();\n }\n\n yield {\n message: {tag: 'backfill-completed', ...backfill, status},\n byteSize: 0,\n };\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported.dequeue();\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAC/C,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IACJ,SAAS;KAAC,KAAK;KAAY,GAAG;KAAU;KAAW;IAAM;IACzD,UAAU;GACZ;GACA;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACzD,UAAU;EACZ;EACA;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GAAC,KAAK;GAAsB,GAAG;GAAU;EAAM;EACxD,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;~~IA0B3B~~;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;~~IA4EpC~~,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}
1	+ {"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IA+B3B;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;IAqFpC,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}

package/out/zero-cache/src/services/change-streamer/broadcast.js CHANGED Viewed

@@ -77,6 +77,7 @@ var Broadcast = class {
 	#setTimeout;
 	#clearTimeout;
 	#earlyReleaseTimer;
+	#needsBackupResponse = false;
 	/**
 	* Broadcasts the `change` to the `subscribers` and tracks their
 	* completion.
@@ -92,6 +93,7 @@ var Broadcast = class {
 		this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;
 		for (const sub of this.#pending) {
 			const changes = sub.numPending + 1;
+			if (sub.mode === "backup") this.#needsBackupResponse ||= !sub.isBacklogged();
 			sub.send(change).catch(() => {}).finally(() => this.#markCompleted(sub, changes));
 		}
 		if (this.#pending.size === 0) this.#setDone("all-subscribers");
@@ -100,6 +102,7 @@ var Broadcast = class {
 		if (this.#isDone) return;
 		const elapsed = performance.now() - this.#start;
 		sub.trackResponseResult("on-time");
+		if (sub.mode === "backup") this.#needsBackupResponse = false;
 		this.#completed.push({
 			sub,
 			changes,
@@ -110,7 +113,7 @@ var Broadcast = class {
 			this.#setDone("all-subscribers");
 			return;
 		}
-		if (this.#earlyReleaseTimeoutProportion !== void 0 && this.#earlyReleaseTimer === void 0 && this.#completed.length === this.#majority) {
+		if (this.#earlyReleaseTimeoutProportion !== void 0 && this.#earlyReleaseTimer === void 0 && this.#completed.length >= this.#majority && !this.#needsBackupResponse) {
 			const timeoutMs = Math.ceil(elapsed * this.#earlyReleaseTimeoutProportion);
 			this.#earlyReleaseTimer = this.#setTimeout(() => this.#setDone("consensus-timeout"), timeoutMs);
 		}

package/out/zero-cache/src/services/change-streamer/broadcast.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' \| 'consensus-timeout';\n\n/*\n Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n /\nexport type EarlyReleaseOptions = {\n /* Time to wait, proportional to majority completion, before releasing. /\n consensusTimeoutProportion: number;\n /* Injectable timer, for deterministic testing. /\n setTimeoutFn?: typeof setTimeout;\n /* Injectable timer cancellation, for deterministic testing. /\n clearTimeoutFn?: typeof clearTimeout;\n};\n\n/\n Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing consensus-based timeouts\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n \n ### Background\n \n The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n \n However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n * Having all subscribers wait for the new subscriber to catch up results\n * in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n * connection liveness checks kick in and disconnect the subscriber.\n \n A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n \n ### Consensus-based Timeout Algorithm\n \n To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n * case of a single node, or the case of one replication-manager\n * and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n * the configured timeout, proportional to the time it took for the\n * majority of subscribers to ack, even if not all subscribers have\n * finished.\n \n In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n \n In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n /\nexport class Broadcast {\n /\n Sends the change to the subscribers without the tracking machinery.\n * This is suitable for fire-and-forget (i.e. pipelined) sends.\n /\n static withoutTracking(\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n ) {\n for (const sub of subscribers) {\n void sub.send(change);\n }\n }\n\n readonly #lc: LogContext;\n readonly #pending: Set<Subscriber>;\n readonly #completed: Completed[];\n readonly #done = resolver();\n #isDone = false;\n #releaseMode: BroadcastReleaseMode \| undefined;\n\n readonly #watermark: string;\n readonly #majority: number;\n\n readonly #start = performance.now();\n\n // When set, the broadcast schedules its own consensus-timeout release instead\n // of waiting for the Forwarder's periodic progress tick.\n readonly #earlyReleaseTimeoutProportion: number \| undefined;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n #earlyReleaseTimer: ReturnType<typeof setTimeout> \| undefined;\n\n /\n Broadcasts the `change` to the `subscribers` and tracks their\n * completion.\n /\n constructor(\n lc: LogContext,\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n earlyRelease?: EarlyReleaseOptions,\n ) {\n this.#lc = lc;\n this.#pending = new Set(subscribers);\n this.#completed = [];\n this.#watermark = change[0];\n this.#majority = Math.floor(this.#pending.size / 2) + 1;\n this.#earlyReleaseTimeoutProportion =\n earlyRelease?.consensusTimeoutProportion;\n this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n for (const sub of this.#pending) {\n const changes = sub.numPending + 1; // add one for this `change`\n void sub\n .send(change)\n .catch(() => {})\n .finally(() => this.#markCompleted(sub, changes));\n }\n\n // set done if there are no subscribers (mainly for tests)\n if (this.#pending.size === 0) {\n this.#setDone('all-subscribers');\n }\n }\n\n #markCompleted(sub: Subscriber, changes: number) {\n if (this.#isDone) {\n return;\n }\n const elapsed = performance.now() - this.#start;\n sub.trackResponseResult('on-time');\n\n this.#completed.push({sub, changes, elapsed});\n this.#pending.delete(sub);\n if (this.#pending.size === 0) {\n this.#setDone('all-subscribers');\n return;\n }\n // Event-driven consensus-timeout: the moment a majority acks, arm a single\n // release timer proportional to how long the majority took to ack. The\n // broadcast resolves when it fires, unless all subscribers ack first.\n if (\n this.#earlyReleaseTimeoutProportion !== undefined &&\n this.#earlyReleaseTimer === undefined &&\n this.#completed.length === this.#majority\n ) {\n const timeoutMs = Math.ceil(\n elapsed this.#earlyReleaseTimeoutProportion,\n );\n this.#earlyReleaseTimer = this.#setTimeout(\n () => this.#setDone('consensus-timeout'),\n timeoutMs,\n );\n }\n }\n\n #setDone(releaseMode: BroadcastReleaseMode) {\n if (this.#isDone) {\n return;\n }\n if (releaseMode === 'consensus-timeout') {\n const elapsed = performance.now() - this.#start;\n this.#logWithState(\n `continuing with ${this.#pending.size} subscriber(s) still pending`,\n elapsed,\n );\n }\n this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n this.#isDone = true;\n this.#releaseMode = releaseMode;\n this.#clearTimeout(this.#earlyReleaseTimer);\n this.#earlyReleaseTimer = undefined;\n this.#done.resolve();\n }\n\n get isDone(): boolean {\n return this.#isDone;\n }\n\n get done(): Promise<void> {\n return this.#done.promise;\n }\n\n get releaseMode(): BroadcastReleaseMode {\n return this.#releaseMode ?? 'all-subscribers';\n }\n\n logSlowRequests(now: number) {\n if (this.#isDone \|\| this.#pending.size === 0) {\n return;\n }\n const elapsed = now - this.#start;\n if (this.#completed.length < this.#majority) {\n if (elapsed >= 1000) {\n this.#logWithState(\n `waiting for at least ${this.#majority} subscribers to finish`,\n elapsed,\n );\n }\n }\n }\n\n #logWithState(msg: string, elapsed: number) {\n this.#lc\n .withContext('watermark', this.#watermark)\n .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n completed: this.#completed.map(d => ({\n id: d.sub.id,\n processed: d.changes,\n elapsed: d.elapsed,\n })),\n pending: Array.from(this.#pending, sub => ({\n id: sub.id,\n ...sub.getStats(),\n })),\n });\n }\n}\n\n/** Tracks the completed result of a single subscriber. /\ntype Completed = {\n sub: Subscriber;\n /* The number of changes processed. /\n changes: number;\n /* The elapsed milliseconds. */\n elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8EA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;;;;;CAMA,YACE,IACA,aACA,QACA,cACA;EACA,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKS,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKT,SAAS,SAAS,GACzB,KAAKU,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;EACzC,IAAI,oBAAoB,SAAS;EAEjC,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKU,SAAS,iBAAiB;GAC/B;EACF;EAIA,IACE,KAAKJ,mCAAmC,KAAA,KACxC,KAAKM,uBAAuB,KAAA,KAC5B,KAAKX,WAAW,WAAW,KAAKG,WAChC;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKE,8BACjB;GACA,KAAKM,qBAAqB,KAAKL,kBACvB,KAAKG,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;GACzC,KAAKQ,cACH,mBAAmB,KAAKb,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKW,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKN,cAAc,KAAKI,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKV,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKS;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKT,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKY,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKX,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKS,cACH,wBAAwB,KAAKT,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF"}
1	+ {"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#needsBackupResponse","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' \| 'consensus-timeout';\n\n/*\n Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n /\nexport type EarlyReleaseOptions = {\n /* Time to wait, proportional to majority completion, before releasing. /\n consensusTimeoutProportion: number;\n /* Injectable timer, for deterministic testing. /\n setTimeoutFn?: typeof setTimeout;\n /* Injectable timer cancellation, for deterministic testing. /\n clearTimeoutFn?: typeof clearTimeout;\n};\n\n/\n Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing consensus-based timeouts\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n \n ### Background\n \n The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n \n However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n * Having all subscribers wait for the new subscriber to catch up results\n * in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n * connection liveness checks kick in and disconnect the subscriber.\n \n A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n \n ### Consensus-based Timeout Algorithm\n \n To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n * case of a single node, or the case of one replication-manager\n * and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n * the configured timeout, proportional to the time it took for the\n * majority of subscribers to ack, even if not all subscribers have\n * finished.\n \n In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n \n In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n /\nexport class Broadcast {\n /\n Sends the change to the subscribers without the tracking machinery.\n * This is suitable for fire-and-forget (i.e. pipelined) sends.\n /\n static withoutTracking(\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n ) {\n for (const sub of subscribers) {\n void sub.send(change);\n }\n }\n\n readonly #lc: LogContext;\n readonly #pending: Set<Subscriber>;\n readonly #completed: Completed[];\n readonly #done = resolver();\n #isDone = false;\n #releaseMode: BroadcastReleaseMode \| undefined;\n\n readonly #watermark: string;\n readonly #majority: number;\n\n readonly #start = performance.now();\n\n // When set, the broadcast schedules its own consensus-timeout release instead\n // of waiting for the Forwarder's periodic progress tick.\n readonly #earlyReleaseTimeoutProportion: number \| undefined;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n #earlyReleaseTimer: ReturnType<typeof setTimeout> \| undefined;\n\n // Tracks whether the backup-replicator responded. It is a required member\n // of the consensus timeout since it cannot be dropped as a laggard (since\n // that would shutdown the replication-manager itself).\n #needsBackupResponse = false;\n\n /\n Broadcasts the `change` to the `subscribers` and tracks their\n * completion.\n /\n constructor(\n lc: LogContext,\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n earlyRelease?: EarlyReleaseOptions,\n ) {\n this.#lc = lc;\n this.#pending = new Set(subscribers);\n this.#completed = [];\n this.#watermark = change[0];\n this.#majority = Math.floor(this.#pending.size / 2) + 1;\n this.#earlyReleaseTimeoutProportion =\n earlyRelease?.consensusTimeoutProportion;\n this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n for (const sub of this.#pending) {\n const changes = sub.numPending + 1; // add one for this `change`\n if (sub.mode === 'backup') {\n // Only gate consensus on the backup-replicator after it has finished\n // its initial catchup.\n this.#needsBackupResponse \|\|= !sub.isBacklogged();\n }\n void sub\n .send(change)\n .catch(() => {})\n .finally(() => this.#markCompleted(sub, changes));\n }\n\n // set done if there are no subscribers (mainly for tests)\n if (this.#pending.size === 0) {\n this.#setDone('all-subscribers');\n }\n }\n\n #markCompleted(sub: Subscriber, changes: number) {\n if (this.#isDone) {\n return;\n }\n const elapsed = performance.now() - this.#start;\n sub.trackResponseResult('on-time');\n if (sub.mode === 'backup') {\n this.#needsBackupResponse = false;\n }\n this.#completed.push({sub, changes, elapsed});\n this.#pending.delete(sub);\n if (this.#pending.size === 0) {\n this.#setDone('all-subscribers');\n return;\n }\n // Event-driven consensus-timeout: the moment a majority acks, and it\n // includes the backup-replicator, arm a single release timer\n // proportional to how long the majority took to ack. The broadcast\n // resolves when it fires, unless all subscribers ack first.\n if (\n this.#earlyReleaseTimeoutProportion !== undefined &&\n this.#earlyReleaseTimer === undefined &&\n this.#completed.length >= this.#majority &&\n !this.#needsBackupResponse\n ) {\n const timeoutMs = Math.ceil(\n elapsed this.#earlyReleaseTimeoutProportion,\n );\n this.#earlyReleaseTimer = this.#setTimeout(\n () => this.#setDone('consensus-timeout'),\n timeoutMs,\n );\n }\n }\n\n #setDone(releaseMode: BroadcastReleaseMode) {\n if (this.#isDone) {\n return;\n }\n if (releaseMode === 'consensus-timeout') {\n const elapsed = performance.now() - this.#start;\n this.#logWithState(\n `continuing with ${this.#pending.size} subscriber(s) still pending`,\n elapsed,\n );\n }\n this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n this.#isDone = true;\n this.#releaseMode = releaseMode;\n this.#clearTimeout(this.#earlyReleaseTimer);\n this.#earlyReleaseTimer = undefined;\n this.#done.resolve();\n }\n\n get isDone(): boolean {\n return this.#isDone;\n }\n\n get done(): Promise<void> {\n return this.#done.promise;\n }\n\n get releaseMode(): BroadcastReleaseMode {\n return this.#releaseMode ?? 'all-subscribers';\n }\n\n logSlowRequests(now: number) {\n if (this.#isDone \|\| this.#pending.size === 0) {\n return;\n }\n const elapsed = now - this.#start;\n if (this.#completed.length < this.#majority) {\n if (elapsed >= 1000) {\n this.#logWithState(\n `waiting for at least ${this.#majority} subscribers to finish`,\n elapsed,\n );\n }\n }\n }\n\n #logWithState(msg: string, elapsed: number) {\n this.#lc\n .withContext('watermark', this.#watermark)\n .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n completed: this.#completed.map(d => ({\n id: d.sub.id,\n processed: d.changes,\n elapsed: d.elapsed,\n })),\n pending: Array.from(this.#pending, sub => ({\n id: sub.id,\n ...sub.getStats(),\n })),\n });\n }\n}\n\n/** Tracks the completed result of a single subscriber. /\ntype Completed = {\n sub: Subscriber;\n /* The number of changes processed. /\n changes: number;\n /* The elapsed milliseconds. */\n elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8EA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IAAI,IAAI,SAAS,UAGf,KAAKS,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKC,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKV,SAAS,SAAS,GACzB,KAAKW,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAKI,uBAAuB;EAE9B,KAAKR,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKW,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAKL,mCAAmC,KAAA,KACxC,KAAKO,uBAAuB,KAAA,KAC5B,KAAKZ,WAAW,UAAU,KAAKG,aAC/B,CAAC,KAAKK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKH,8BACjB;GACA,KAAKO,qBAAqB,KAAKN,kBACvB,KAAKI,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;GACzC,KAAKS,cACH,mBAAmB,KAAKd,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKY,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKP,cAAc,KAAKK,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKX,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKU;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKV,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKa,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKZ,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKU,cACH,wBAAwB,KAAKV,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js CHANGED Viewed

@@ -421,7 +421,7 @@ var ChangeStreamerImpl = class {
 		if (mode === "serving") this.#serving.resolve();
 		let cleanupSubscriber = () => {};
 		const downstream = Subscription.create({ cleanup: () => cleanupSubscriber() });
-		const subscriber = new Subscriber(protocolVersion, id, watermark, downstream, () => this.#latestStatus, {});
+		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, () => this.#latestStatus, {});
 		const lc = this.#lc.withContext("subscriber", subscriber.id);
 		const removeFromForwarder = () => {
 			lc.info?.(`removing subscriber ${subscriber.id}`);