npm - @rocicorp/zero - Versions diffs - 1.6.0-canary.0 → 1.6.0-canary.10 - Mend

@rocicorp/zero 1.6.0-canary.0 → 1.6.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1777) hide show

package/out/packages/zero-cache/src/services/analyze.js ADDED Viewed

@@ -0,0 +1,72 @@
+import { MemoryStorage } from "../../../zql/src/ivm/memory-storage.js";
+import { AccumulatorDebugger, serializePlanDebugEvents } from "../../../zql/src/planner/planner-debug.js";
+import { Debug } from "../../../zql/src/builder/debug-delegate.js";
+import { Database } from "../../../zqlite/src/db.js";
+import { TableSource } from "../../../zqlite/src/table-source.js";
+import { computeZqlSpecs, mustGetTableSpec } from "../db/lite-tables.js";
+import { createSQLiteCostModel } from "../../../zqlite/src/sqlite-cost-model.js";
+import { runAst } from "./run-ast.js";
+import { explainQueries } from "../../../zqlite/src/explain-queries.js";
+import { _usingCtx } from "../../../../_virtual/_@oxc-project_runtime@0.130.0/helpers/usingCtx.js";
+import { TimeSliceTimer } from "./view-syncer/view-syncer.js";
+//#region ../zero-cache/src/services/analyze.ts
+var TIME_SLICE_LAP_THRESHOLD_MS = 200;
+async function analyzeQuery(lc, config, clientSchema, ast, syncedRows = true, vendedRows = false, permissions, auth, joinPlans = false) {
+	try {
+		var _usingCtx$1 = _usingCtx();
+		const db = _usingCtx$1.u(new Database(lc, config.replica.file));
+		const fullTables = /* @__PURE__ */ new Map();
+		const tableSpecs = /* @__PURE__ */ new Map();
+		const tables = /* @__PURE__ */ new Map();
+		computeZqlSpecs(lc, db, { includeBackfillingColumns: false }, tableSpecs, fullTables);
+		const costModel = config.enableQueryPlanner ? createSQLiteCostModel(db, tableSpecs) : void 0;
+		const planDebugger = joinPlans ? new AccumulatorDebugger() : void 0;
+		const timer = await new TimeSliceTimer(lc).start();
+		const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;
+		const yieldProcess = () => timer.yieldProcess();
+		const result = await runAst(lc, clientSchema, ast, true, {
+			applyPermissions: permissions !== void 0,
+			syncedRows,
+			vendedRows,
+			auth,
+			db,
+			tableSpecs,
+			permissions,
+			costModel,
+			planDebugger,
+			host: {
+				debug: new Debug(),
+				getSource(tableName) {
+					let source = tables.get(tableName);
+					if (source) return source;
+					const tableSpec = mustGetTableSpec(tableSpecs, tableName);
+					const { primaryKey } = tableSpec.tableSpec;
+					source = new TableSource(lc, config.log, db, tableName, tableSpec.zqlSpec, primaryKey, shouldYield);
+					tables.set(tableName, source);
+					return source;
+				},
+				createStorage() {
+					return new MemoryStorage();
+				},
+				decorateSourceInput: (input) => input,
+				decorateInput: (input) => input,
+				addEdge() {},
+				decorateFilterInput: (input) => input
+			}
+		}, yieldProcess);
+		const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);
+		const captured = result.sqlitePlans ?? {};
+		for (const [query, plan] of Object.entries(fallback)) if (!captured[query]) captured[query] = plan;
+		result.sqlitePlans = captured;
+		if (planDebugger) result.joinPlans = serializePlanDebugEvents(planDebugger.events);
+		return result;
+	} catch (_) {
+		_usingCtx$1.e = _;
+	} finally {
+		_usingCtx$1.d();
+	}
+}
+//#endregion
+export { analyzeQuery };
+//# sourceMappingURL=analyze.js.map

package/out/packages/zero-cache/src/services/analyze.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"analyze.js","names":[],"sources":["../../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n AccumulatorDebugger,\n serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n lc: LogContext,\n config: NormalizedZeroConfig,\n clientSchema: ClientSchema,\n ast: AST,\n syncedRows = true,\n vendedRows = false,\n permissions?: PermissionsConfig,\n auth?: JWTAuth,\n joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n using db = new Database(lc, config.replica.file);\n const fullTables = new Map<string, LiteTableSpec>();\n const tableSpecs = new Map<string, LiteAndZqlSpec>();\n const tables = new Map<string, TableSource>();\n\n computeZqlSpecs(\n lc,\n db,\n {includeBackfillingColumns: false},\n tableSpecs,\n fullTables,\n );\n\n // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n // on the server, so the analysis reflects what actually executes. Diagnostic\n // event collection is orthogonal and opt-in via `joinPlans`.\n const costModel = config.enableQueryPlanner\n ? createSQLiteCostModel(db, tableSpecs)\n : undefined;\n const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n const timer = await new TimeSliceTimer(lc).start();\n const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n const yieldProcess = () => timer.yieldProcess();\n const result = await runAst(\n lc,\n clientSchema,\n ast,\n true,\n {\n applyPermissions: permissions !== undefined,\n syncedRows,\n vendedRows,\n auth,\n db,\n tableSpecs,\n permissions,\n costModel,\n planDebugger,\n host: {\n debug: new Debug(),\n getSource(tableName: string) {\n let source = tables.get(tableName);\n if (source) {\n return source;\n }\n\n const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n const {primaryKey} = tableSpec.tableSpec;\n\n source = new TableSource(\n lc,\n config.log,\n db,\n tableName,\n tableSpec.zqlSpec,\n primaryKey,\n shouldYield,\n );\n tables.set(tableName, source);\n return source;\n },\n createStorage() {\n return new MemoryStorage();\n },\n decorateSourceInput: input => input,\n decorateInput: input => input,\n addEdge() {},\n decorateFilterInput: input => input,\n },\n },\n yieldProcess,\n );\n\n // Fill in plans for any queries SQLite did not actually execute (and thus\n // did not populate scanStatus EXPLAIN for) using the substituted-binding\n // fallback. Plans captured at execution time use the real bindings and win.\n const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n const captured = result.sqlitePlans ?? {};\n for (const [query, plan] of Object.entries(fallback)) {\n if (!captured[query]) {\n captured[query] = plan;\n }\n }\n result.sqlitePlans = captured;\n\n if (planDebugger) {\n result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n }\n\n return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}

package/out/packages/zero-cache/src/services/change-source/common/backfill-manager.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n stateVersionFromString,\n stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n ChangeStreamMessage,\n Identifier,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n Cancelable,\n ChangeStreamMultiplexer,\n Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill | BackfillCompleted>;\n\ntype RunningBackfillState = {\n request: BackfillRequest;\n canceledReason?: string | undefined;\n minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n watermark: string;\n reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n readonly #lc: LogContext;\n\n /**\n * Tracks the metadata of required backfills based on schema changes\n * and initial backfill requests.\n */\n readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n tableKey,\n );\n readonly #changeStreamer: ChangeStreamMultiplexer;\n readonly #backfillStreamer: BackfillStreamer;\n readonly #jsonFormat: JSONFormat;\n\n /**\n * The current running backfill. The backfill request is always also in\n * `#requiredBackfills` (technically, it can be a subset of what's in\n * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n * upon completion.\n */\n #runningBackfill: RunningBackfillState | null = null;\n\n /** The last seen watermark in the change stream. */\n #lastStatusWatermark: string | null = null;\n\n readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n /** The watermark of the current transaction in the change stream. */\n #currentTxWatermark: string | null = null;\n\n constructor(\n lc: LogContext,\n changeStreamer: ChangeStreamMultiplexer,\n backfillStreamer: BackfillStreamer,\n jsonFormat: JSONFormat = JSON_STRINGIFIED,\n minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n ) {\n this.#lc = lc.withContext('component', 'backfill-manager');\n this.#changeStreamer = changeStreamer;\n this.#backfillStreamer = backfillStreamer;\n this.#jsonFormat = jsonFormat;\n this.#minBackoffMs = minBackoffMs;\n this.#maxBackoffMs = maxBackoffMs;\n this.#retryDelayMs = minBackoffMs;\n }\n\n run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n this.#lc.info?.(\n `starting backfill manager with ${initialRequests.length} initial requests`,\n {requests: initialRequests},\n );\n this.#lastStatusWatermark = lastWatermark;\n initialRequests.forEach(req =>\n this.#setRequiredBackfill('initial-request', req),\n );\n this.#checkAndStartBackfill();\n }\n\n #setLastStatusWatermark({watermark}: {watermark: string}) {\n // Only allow the watermark to move forward. This prevents a backfill\n // transaction (whose watermark is unrelated to change-stream state)\n // from moving the watermark backwards.\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n this.#lastStatusWatermark = watermark;\n for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n const awaiting = this.#awaitingStatusWatermarks[i];\n if (watermark >= awaiting.watermark) {\n awaiting.reached();\n this.#awaitingStatusWatermarks.splice(i, 1);\n }\n }\n }\n }\n\n #changeStreamReached(\n lc: LogContext,\n watermark: string,\n ): Promise<void> | null {\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n const {promise, resolve: reached} = resolver();\n this.#awaitingStatusWatermarks.push({watermark, reached});\n lc.info?.(\n `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n );\n return promise;\n }\n return null;\n }\n\n readonly #minBackoffMs: number;\n readonly #maxBackoffMs: number;\n #retryDelayMs: number;\n #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n #checkAndStartBackfill() {\n if (\n !this.#backfillRetryTimer &&\n !this.#runningBackfill &&\n this.#requiredBackfills.size\n ) {\n // Pick a random backfill to avoid head-of-line blocking by a\n // problematic backfill (e.g. awaiting a primary key). This is\n // simpler that adding logic to classify (and declassify)\n // problematic backfills.\n const candidates = [...this.#requiredBackfills.values()];\n const request = candidates[randInt(0, candidates.length - 1)];\n const state = {request, minWatermark: ''};\n const lc = this.#lc.withContext('table', request.table.name);\n\n this.#runningBackfill = state;\n void this.#runBackfill(lc, state)\n .then(() => {\n this.#stopRunningBackfill('backfill exited', state);\n this.#retryDelayMs = this.#minBackoffMs; // reset on success\n })\n // For unexpected errors (e.g. upstream replication slot\n // unavailability), retry with exponential backoff.\n .catch(e => {\n this.#stopRunningBackfill(String(e), state);\n this.#retryBackfillWithBackoff(e);\n });\n }\n }\n\n #retryBackfillWithBackoff(e: unknown) {\n const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n this.#lc[log]?.(\n `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n e,\n );\n this.#backfillRetryTimer = setTimeout(() => {\n this.#backfillRetryTimer = undefined;\n this.#checkAndStartBackfill();\n }, this.#retryDelayMs);\n\n this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n }\n\n async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n const changeStream = this.#changeStreamer; // Purely for readability\n\n // backfillTx is set if and only if a changeStreamer reservation has been\n // acquired and the backfill stream is inside a transaction.\n let backfillTx: string | null = null;\n\n /**\n * @returns the new tx watermark, or null if backfill was cancelled\n */\n const beginTxFor = async (\n msg: MessageBackfill | BackfillCompleted,\n ): Promise<string | null> => {\n assert(backfillTx === null, 'Expected no active backfill transaction');\n const lastWatermark = await changeStream.reserve('backfill');\n\n // After obtaining the changeStream reservation, check if the stream\n // had changes that resulted in invalidating / canceling this backfill.\n if (\n state.canceledReason ||\n (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n ) {\n if (state.canceledReason === undefined) {\n assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n this.#stopRunningBackfill(\n `row key change at ${state.minWatermark} ` +\n `postdates backfill watermark at ${msg.watermark}`,\n state,\n );\n }\n changeStream.release(lastWatermark);\n return null;\n }\n\n const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n let tx = stateVersionToString({\n major,\n minor: BigInt(minor) + 1n,\n });\n\n if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n // At this point it must be the case that the #changeStreamReached() the\n // backfill watermark. Given that guarantee, ensure that the version of the\n // transaction containing the backfill-completed message is at least up\n // to the backfill watermark, so that the final database state version is\n // never earlier than the version of any backfilled rows.\n tx = msg.watermark;\n }\n\n void changeStream.push([\n 'begin',\n {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n {commitWatermark: tx},\n ]);\n return (backfillTx = tx);\n };\n\n const commitTx = () => {\n if (backfillTx) {\n void changeStream.push([\n 'commit',\n {tag: 'commit'},\n {watermark: backfillTx},\n ]);\n changeStream.release(backfillTx);\n }\n backfillTx = null;\n };\n\n for await (const msg of this.#backfillStreamer(state.request)) {\n // Before sending `backfill-completed`, the main replication stream\n // may need to catch up, and/or the current transaction may need to be\n // committed to open a new transaction that's up to backfill watermark.\n const mustWaitBeforeFlush =\n msg.tag === 'backfill-completed' &&\n (this.#changeStreamReached(lc, msg.watermark) ||\n (backfillTx !== null && backfillTx < msg.watermark));\n\n // If necessary, yield the reservation to the main stream.\n if (\n backfillTx &&\n (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)\n ) {\n commitTx();\n }\n\n mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n if (\n msg.tag === 'backfill' &&\n msg.rowValues.length > 0 &&\n msg.relation.rowKey.columns.length === 0\n ) {\n throw new MissingRowKeyError(state.request);\n }\n\n // Reserve the changeStreamer if not in a transaction.\n if ((backfillTx ??= await beginTxFor(msg)) === null) {\n lc.info?.(\n `backfill stream canceled: ${state.canceledReason}`,\n state.request,\n );\n this.#checkAndStartBackfill(); // start the next backfill if present\n return; // this backfill is canceled\n }\n\n // `await` to allow the change streamer to exert back pressure\n // on backfills.\n await changeStream.push(['data', msg]);\n }\n\n // Flush any final tx and release the stream.\n backfillTx && commitTx();\n lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n }\n\n #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n const state = this.#runningBackfill;\n return state?.request.table.schema === table.schema &&\n state.request.table.name === table.name\n ? state\n : null;\n }\n\n /**\n * Stops the running backfill for the specified `reason`. If `instance` is\n * specified, the running backfill is stopped only if it is that instance.\n * This allows the running backfill itself to clear backfill state without\n * accidentally stopping a different (e.g. subsequent) backfill.\n */\n #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n const backfill = this.#runningBackfill;\n if (backfill && backfill === (instance ?? backfill)) {\n backfill.canceledReason = reason;\n this.#runningBackfill = null;\n reason && this.#lc.info?.(`canceling backfill:`, reason);\n }\n }\n\n #setRequiredBackfill(source: string, req: BackfillRequest) {\n const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.set(req.table, req);\n }\n\n #deleteRequiredBackfill(source: string, id: Identifier) {\n const req = this.#requiredBackfills.get(id);\n if (req) {\n const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.delete(id);\n }\n }\n\n /**\n * Implements {@link Listener.onChange()}, invoked by the\n * {@link ChangeStreamMultiplexer}.\n */\n onChange(message: ChangeStreamMessage): void {\n if (message[0] === 'begin') {\n this.#currentTxWatermark = message[2].commitWatermark;\n return;\n }\n if (message[0] === 'commit') {\n this.#currentTxWatermark = null;\n this.#setLastStatusWatermark(message[2]);\n // Every commit is a candidate for starting the next backfill\n // (if one is not currently running).\n this.#checkAndStartBackfill();\n return;\n }\n if (message[0] === 'status') {\n this.#setLastStatusWatermark(message[2]);\n return;\n }\n if (message[0] !== 'data') {\n return;\n }\n const change = message[1];\n const {tag} = change;\n switch (tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest) {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n });\n if (this.#backfillRunningFor(table)) {\n this.#stopRunningBackfill(`TableMetadata updated`);\n }\n }\n break;\n }\n case 'create-table': {\n const {\n spec: {schema, name},\n metadata = null,\n backfill,\n } = change;\n\n if (backfill) {\n this.#setRequiredBackfill(tag, {\n table: {schema, name, metadata},\n columns: backfill,\n });\n }\n break;\n }\n case 'rename-table': {\n const {old, new: newTable} = change;\n const backfillRequest = this.#requiredBackfills.get(old);\n if (backfillRequest) {\n const {schema, name} = newTable;\n this.#deleteRequiredBackfill(tag, old);\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, schema, name},\n });\n if (this.#backfillRunningFor(old)) {\n this.#stopRunningBackfill(`table renamed`);\n }\n }\n break;\n }\n case 'drop-table': {\n const {id} = change;\n const backfillRequest = this.#requiredBackfills.get(id);\n if (backfillRequest) {\n this.#deleteRequiredBackfill(tag, id);\n if (this.#backfillRunningFor(id)) {\n this.#stopRunningBackfill(`table dropped`);\n }\n }\n break;\n }\n case 'add-column': {\n const {\n table,\n tableMetadata: metadata = null,\n column,\n backfill,\n } = change;\n if (backfill) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (!backfillRequest) {\n this.#setRequiredBackfill(tag, {\n table: {...table, metadata},\n columns: {[column.name]: backfill},\n });\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n columns: {\n ...backfillRequest.columns,\n [column.name]: backfill,\n },\n });\n // Note: The running backfill need not be canceled if a\n // new column is added. The new column will be backfilled\n // by its own stream after the current backfill completes.\n }\n }\n break;\n }\n case 'update-column': {\n const {\n table,\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && oldName in backfillRequest.columns) {\n const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: {...otherCols, [newName]: colSpec},\n });\n const backfill = this.#backfillRunningFor(table);\n if (backfill && oldName in backfill.request.columns) {\n this.#stopRunningBackfill(`column renamed`);\n }\n }\n }\n break;\n }\n case 'drop-column': {\n const {table, column} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && column in backfillRequest.columns) {\n const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n if (Object.keys(remaining).length === 0) {\n this.#deleteRequiredBackfill(tag, table);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: remaining,\n });\n }\n const backfill = this.#backfillRunningFor(table);\n if (backfill && column in backfill.request.columns) {\n this.#stopRunningBackfill(`column dropped`);\n }\n }\n break;\n }\n case 'update': {\n const {relation, key, new: row} = change;\n const backfill = this.#backfillRunningFor(relation);\n const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n if (backfill?.request.table.metadata && key !== null) {\n // A corner case that backfill is unable to correctly handle is\n // when a row's key changes; this is decomposed into a delete\n // of the old key and a set of the new key in the replica change\n // log, at which point the backfill algorithm assumes that the\n // (old) row is deleted but does not know to backfill the new row.\n // In these corner cases, the current backfill is canceled and\n // retried if its version precedes this update.\n for (const col of Object.keys(\n backfill.request.table.metadata.rowKey,\n )) {\n if (key[col] !== row[col]) {\n backfill.minWatermark = txWatermark;\n this.#lc.info?.(\n `key for row as changed (col: ${col}). ` +\n `backfill data must not predate ${backfill.minWatermark}`,\n );\n break;\n }\n }\n }\n break;\n }\n case 'backfill-completed': {\n const {relation, columns} = change;\n const backfillRequest = this.#requiredBackfills.get(relation);\n assert(\n backfillRequest,\n () => `No BackfillRequest completed backfill ${stringify(change)}`,\n );\n const remaining = Object.entries(backfillRequest.columns).filter(\n ([col]) =>\n !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n );\n if (remaining.length === 0) {\n this.#deleteRequiredBackfill(tag, relation);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: Object.fromEntries(remaining),\n });\n }\n // Technically the backfill is already stopping, but this method\n // cleans up the state that tracks it.\n this.#stopRunningBackfill();\n break;\n }\n }\n }\n\n cancel(): void {\n this.#stopRunningBackfill(`change stream canceled`);\n clearTimeout(this.#backfillRetryTimer);\n }\n}\n\nabstract class BackfillStreamError extends Error {\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(\n `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n {cause},\n );\n }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n readonly name = 'MissingRowKeyError';\n\n constructor(bf: BackfillRequest, cause?: unknown) {\n super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n readonly name = 'SchemaIncompatibilityError';\n\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(bf, msg, cause);\n }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;AAYA,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKE,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;CACvB;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKR,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKS,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKJ,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKI,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKJ,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKI,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,uBACN,CAAC,KAAKC,oBACN,KAAKZ,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKa,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,MAAM,MAAM,KAAKR,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKP,IAAI,OACP,uCAAuC,KAAKQ,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKZ;EAI1B,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKa,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKX;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;EACf;EAEA,WAAW,MAAM,OAAO,KAAKD,kBAAkB,MAAM,OAAO,GAAG;GAI7D,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKc,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAG7C,IACE,eACC,aAAa,YAAY,IAAI,KAAK,sBAEnC,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKN,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;EACvC;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKE;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKb,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKiB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKR,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKQ,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKlB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKS,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKU,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKT,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKoB,wBAAwB,KAAK,GAAG;KACrC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKU,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKd,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKoB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKd,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKS,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKT,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKS,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKU,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKd,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKoB,wBAAwB,KAAK,KAAK;UAEvC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKU,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKlB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKoB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKX,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKA,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;CACvC;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/packages/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#lastWatermark"],"sources":["../../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n ChangeStreamMessage,\n DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n cancel(): void;\n};\n\nexport type Listener = {\n onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n producer: string;\n startTime: number;\n grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n readonly #lc: LogContext;\n readonly #sub: Subscription<ChangeStreamMessage>;\n readonly #producers: Cancelable[] = [];\n readonly #listeners: Listener[] = [];\n\n /**\n * The `#lastWatermark` tracks the watermark of the last transaction\n * committed to stream. It is set to null when a producer has reserved\n * the stream, and set to the new watermark when the producer releases\n * the reservation.\n */\n #lastWatermark: string | null;\n\n /**\n * Tracks the queue of producers waiting for a reservation.\n */\n readonly #waiters: Waiter[] = [];\n\n constructor(lc: LogContext, lastWatermark: string) {\n this.#lc = lc;\n this.#sub = Subscription.create<ChangeStreamMessage>({\n cleanup: () => this.#producers.forEach(p => p.cancel()),\n });\n this.#lastWatermark = lastWatermark;\n }\n\n addProducers(...p: Cancelable[]): this {\n this.#producers.push(...p);\n return this;\n }\n\n addListeners(...l: Listener[]): this {\n this.#listeners.push(...l);\n return this;\n }\n\n /**\n * Called by a producer to \"reserve\" the exclusive right to push data\n * changes to the stream. If the stream is currently reserved,\n * the resulting Promise must be awaited.\n *\n * The producer must then {@link release()} the reservation when it sees\n * fit.\n *\n * @param producer The name of the producer, purely for debugging output\n */\n reserve(producer: string): string | Promise<string> {\n if (this.#lastWatermark !== null) {\n // If the stream is not reserved, reserve it and return the\n // watermark.\n const lastWatermark = this.#lastWatermark;\n this.#lastWatermark = null;\n return lastWatermark;\n }\n\n // Otherwise, wait for the current reservation to be released.\n const startTime = performance.now();\n const {promise, resolve: grantReservation} = resolver<string>();\n this.#waiters.push({producer, startTime, grantReservation});\n\n return promise;\n }\n\n /**\n * If there are producers currently awaiting a reservation, returns\n * the duration (in milliseconds) of the oldest reservation request.\n * Returns a negative number if there are no waiters.\n */\n waiterDelay(): number {\n if (this.#waiters.length === 0) {\n return -1;\n }\n return performance.now() - this.#waiters[0].startTime;\n }\n\n /**\n * Called by a producer to release its reservation after committing its\n * last transaction.\n */\n release(newWatermark: string) {\n const waiter = this.#waiters.shift();\n if (!waiter) {\n this.#lastWatermark = newWatermark;\n } else {\n const {producer, startTime, grantReservation} = waiter;\n grantReservation(newWatermark);\n const elapsed = performance.now() - startTime;\n this.#lc.info?.(\n `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n );\n }\n }\n\n /**\n * `pushStatus()` can be called without a reservation, as it\n * does not constitute a data change and can appear anywhere\n * in the stream.\n */\n pushStatus(message: DownstreamStatusMessage) {\n // Let listeners know about all status messages.\n this.#listeners.forEach(l => l.onChange(message));\n // The ChangeStreamer only cares about status messages requiring an ack\n // or containing a lagReport. To reduce churn, avoid sending other status\n // messages.\n if (message[1].ack || message[1].lagReport) {\n this.#sub.push(message);\n }\n }\n\n /**\n * `push()` must only be called by a producer after it has\n * {@link reserve}d the stream.\n */\n push(message: ChangeStreamMessage): Promise<unknown> {\n assert(\n this.#lastWatermark === null,\n `push() called without reserve()-ing the stream`,\n );\n this.#listeners.forEach(l => l.onChange(message));\n return this.#sub.push(message).result;\n }\n\n fail(err: Error) {\n this.#sub.fail(err);\n }\n\n asSource(): Source<ChangeStreamMessage> {\n return this.#sub;\n }\n}\n"],"mappings":";;;;;;;;AA4BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKC,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC,EACxD,CAAC;EACD,KAAKG,iBAAiB;CACxB;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKH,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKE,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EAAC,SAAS,SAAS,qBAAoB,SAAiB;EAC9D,KAAKD,SAAS,KAAK;GAAC;GAAU;GAAW;EAAgB,CAAC;EAE1D,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKC,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKL,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKI,mBAAmB,MACxB,gDACF;EACA,KAAKF,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}

package/out/packages/zero-cache/src/services/change-source/common/replica-schema.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../db/migration-lite.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n CREATE_RUNTIME_EVENTS_TABLE,\n recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n const setupMigration: Migration = {\n migrateSchema: (log, tx) => initialSync(log, tx),\n minSafeVersion: 1,\n };\n\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n setupMigration,\n schemaVersionMigrationMap,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n throw new AutoResetSignal(e.message);\n }\n throw e;\n }\n}\n\nexport async function upgradeReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n) {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n // setupMigration should never be invoked\n {\n migrateSchema: () => {\n throw new Error(\n 'This should only be called for already synced replicas',\n );\n },\n },\n schemaVersionMigrationMap,\n );\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n CREATE TABLE \"_zero.changeLog2\" (\n \"stateVersion\" TEXT NOT NULL,\n \"pos\" INT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"rowKey\" TEXT NOT NULL,\n \"op\" TEXT NOT NULL,\n PRIMARY KEY(\"stateVersion\", \"pos\"),\n UNIQUE(\"table\", \"rowKey\")\n );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.tableMetadata\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n // There's no incremental migration from v1. Just reset the replica.\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('upgrading replica to new schema');\n },\n minSafeVersion: 3,\n },\n\n 5: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n },\n migrateData: (_, db) => {\n recordEvent(db, 'upgrade');\n },\n },\n\n // Revised in the migration to v8 because the v6 code was incomplete.\n 6: {},\n\n 7: {\n migrateSchema: (_, db) => {\n // Note: The original \"changeLog\" table is kept so that the replica file\n // is compatible with older zero-caches. However, it is truncated for\n // space savings (since historic changes were never read).\n db.exec(`DELETE FROM \"_zero.changeLog\"`);\n // First version of changeLog2\n db.exec(CREATE_V7_CHANGE_LOG);\n },\n },\n\n 8: {\n migrateSchema: (_, db) => {\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n }\n },\n migrateData: (_, db) => {\n // Re-populate the ColumnMetadataStore; the original migration\n // at v6 was incomplete, as covered replicas migrated from earlier\n // versions but did not initialize the table for new replicas.\n db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n const tables = listTables(db, false, false);\n populateFromExistingTables(db, tables);\n },\n },\n\n 9: {\n migrateSchema: (_, db) => {\n db.exec(\n /*sql*/ `\n ALTER TABLE \"_zero.changeLog2\" \n ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n ALTER TABLE \"_zero.column_metadata\"\n ADD COLUMN backfill TEXT;\n ` + CREATE_V9_TABLE_METADATA_TABLE,\n );\n },\n },\n\n 10: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationConfig\" \n ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n `);\n },\n },\n\n 11: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n -- Removing the NOT NULL constraint from \"metadata\" requires copying\n -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"upstreamMetadata\" TEXT;\n UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n `);\n },\n },\n\n 12: {\n migrateSchema: (_, db) => {\n // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n // It is needed for backwards compatibility.\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"metadata\" TEXT;\n `);\n },\n\n migrateData: (_, db) => {\n // For rollback then roll forward, re-copy anything written to metadata.\n db.exec(/*sql*/ `\n UPDATE \"_zero.tableMetadata\" \n SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n \"metadata\" = NULL;\n `);\n },\n },\n\n 13: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n `);\n },\n\n migrateData: (_, db) => {\n db.exec(/*sql*/ `\n UPDATE \"_zero.replicationState\" \n SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n },\n },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAgBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,IAAI,aAAa,eAAe,EAAE,SAAS,kBACzC,MAAM,IAAI,gBAAgB,EAAE,OAAO;EAErC,MAAM;CACR;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;EACnB,MAAM,IAAI,MACR,wDACF;CACF,EACF,GACA,yBACF;AACF;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/packages/zero-cache/src/services/change-source/custom/change-source.js ADDED Viewed

@@ -0,0 +1,176 @@
+import { assert, unreachable } from "../../../../../shared/src/asserts.js";
+import { deepEqual } from "../../../../../shared/src/json.js";
+import { stringify } from "../../../../../shared/src/bigint-json.js";
+import { changeStreamMessageSchema } from "../protocol/current/downstream.js";
+import "../protocol/current/upstream.js";
+import { Database } from "../../../../../zqlite/src/db.js";
+import { computeZqlSpecs } from "../../../db/lite-tables.js";
+import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
+import { StatementRunner } from "../../../db/statements.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { initReplica } from "../common/replica-schema.js";
+import { stream } from "../../../types/streams.js";
+import { ChangeProcessor } from "../../replicator/change-processor.js";
+import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
+import { WebSocket } from "ws";
+//#region ../zero-cache/src/services/change-source/custom/change-source.ts
+/**
+* Initializes a Custom change source before streaming changes from the
+* corresponding logical replication stream.
+*/
+async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context) {
+	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => initialSync(log, shard, tx, upstreamURI, context));
+	const replica = new Database(lc, replicaDbFile);
+	const subscriptionState = getSubscriptionState(new StatementRunner(replica));
+	replica.close();
+	if (shard.publications.length) {
+		const requested = shard.publications.toSorted();
+		const replicated = subscriptionState.publications.sort();
+		if (!deepEqual(requested, replicated)) throw new Error(`Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`);
+	}
+	return {
+		subscriptionState,
+		changeSource: new CustomChangeSource(lc, upstreamURI, shard, subscriptionState)
+	};
+}
+var CustomChangeSource = class {
+	#lc;
+	#upstreamUri;
+	#shard;
+	#replicationConfig;
+	constructor(lc, upstreamUri, shard, replicationConfig) {
+		this.#lc = lc.withContext("component", "change-source");
+		this.#upstreamUri = upstreamUri;
+		this.#shard = shard;
+		this.#replicationConfig = replicationConfig;
+	}
+	initialSync() {
+		return this.#startStream();
+	}
+	startLagReporter() {
+		return null;
+	}
+	stop() {
+		return Promise.resolve();
+	}
+	startStream(clientWatermark, backfillRequests = []) {
+		if (backfillRequests?.length) throw new Error("backfill is yet not supported for custom change sources");
+		return Promise.resolve(this.#startStream(clientWatermark));
+	}
+	#startStream(clientWatermark) {
+		const { publications, replicaVersion } = this.#replicationConfig;
+		const { appID, shardNum } = this.#shard;
+		const url = new URL(this.#upstreamUri);
+		url.searchParams.set("appID", appID);
+		url.searchParams.set("shardNum", String(shardNum));
+		for (const pub of publications) url.searchParams.append("publications", pub);
+		if (clientWatermark) {
+			assert(replicaVersion.length, "replicaVersion is required when clientWatermark is set");
+			url.searchParams.set("lastWatermark", clientWatermark);
+			url.searchParams.set("replicaVersion", replicaVersion);
+		}
+		const ws = new WebSocket(url);
+		const { instream, outstream } = stream(this.#lc, ws, changeStreamMessageSchema, { coalesce: (curr) => curr });
+		return {
+			changes: instream,
+			acks: outstream
+		};
+	}
+};
+/**
+* Initial sync for a custom change source makes a request to the
+* change source endpoint with no `replicaVersion` or `lastWatermark`.
+* The initial transaction returned by the endpoint is treated as
+* the initial sync, and the commit watermark of that transaction
+* becomes the `replicaVersion` of the initialized replica.
+*
+* Note that this is equivalent to how the LSN of the Postgres WAL
+* at initial sync time is the `replicaVersion` (and starting
+* version for all initially-synced rows).
+*/
+async function initialSync(lc, shard, tx, upstreamURI, context) {
+	const { appID: id, publications } = shard;
+	const { changes } = new CustomChangeSource(lc, upstreamURI, shard, {
+		replicaVersion: "",
+		publications
+	}).initialSync();
+	createReplicationStateTables(tx);
+	const processor = new ChangeProcessor(new StatementRunner(tx), "initial-sync", (_, err) => {
+		throw err;
+	});
+	const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);
+	try {
+		let num = 0;
+		for await (const change of changes) {
+			const [tag] = change;
+			switch (tag) {
+				case "begin": {
+					const { commitWatermark } = change[2];
+					lc.info?.(`initial sync of shard ${id} at replicaVersion ${commitWatermark}`);
+					statusPublisher.publish(lc, "Initializing", `Copying upstream tables at version ${commitWatermark}`, 5e3);
+					initReplicationState(tx, publications.toSorted(), commitWatermark, context, false);
+					processor.processMessage(lc, change);
+					break;
+				}
+				case "data":
+					processor.processMessage(lc, change);
+					if (++num % 1e3 === 0) lc.debug?.(`processed ${num} changes`);
+					break;
+				case "commit":
+					processor.processMessage(lc, change);
+					validateInitiallySyncedData(lc, tx, shard);
+					lc.info?.(`finished initial-sync of ${num} changes`);
+					return;
+				case "status": break;
+				case "control": {
+					const { tag, message } = change[1];
+					if (tag === "reset-required") throw new AutoResetSignal(message ?? "auto-reset signaled by change source");
+				}
+				case "rollback": throw new Error(`unexpected message during initial-sync: ${stringify(change)}`);
+				default: unreachable(change);
+			}
+		}
+		throw new Error(`change source ${upstreamURI} closed before initial-sync completed`);
+	} catch (e) {
+		await statusPublisher.publishAndThrowError(lc, "Initializing", e);
+	} finally {
+		statusPublisher.stop();
+	}
+}
+function getRequiredTables({ appID, shardNum }) {
+	return {
+		[`${appID}_${shardNum}.clients`]: {
+			clientGroupID: { type: "string" },
+			clientID: { type: "string" },
+			lastMutationID: { type: "number" },
+			userID: { type: "string" }
+		},
+		[`${appID}_${shardNum}.mutations`]: {
+			clientGroupID: { type: "string" },
+			clientID: { type: "string" },
+			mutationID: { type: "number" },
+			mutation: { type: "json" }
+		},
+		[`${appID}.permissions`]: {
+			permissions: { type: "json" },
+			hash: { type: "string" }
+		}
+	};
+}
+function validateInitiallySyncedData(lc, db, shard) {
+	const tables = computeZqlSpecs(lc, db, { includeBackfillingColumns: true });
+	const required = getRequiredTables(shard);
+	for (const [name, columns] of Object.entries(required)) {
+		const table = tables.get(name)?.zqlSpec;
+		if (!table) throw new Error(`Upstream is missing the "${name}" table. (Found ${[...tables.keys()]})Please ensure that each table has a unique index over one or more non-null columns.`);
+		for (const [col, { type }] of Object.entries(columns)) {
+			const found = table[col];
+			if (!found) throw new Error(`Upstream "${table}" table is missing the "${col}" column`);
+			if (found.type !== type) throw new Error(`Upstream "${table}.${col}" column is a ${found.type} type but must be a ${type} type.`);
+		}
+	}
+}
+//#endregion
+export { initializeCustomChangeSource };
+//# sourceMappingURL=change-source.js.map

package/out/packages/zero-cache/src/services/change-source/custom/change-source.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;CAC7E,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO,CAC/D;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EAAC;EAAmB,cAAA,IAPF,mBACvB,IACA,aACA,OACA,iBAGyB;CAAY;AACzC;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/packages/zero-cache/src/services/change-source/pg/backfill-metadata.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"backfill-metadata.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/backfill-metadata.ts"],"sourcesContent":["import * as v from '../../../../../shared/src/valita.ts';\n\n// PG-specific messages passed in the table metadata and backfill ID\n// messages.\n\nexport const columnMetadataSchema = v.object({\n attNum: v.number(),\n});\n\nexport type ColumnMetadata = v.Infer<typeof columnMetadataSchema>;\n\nexport const tableMetadataSchema = v.object({\n schemaOID: v.number(),\n relationOID: v.number(),\n rowKey: v.record(columnMetadataSchema),\n});\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n"],"mappings":";;AAKA,IAAa,uBAAuB,eAAE,OAAO,EAC3C,QAAQ,eAAE,OAAO,EACnB,CAAC;AAID,IAAa,sBAAsB,eAAE,OAAO;CAC1C,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO;CACtB,QAAQ,eAAE,OAAO,oBAAoB;AACvC,CAAC"}

package/out/packages/zero-cache/src/services/change-source/pg/backfill-stream.js ADDED Viewed

@@ -0,0 +1,197 @@
+import { assert } from "../../../../../shared/src/asserts.js";
+import { parse } from "../../../../../shared/src/valita.js";
+import { equals } from "../../../../../shared/src/set-utils.js";
+import { connectPgClient, pgClient } from "../../../types/pg.js";
+import { toStateVersionString } from "./lsn.js";
+import { READONLY } from "../../../db/mode-enum.js";
+import { getPublicationInfo } from "./schema/published.js";
+import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
+import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
+import { TsvParser } from "../../../db/pg-copy.js";
+import { getTypeParsers } from "../../../db/pg-type-parser.js";
+import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
+import { columnMetadataSchema, tableMetadataSchema } from "./backfill-metadata.js";
+import { createReplicationSlot } from "./replication-slots.js";
+import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js";
+import postgres from "postgres";
+import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
+//#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
+var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
+var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
+/**
+* Streams a series of `backfill` messages (ending with `backfill-complete`)
+* at a set watermark (i.e. LSN). The data is retrieved via a COPY stream
+* made at a transaction snapshot corresponding to specific LSN, obtained by
+* creating a short-lived replication slot.
+*/
+async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
+	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
+	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
+	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", { ["max_lifetime"]: 7200 });
+	let tx;
+	let watermark;
+	try {
+		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
+		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
+		const { relation, columns } = backfill;
+		const cols = [...relation.rowKey.columns, ...columns];
+		const stmts = makeDownloadStatements(tableSpec, cols);
+		if (textCopy) {
+			const types = await getTypeParsers(db, { returnJsonAsString: true });
+			yield* stream(lc, tx, backfill, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
+				const parser = types.getTypeParser(tableSpec.columns[col].typeOID);
+				return (text) => parser(text);
+			}), flushThresholdBytes);
+		} else {
+			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, makeBinarySelectExprs(tableSpec, cols));
+			yield* stream(lc, tx, backfill, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
+				const spec = tableSpec.columns[col];
+				const decoder = hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder;
+				return (buf) => decoder(buf);
+			}), flushThresholdBytes);
+		}
+	} catch (e) {
+		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
+		throw e;
+	} finally {
+		tx?.setDone();
+		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
+	}
+}
+async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, flushThresholdBytes) {
+	assert(!SAMPLE_OR_LIMIT_RE.test(copyCommand), `backfill COPY must not sample or limit: ${copyCommand}`);
+	const start = performance.now();
+	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
+	const status = {
+		rows: 0,
+		totalRows: Number(rows[0].totalRows),
+		totalBytes: Number(bytes[0].totalBytes)
+	};
+	let elapsed = (performance.now() - start).toFixed(3);
+	lc.info?.(`Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`, { status });
+	const copyStream = await tx.processReadTask((sql) => sql.unsafe(copyCommand).readable());
+	let totalBytes = 0;
+	let totalMsgs = 0;
+	let rowValues = [];
+	let bufferedBytes = 0;
+	const logFlushed = () => {
+		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
+	};
+	let row = Array.from({ length: decoders.length });
+	let col = 0;
+	for await (const data of copyStream) {
+		const chunk = data;
+		for (const field of parser.parse(chunk)) {
+			row[col] = field === null ? null : decoders[col](field);
+			if (++col === decoders.length) {
+				rowValues.push(row);
+				status.rows++;
+				row = Array.from({ length: decoders.length });
+				col = 0;
+			}
+		}
+		bufferedBytes += chunk.byteLength;
+		totalBytes += chunk.byteLength;
+		if (bufferedBytes >= flushThresholdBytes) {
+			yield {
+				tag: "backfill",
+				...backfill,
+				rowValues,
+				status
+			};
+			totalMsgs++;
+			logFlushed();
+			rowValues = [];
+			bufferedBytes = 0;
+		}
+	}
+	if (rowValues.length > 0) {
+		yield {
+			tag: "backfill",
+			...backfill,
+			rowValues,
+			status
+		};
+		totalMsgs++;
+		logFlushed();
+	}
+	yield {
+		tag: "backfill-completed",
+		...backfill,
+		status
+	};
+	elapsed = (performance.now() - start).toFixed(3);
+	lc.info?.(`Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`);
+}
+/**
+* Creates (and drops) a replication slot in order to obtain a snapshot
+* that corresponds with a specific LSN. Sets the snapshot on the
+* TransactionPool and returns the watermark corresponding to the LSN.
+*
+* (Note that PG's other LSN-related functions are not scoped to a
+*  transaction; this is the only way to get set a transaction at a specific
+*  LSN.)
+*/
+async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
+	const replicationSession = pgClient(lc, upstreamURI, "backfill-replication-session", {
+		["fetch_types"]: false,
+		connection: { replication: "database" }
+	});
+	const slotName = `${slotNamePrefix}_bf_${Date.now()}`;
+	try {
+		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, { slotName });
+		const { init, imported } = importSnapshot(snapshot);
+		const tx = new TransactionPool(lc, {
+			mode: READONLY,
+			init
+		}).run(db);
+		await imported;
+		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${slotName}"`);
+		const watermark = toStateVersionString(lsn);
+		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
+		return {
+			tx,
+			watermark
+		};
+	} catch (e) {
+		await replicationSession.unsafe(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
+         WHERE slot_name = '${slotName}'`);
+		lc.warn?.(`Failed to create backfill snapshot`, e);
+		throw e;
+	} finally {
+		await replicationSession.end();
+	}
+}
+function validateSchema(tx, publications, bf, watermark) {
+	return tx.processReadTask(async (sql) => {
+		const { tables } = await getPublicationInfo(sql, publications);
+		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
+		if (!spec) throw new SchemaIncompatibilityError(bf, `Table has been renamed or dropped`);
+		const tableMeta = parse(bf.table.metadata, tableMetadataSchema);
+		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
+		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
+		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
+		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
+		for (const [col, val] of allCols) {
+			const colSpec = spec.columns[col];
+			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
+			if (parse(val, columnMetadataSchema).attNum !== colSpec.pos) throw new SchemaIncompatibilityError(bf, `Column ${col} no longer corresponds to the original column`);
+		}
+		return {
+			tableSpec: spec,
+			backfill: {
+				relation: {
+					schema: bf.table.schema,
+					name: bf.table.name,
+					rowKey: { columns: Object.keys(tableMeta.rowKey) }
+				},
+				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
+				watermark
+			}
+		};
+	});
+}
+//#endregion
+export { streamBackfill };
+//# sourceMappingURL=backfill-stream.js.map