npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.4 → 1.11.0-canary.41 - Mend

@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1015) hide show

package/out/zero-cache/src/services/change-source/common/replica-schema.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { listTables } from "../../../db/lite-tables.js";
-import { CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
+import { BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
 import { deleteLiteDB } from "../../../db/delete-lite-db.js";
@@ -70,6 +70,15 @@ var CREATE_V9_TABLE_METADATA_TABLE = `
     PRIMARY KEY ("schema", "table")
   );
 `;
+var CREATE_V17_BACKFILLING_TABLE = `
+  CREATE TABLE "${BACKFILLING_TABLE}" (
+    "schema"   TEXT NOT NULL,
+    "table"    TEXT NOT NULL,
+    "column"   TEXT NOT NULL,
+    "backfill" TEXT NOT NULL,
+    PRIMARY KEY ("schema", "table", "column")
+  );
+`;
 var V14_CHANGE_LOG_STREAM_TABLE = "_zero.changeLogStream";
 var V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX = "_zero.changeLogStream_writeTimeMs";
 `${V14_CHANGE_LOG_STREAM_TABLE}${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}${V14_CHANGE_LOG_STREAM_TABLE}`;
@@ -181,12 +190,15 @@ var schemaVersionMigrationMap = {
 	} },
 	17: {
 		migrateSchema: (_, db) => {
-			db.exec(CREATE_BACKFILLING_TABLE);
+			db.exec(CREATE_V17_BACKFILLING_TABLE);
 		},
 		migrateData: (lc, db) => {
 			populateBackfillingFromColumnMetadata(lc, db);
 		}
-	}
+	},
+	18: { migrateSchema: (_, db) => {
+		db.exec(`ALTER TABLE "${BACKFILLING_TABLE}" ADD COLUMN "progress" TEXT`);
+	} }
 };
 Object.keys(schemaVersionMigrationMap).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);
 //#endregion

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import {existsSync, renameSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../../db/delete-lite-db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n CREATE_BACKFILLING_TABLE,\n populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n CREATE_RUNTIME_EVENTS_TABLE,\n recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n const isInitialSync = !existsSync(dbPath);\n const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;\n if (isInitialSync) {\n deleteLiteDB(migrationPath);\n }\n\n const setupMigration: Migration = {\n migrateSchema: (log, tx) => initialSync(log, tx),\n minSafeVersion: 1,\n };\n\n try {\n await runSchemaMigrations(\n log,\n debugName,\n migrationPath,\n setupMigration,\n schemaVersionMigrationMap,\n );\n if (isInitialSync) {\n renameSync(migrationPath, dbPath);\n }\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, migrationPath, e);\n }\n}\n\nexport async function upgradeReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n) {\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n // setupMigration should never be invoked\n {\n migrateSchema: () => {\n throw new Error(\n 'This should only be called for already synced replicas',\n );\n },\n },\n schemaVersionMigrationMap,\n );\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, dbPath, e);\n }\n}\n\nfunction throwAutoResetForCorruption(\n log: LogContext,\n debugName: string,\n dbPath: string,\n e: unknown,\n): never {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n throw new AutoResetSignal(\n `replica database appears corrupt: ${String(e)}`,\n {cause: e},\n );\n }\n throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /sql/ `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /sql/ `\n CREATE TABLE \"_zero.changeLog2\" (\n \"stateVersion\" TEXT NOT NULL,\n \"pos\" INT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"rowKey\" TEXT NOT NULL,\n \"op\" TEXT NOT NULL,\n PRIMARY KEY(\"stateVersion\", \"pos\"),\n UNIQUE(\"table\", \"rowKey\")\n );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /sql/ `\n CREATE TABLE \"_zero.tableMetadata\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// describe replica files created by pre-release v14 builds, while that module\n// describes the separate change-log database. The two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table created by pre-release v14 builds and dropped by\n// migration 16. Retained at its old shape for migration tests only.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /sql/ `\n CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n \"watermark\" TEXT NOT NULL,\n \"pos\" INTEGER NOT NULL,\n \"change\" TEXT NOT NULL,\n \"precommit\" TEXT,\n \"writeTimeMs\" INTEGER,\n PRIMARY KEY (\"watermark\", \"pos\")\n );\n\n CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n // There's no incremental migration from v1. Just reset the replica.\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('upgrading replica to new schema');\n },\n minSafeVersion: 3,\n },\n\n 5: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n },\n migrateData: (_, db) => {\n recordEvent(db, 'upgrade');\n },\n },\n\n // Revised in the migration to v8 because the v6 code was incomplete.\n 6: {},\n\n 7: {\n migrateSchema: (_, db) => {\n // Note: The original \"changeLog\" table is kept so that the replica file\n // is compatible with older zero-caches. However, it is truncated for\n // space savings (since historic changes were never read).\n db.exec(`DELETE FROM \"_zero.changeLog\"`);\n // First version of changeLog2\n db.exec(CREATE_V7_CHANGE_LOG);\n },\n },\n\n 8: {\n migrateSchema: (_, db) => {\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n }\n },\n migrateData: (_, db) => {\n // Re-populate the ColumnMetadataStore; the original migration\n // at v6 was incomplete, as covered replicas migrated from earlier\n // versions but did not initialize the table for new replicas.\n db.exec(/sql/ `DELETE FROM \"_zero.column_metadata\"`);\n\n const tables = listTables(db, false, false);\n populateFromExistingTables(db, tables);\n },\n },\n\n 9: {\n migrateSchema: (_, db) => {\n db.exec(\n /sql/ `\n ALTER TABLE \"_zero.changeLog2\" \n ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n ALTER TABLE \"_zero.column_metadata\"\n ADD COLUMN backfill TEXT;\n ` + CREATE_V9_TABLE_METADATA_TABLE,\n );\n },\n },\n\n 10: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.replicationConfig\" \n ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n `);\n },\n },\n\n 11: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n -- Removing the NOT NULL constraint from \"metadata\" requires copying\n -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"upstreamMetadata\" TEXT;\n UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n `);\n },\n },\n\n 12: {\n migrateSchema: (_, db) => {\n // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n // It is needed for backwards compatibility.\n db.exec(/sql/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"metadata\" TEXT;\n `);\n },\n\n migrateData: (_, db) => {\n // For rollback then roll forward, re-copy anything written to metadata.\n db.exec(/sql/ `\n UPDATE \"_zero.tableMetadata\" \n SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n \"metadata\" = NULL;\n `);\n },\n },\n\n 13: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n `);\n },\n\n migrateData: (_, db) => {\n db.exec(/sql/ `\n UPDATE \"_zero.replicationState\" \n SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n },\n },\n\n // The replica-local change log moved to its own database before v14 shipped.\n 14: {},\n\n 15: {\n migrateSchema: (_, db) => {\n // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n // create a new table, copy the data over, delete the old table, and\n // rename. This is cheap because it's a single row table.\n db.exec(/sql/ `\n CREATE TABLE \"_zero.replicationState2\" (\n stateVersion TEXT NOT NULL,\n writeTimeMs INTEGER NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n\n INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n DROP TABLE \"_zero.replicationState\";\n\n ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n `);\n },\n },\n\n // The change log moved out of the replica and into its own database\n // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n // every file the backup ships and every follower downloads. Nothing reads it\n // as of this version.\n //\n // `IF EXISTS` because fresh and post-v13 replicas never create this table.\n // Pre-release v14 builds did, so keep their cleanup idempotent.\n //\n // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n // trigger in `workers/replicator.ts`; no special handling here.\n //\n // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n // table. Bumping the rollback limit would turn every rollback into a hard\n // startup failure to protect a configuration that is not enabled anywhere.\n 16: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n `);\n },\n },\n\n // `_zero.column_metadata.backfill` records which columns are backfilling but\n // is keyed by lite table name, which has no inverse: it cannot be turned back\n // into the `BackfillRequest`s that a change log initialized from this replica\n // would have to send. This table carries the upstream identity alongside the\n // same state. See `replicator/schema/backfilling.ts`.\n //\n // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n // since nothing at v16 reads the new table. Its writes to\n // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n // forward again, so a rollback costs nothing but the re-seed.\n 17: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_BACKFILLING_TABLE);\n },\n\n migrateData: (lc, db) => {\n populateBackfillingFromColumnMetadata(lc, db);\n },\n },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;AAyBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,gBAAgB,CAAC,WAAW,MAAM;CACxC,MAAM,gBAAgB,gBAAgB,GAAG,OAAO,QAAQ;CACxD,IAAI,eACF,aAAa,aAAa;CAG5B,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,eACA,gBACA,yBACF;EACA,IAAI,eACF,WAAW,eAAe,MAAM;CAEpC,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,eAAe,CAAC;CAC9D;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAYtD,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAIkD,GAClC,4BADkC,EAUlC,uCAVkC,EAW1C,4BAX0C;AAepD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAGA,IAAI,CAAC;CAEL,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAiBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,wBAAwB;EAClC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
1	+ {"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import {existsSync, renameSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../../db/delete-lite-db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n BACKFILLING_TABLE,\n populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n CREATE_RUNTIME_EVENTS_TABLE,\n recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n const isInitialSync = !existsSync(dbPath);\n const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;\n if (isInitialSync) {\n deleteLiteDB(migrationPath);\n }\n\n const setupMigration: Migration = {\n migrateSchema: (log, tx) => initialSync(log, tx),\n minSafeVersion: 1,\n };\n\n try {\n await runSchemaMigrations(\n log,\n debugName,\n migrationPath,\n setupMigration,\n schemaVersionMigrationMap,\n );\n if (isInitialSync) {\n renameSync(migrationPath, dbPath);\n }\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, migrationPath, e);\n }\n}\n\nexport async function upgradeReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n) {\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n // setupMigration should never be invoked\n {\n migrateSchema: () => {\n throw new Error(\n 'This should only be called for already synced replicas',\n );\n },\n },\n schemaVersionMigrationMap,\n );\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, dbPath, e);\n }\n}\n\nfunction throwAutoResetForCorruption(\n log: LogContext,\n debugName: string,\n dbPath: string,\n e: unknown,\n): never {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n throw new AutoResetSignal(\n `replica database appears corrupt: ${String(e)}`,\n {cause: e},\n );\n }\n throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /sql/ `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /sql/ `\n CREATE TABLE \"_zero.changeLog2\" (\n \"stateVersion\" TEXT NOT NULL,\n \"pos\" INT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"rowKey\" TEXT NOT NULL,\n \"op\" TEXT NOT NULL,\n PRIMARY KEY(\"stateVersion\", \"pos\"),\n UNIQUE(\"table\", \"rowKey\")\n );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /sql/ `\n CREATE TABLE \"_zero.tableMetadata\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n`;\n\nexport const CREATE_V17_BACKFILLING_TABLE = /sql/ `\n CREATE TABLE \"${BACKFILLING_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"column\" TEXT NOT NULL,\n \"backfill\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\", \"column\")\n );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// describe replica files created by pre-release v14 builds, while that module\n// describes the separate change-log database. The two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table created by pre-release v14 builds and dropped by\n// migration 16. Retained at its old shape for migration tests only.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /sql/ `\n CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n \"watermark\" TEXT NOT NULL,\n \"pos\" INTEGER NOT NULL,\n \"change\" TEXT NOT NULL,\n \"precommit\" TEXT,\n \"writeTimeMs\" INTEGER,\n PRIMARY KEY (\"watermark\", \"pos\")\n );\n\n CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n // There's no incremental migration from v1. Just reset the replica.\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('upgrading replica to new schema');\n },\n minSafeVersion: 3,\n },\n\n 5: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n },\n migrateData: (_, db) => {\n recordEvent(db, 'upgrade');\n },\n },\n\n // Revised in the migration to v8 because the v6 code was incomplete.\n 6: {},\n\n 7: {\n migrateSchema: (_, db) => {\n // Note: The original \"changeLog\" table is kept so that the replica file\n // is compatible with older zero-caches. However, it is truncated for\n // space savings (since historic changes were never read).\n db.exec(`DELETE FROM \"_zero.changeLog\"`);\n // First version of changeLog2\n db.exec(CREATE_V7_CHANGE_LOG);\n },\n },\n\n 8: {\n migrateSchema: (_, db) => {\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n }\n },\n migrateData: (_, db) => {\n // Re-populate the ColumnMetadataStore; the original migration\n // at v6 was incomplete, as covered replicas migrated from earlier\n // versions but did not initialize the table for new replicas.\n db.exec(/sql/ `DELETE FROM \"_zero.column_metadata\"`);\n\n const tables = listTables(db, false, false);\n populateFromExistingTables(db, tables);\n },\n },\n\n 9: {\n migrateSchema: (_, db) => {\n db.exec(\n /sql/ `\n ALTER TABLE \"_zero.changeLog2\" \n ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n ALTER TABLE \"_zero.column_metadata\"\n ADD COLUMN backfill TEXT;\n ` + CREATE_V9_TABLE_METADATA_TABLE,\n );\n },\n },\n\n 10: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.replicationConfig\" \n ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n `);\n },\n },\n\n 11: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n -- Removing the NOT NULL constraint from \"metadata\" requires copying\n -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"upstreamMetadata\" TEXT;\n UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n `);\n },\n },\n\n 12: {\n migrateSchema: (_, db) => {\n // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n // It is needed for backwards compatibility.\n db.exec(/sql/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"metadata\" TEXT;\n `);\n },\n\n migrateData: (_, db) => {\n // For rollback then roll forward, re-copy anything written to metadata.\n db.exec(/sql/ `\n UPDATE \"_zero.tableMetadata\" \n SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n \"metadata\" = NULL;\n `);\n },\n },\n\n 13: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n `);\n },\n\n migrateData: (_, db) => {\n db.exec(/sql/ `\n UPDATE \"_zero.replicationState\" \n SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n },\n },\n\n // The replica-local change log moved to its own database before v14 shipped.\n 14: {},\n\n 15: {\n migrateSchema: (_, db) => {\n // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n // create a new table, copy the data over, delete the old table, and\n // rename. This is cheap because it's a single row table.\n db.exec(/sql/ `\n CREATE TABLE \"_zero.replicationState2\" (\n stateVersion TEXT NOT NULL,\n writeTimeMs INTEGER NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n\n INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n DROP TABLE \"_zero.replicationState\";\n\n ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n `);\n },\n },\n\n // The change log moved out of the replica and into its own database\n // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n // every file the backup ships and every follower downloads. Nothing reads it\n // as of this version.\n //\n // `IF EXISTS` because fresh and post-v13 replicas never create this table.\n // Pre-release v14 builds did, so keep their cleanup idempotent.\n //\n // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n // trigger in `workers/replicator.ts`; no special handling here.\n //\n // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n // table. Bumping the rollback limit would turn every rollback into a hard\n // startup failure to protect a configuration that is not enabled anywhere.\n 16: {\n migrateSchema: (_, db) => {\n db.exec(/sql/ `\n DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n `);\n },\n },\n\n // `_zero.column_metadata.backfill` records which columns are backfilling but\n // is keyed by lite table name, which has no inverse: it cannot be turned back\n // into the `BackfillRequest`s that a change log initialized from this replica\n // would have to send. This table carries the upstream identity alongside the\n // same state. See `replicator/schema/backfilling.ts`.\n //\n // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n // since nothing at v16 reads the new table. Its writes to\n // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n // forward again, so a rollback costs nothing but the re-seed.\n 17: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_V17_BACKFILLING_TABLE);\n },\n\n migrateData: (lc, db) => {\n populateBackfillingFromColumnMetadata(lc, db);\n },\n },\n\n // Adds `_zero.backfilling.progress`, with which the replica tracks the\n // progress of its pending backfills and reports it to the change-streamer\n // (protocol v8).\n //\n // No `minSafeVersion`: an older zero-cache ignores the column (its writes\n // name their columns explicitly), and a `progress` left stale by one is at\n // or before the actual progress, which only results in redundant backfill.\n 18: {\n migrateSchema: (_, db) => {\n db.exec(\n /sql/ `ALTER TABLE \"${BACKFILLING_TABLE}\" ADD COLUMN \"progress\" TEXT`,\n );\n },\n },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;AAyBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,gBAAgB,CAAC,WAAW,MAAM;CACxC,MAAM,gBAAgB,gBAAgB,GAAG,OAAO,QAAQ;CACxD,IAAI,eACF,aAAa,aAAa;CAG5B,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,eACA,gBACA,yBACF;EACA,IAAI,eACF,WAAW,eAAe,MAAM;CAEpC,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,eAAe,CAAC;CAC9D;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,+BAAuC;kBAClC,kBAAkB;;;;;;;;AAYpC,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAIkD,GAClC,4BADkC,EAUlC,uCAVkC,EAW1C,4BAX0C;AAepD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAGA,IAAI,CAAC;CAEL,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAiBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,4BAA4B;EACtC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;CASA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO,gBAAgB,kBAAkB,6BAC5C;CACF,EACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts CHANGED Viewed

@@ -9,7 +9,7 @@ export type ServerContext = JSONObject;
  * Initializes a Custom change source before streaming changes from the
  * corresponding logical replication stream.
  */
-export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext, { litestream, constraints }?: RestoreOptions): Promise<InitializeResult>;
+export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext, { litestream, acquirePurgeLock }?: RestoreOptions): Promise<InitializeResult>;
 /**
  * Initial sync for a custom change source makes a request to the
  * change source endpoint with no `replicaVersion` or `lastWatermark`.

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAcnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAQtC,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,EACtB,EAAC,UAAU,EAAE,~~WAAW~~,EAAC,GAAE,cAAmB,~~GAC7C~~,OAAO,CAAC,gBAAgB,CAAC,~~CAwD3B~~;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAcnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAQtC,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,EACtB,EAAC,UAAU,EAAE,gBAAgB,EAAC,GAAE,cAAmB,GAClD,OAAO,CAAC,gBAAgB,CAAC,CA8D3B;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}

package/out/zero-cache/src/services/change-source/custom/change-source.js CHANGED Viewed

@@ -19,8 +19,11 @@ import { WebSocket } from "ws";
 * Initializes a Custom change source before streaming changes from the
 * corresponding logical replication stream.
 */
-async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context, { litestream, constraints } = {}) {
-	if (litestream?.backupURL) await restoreReplica(lc, litestream, replicaDbFile, constraints);
+async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context, { litestream, acquirePurgeLock } = {}) {
+	if (litestream?.backupURL) {
+		const lock = await acquirePurgeLock?.();
+		await restoreReplica(lc, litestream, replicaDbFile, lock === "behind-slot" ? void 0 : lock ?? void 0);
+	}
 	let initialSynced = false;
 	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => {
 		initialSynced = true;
@@ -39,7 +42,8 @@ async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFil
 		changeSource: new CustomChangeSource(lc, upstreamURI, shard, subscriptionState),
 		destinationBackupURL: litestream?.backupURL,
 		replicaID: null,
-		waitForBackupBeforeServing: initialSynced
+		waitForBackupBeforeServing: initialSynced,
+		newBackupLineage: false
 	};
 }
 var CustomChangeSource = class {

package/out/zero-cache/src/services/change-source/custom/change-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n restoreReplica,\n type InitializeResult,\n type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n {litestream, constraints}: RestoreOptions = {},\n): Promise<InitializeResult> {\n // At the moment, the custom change-source implementation does not support\n // per-RM backups (and thus does not support litestream v5 or RMv2). The\n // current proposal to support this without requiring additional upstream\n // state tracking is to:\n // - use litestream to (ltx) query a fixed pool of subfolders within the\n // backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n // - restore from the subfolder with the latest ltx file\n // - backup to the subfolder with the oldest (or non-existent) ltx file\n if (litestream?.backupURL) {\n await restoreReplica(lc, litestream, replicaDbFile, constraints);\n }\n\n let initialSynced = false;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => {\n initialSynced = true;\n return initialSync(log, shard, tx, upstreamURI, context);\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL: litestream?.backupURL,\n // A custom change source has no upstream `replicas` table to identify\n // itself from, so the generation is the whole of the SQLite change log's\n // identity here.\n replicaID: null,\n waitForBackupBeforeServing: initialSynced,\n };\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,gBAA+B,CAAC,GAClB;CAS3B,IAAI,YAAY,WACd,MAAM,eAAe,IAAI,YAAY,eAAe,WAAW;CAGjE,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}
1	+ {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n restoreReplica,\n type InitializeResult,\n type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n {litestream, acquirePurgeLock}: RestoreOptions = {},\n): Promise<InitializeResult> {\n // At the moment, the custom change-source implementation does not support\n // per-RM backups (and thus does not support litestream v5 or RMv2). The\n // current proposal to support this without requiring additional upstream\n // state tracking is to:\n // - use litestream to (ltx) query a fixed pool of subfolders within the\n // backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n // - restore from the subfolder with the latest ltx file\n // - backup to the subfolder with the oldest (or non-existent) ltx file\n if (litestream?.backupURL) {\n // Without a replication slot, the change-log is never behind one.\n const lock = await acquirePurgeLock?.();\n const constraints =\n lock === 'behind-slot' ? undefined : (lock ?? undefined);\n await restoreReplica(lc, litestream, replicaDbFile, constraints);\n }\n\n let initialSynced = false;\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => {\n initialSynced = true;\n return initialSync(log, shard, tx, upstreamURI, context);\n },\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {\n subscriptionState,\n changeSource,\n destinationBackupURL: litestream?.backupURL,\n // A custom change source has no upstream `replicas` table to identify\n // itself from, so the generation is the whole of the SQLite change log's\n // identity here.\n replicaID: null,\n waitForBackupBeforeServing: initialSynced,\n // Backs up to the (non-lineage) base URL.\n newBackupLineage: false,\n };\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,qBAAoC,CAAC,GACvB;CAS3B,IAAI,YAAY,WAAW;EAEzB,MAAM,OAAO,MAAM,mBAAmB;EAGtC,MAAM,eAAe,IAAI,YAAY,eADnC,SAAS,gBAAgB,KAAA,IAAa,QAAQ,KAAA,CACe;CACjE;CAEA,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;EAE5B,kBAAkB;CACpB;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts CHANGED Viewed

@@ -2,6 +2,9 @@ import type { LogContext } from '@rocicorp/logger';
 import { type BackfillMessage } from '../common/backfill-manager.ts';
 import type { BackfillRequest } from '../protocol/current.ts';
 import type { Replica } from './schema/shard.ts';
+export declare function ctidToProgressMark(ctid: string): string;
+/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */
+export declare function progressMarkToCtid(mark: string): string;
 type StreamOptions = {
     /**
      * The number of bytes at which to flush a batch of rows in a
@@ -15,6 +18,12 @@ type StreamOptions = {
      * revert to the old code path if needed.
      */
     textCopy?: boolean | undefined;
+    /**
+     * @visibleForTesting
+     * Called after the snapshot transaction is opened, before the table is
+     * locked, e.g. to run a concurrent heap rewrite in between.
+     */
+    afterSnapshotForTesting?: (() => Promise<void>) | undefined;
 };
 /**
  * Streams a series of `backfill` messages (ending with `backfill-complete`)

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAkBjD~~,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;~~AACvC~~,OAAO,KAAK,~~EAEV~~,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;~~AAI~~/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;~~CAChC~~,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,CAAC,~~CAsGjC~~"}
1	+ {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAqBjD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,EAGV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAoC/C,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIvD;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIvD;AAmCD,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE/B;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;CAC7D,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,CAAC,CAkJjC"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -1,11 +1,15 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { parse } from "../../../../../shared/src/valita.js";
+import { must } from "../../../../../shared/src/must.js";
+import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
+import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { Queue } from "../../../../../shared/src/queue.js";
+import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
 import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
 import { TsvParser } from "../../../db/pg-copy.js";
@@ -17,6 +21,52 @@ import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js
 import postgres from "postgres";
 import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
+/**
+* Encodes a `ctid` text representation, e.g. `(123,4)`, as a
+* fixed-width string that sorts lexicographically in `ctid` order.
+*/
+var CTID_RE = /^\((\d+),(\d+)\)$/;
+var CTID_MARK_RE = /^(\d{10})\.(\d{5})$/;
+function ctidToProgressMark(ctid) {
+	const match = CTID_RE.exec(ctid);
+	assert(match, () => `invalid ctid: ${ctid}`);
+	return `${match[1].padStart(10, "0")}.${match[2].padStart(5, "0")}`;
+}
+/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */
+function progressMarkToCtid(mark) {
+	const match = CTID_MARK_RE.exec(mark);
+	assert(match, () => `invalid ctid progress mark: ${mark}`);
+	return `(${Number(match[1])},${Number(match[2])})`;
+}
+function progressMarks(previous, current) {
+	return previous ? {
+		previous,
+		current
+	} : { current };
+}
+/** The `ctid` of each row is appended as the last column of the download. */
+var CTID_SELECT = "ctid::text";
+/**
+* Settings that ensure that the COPY scans the table in physical (i.e.
+* `ctid`) order, which is necessary for the progress marks of a resumable
+* backfill to be meaningful. (Monotonicity is nonetheless verified as rows
+* are streamed.)
+*
+* * Synchronized seqscans can start in the middle of the table.
+* * Parallel scans interleave the rows of different workers.
+* * Index scans follow index order. (A publication row filter could
+*   otherwise be satisfied with an index.)
+*/
+var PHYSICAL_ORDER_SETTINGS = [
+	"synchronize_seqscans = off",
+	"max_parallel_workers_per_gather = 0",
+	"enable_indexscan = off",
+	"enable_indexonlyscan = off",
+	"enable_bitmapscan = off"
+];
+function rowCountMark(rows) {
+	return String(rows).padStart(16, "0");
+}
 var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
 var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 /**
@@ -27,42 +77,50 @@ var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 */
 async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
 	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
-	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
+	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false, afterSnapshotForTesting } = opts;
 	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", {
 		["max_lifetime"]: 1440 * 60,
 		liveness: "keepalive"
 	});
 	let tx;
 	let watermark;
+	let completed = false;
 	try {
 		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
-		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
+		await afterSnapshotForTesting?.();
+		const { tableSpec, backfill, progress } = await validateSchema(lc, tx, publications, bf, watermark);
 		const { relation, columns } = backfill;
 		const cols = [...relation.rowKey.columns, ...columns];
-		const stmts = makeDownloadStatements(tableSpec, cols);
+		const order = progress.resumable && progress.start ? { after: `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid` } : void 0;
+		const stmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...cols.map((col) => id(col)), CTID_SELECT], order);
+		if (progress.start) lc.info?.(`resuming backfill from ${progress.start.progressMark} (timeline ${progress.timeline})`);
 		if (textCopy) {
 			const types = await getTypeParsers(db, { returnJsonAsString: true });
-			yield* stream(lc, tx, backfill, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
 				const parser = types.getTypeParser(tableSpec.columns[col].typeOID);
 				return (text) => parser(text);
-			}), flushThresholdBytes);
+			}), (text) => text, flushThresholdBytes);
 		} else {
-			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, makeBinarySelectExprs(tableSpec, cols));
-			yield* stream(lc, tx, backfill, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
+			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT], order);
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
 				const spec = tableSpec.columns[col];
 				const decoder = hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder;
 				return (buf) => decoder(buf);
-			}), flushThresholdBytes);
+			}), (buf) => buf.toString("utf8"), flushThresholdBytes);
 		}
+		completed = true;
 	} catch (e) {
 		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
 		throw e;
 	} finally {
 		tx?.setDone();
-		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
+		if (completed) {
+			await tx?.done();
+			await db.end();
+		} else db.end({ timeout: 5 }).catch((e) => lc.warn?.(`error closing backfill connection`, e));
 	}
 }
-async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, flushThresholdBytes) {
+async function* stream(lc, tx, backfill, { timeline, resumable, start: startMark }, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, ctidDecoder, flushThresholdBytes) {
 	assert(!SAMPLE_OR_LIMIT_RE.test(copyCommand), `backfill COPY must not sample or limit: ${copyCommand}`);
 	const start = performance.now();
 	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
@@ -76,6 +134,7 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const chunks = new Queue();
 	const acks = new Queue();
 	const copyDone = tx.processReadTask(async (sql) => {
+		for (const setting of PHYSICAL_ORDER_SETTINGS) await sql.unsafe(`SET LOCAL ${setting}`);
 		const readable = await sql.unsafe(copyCommand).readable();
 		for await (const chunk of readable) {
 			chunks.enqueue(chunk);
@@ -91,14 +150,26 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const logFlushed = () => {
 		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
 	};
+	let current = startMark;
+	let previous = startMark;
+	const markOf = (progressMark) => ({
+		progressMark,
+		timeline
+	});
+	const numCols = decoders.length + 1;
 	let row = Array.from({ length: decoders.length });
 	let col = 0;
 	for (;;) {
 		const chunk = await chunks.dequeue();
 		if (chunk === "done") break;
 		for (const field of parser.parse(chunk)) {
-			row[col] = field === null ? null : decoders[col](field);
-			if (++col === decoders.length) {
+			if (col < decoders.length) row[col] = field === null ? null : decoders[col](field);
+			else {
+				const mark = resumable ? ctidToProgressMark(ctidDecoder(field)) : rowCountMark(status.rows + 1);
+				if (current !== void 0 && mark <= current.progressMark) throw new Error(`backfill rows are not in ctid order (${mark} after ${current.progressMark})`);
+				current = markOf(mark);
+			}
+			if (++col === numCols) {
 				rowValues.push(row);
 				status.rows++;
 				row = Array.from({ length: decoders.length });
@@ -107,16 +178,18 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		}
 		bufferedBytes += chunk.byteLength;
 		totalBytes += chunk.byteLength;
-		if (bufferedBytes >= flushThresholdBytes) {
+		if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {
 			yield {
 				message: {
 					tag: "backfill",
 					...backfill,
 					rowValues,
-					status
+					status,
+					progressMarks: progressMarks(previous, must(current))
 				},
 				byteSize: bufferedBytes
 			};
+			previous = current;
 			totalMsgs++;
 			logFlushed();
 			rowValues = [];
@@ -131,10 +204,12 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 				tag: "backfill",
 				...backfill,
 				rowValues,
-				status
+				status,
+				progressMarks: progressMarks(previous, must(current))
 			},
 			byteSize: bufferedBytes
 		};
+		previous = current;
 		totalMsgs++;
 		logFlushed();
 	}
@@ -142,7 +217,8 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		message: {
 			tag: "backfill-completed",
 			...backfill,
-			status
+			status,
+			progressMarks: previous ? { previous } : {}
 		},
 		byteSize: 0
 	};
@@ -167,7 +243,8 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 	try {
 		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, {
 			slotName,
-			temporary: true
+			temporary: true,
+			diagnosticsDB: db
 		});
 		const { init, imported } = importSnapshot(snapshot);
 		const tx = new TransactionPool(lc, {
@@ -182,10 +259,10 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 			watermark
 		};
 	} finally {
-		await replicationSession.end();
+		await replicationSession.end({ timeout: 5 });
 	}
 }
-function validateSchema(tx, publications, bf, watermark) {
+function validateSchema(lc, tx, publications, bf, watermark) {
 	return tx.processReadTask(async (sql) => {
 		const { tables } = await getPublicationInfo(sql, publications);
 		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
@@ -194,7 +271,7 @@ function validateSchema(tx, publications, bf, watermark) {
 		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
 		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
 		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
-		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
+		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns).map(([col, { id }]) => [col, id])];
 		for (const [col, val] of allCols) {
 			const colSpec = spec.columns[col];
 			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
@@ -210,10 +287,125 @@ function validateSchema(tx, publications, bf, watermark) {
 				},
 				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
 				watermark
-			}
+			},
+			progress: await getProgress(lc, sql, spec.oid, bf)
 		};
 	});
 }
+/**
+* Determines the timeline of the table's progress marks (as of the snapshot),
+* and whether the backfill can be resumed from the requested progress.
+*
+* The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest
+* of the transaction) so that its storage cannot be rewritten (by `VACUUM
+* FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which
+* require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note
+* that `pg_relation_filenode()` reflects the table's current storage (only
+* refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at
+* the transaction snapshot. If they differ, the storage was rewritten after the
+* snapshot, and the COPY would scan the new storage with the old snapshot,
+* which is not MVCC-safe for all rewrites (e.g. the table appears empty after
+* a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.
+*/
+async function getProgress(lc, sql, relationOID, bf) {
+	const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;
+	await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);
+	const [{ lockedOID, relkind, snapshotFilenode, filenode }] = await sql`
+    SELECT to_regclass(${table})::oid::int8 AS "lockedOID",
+           relkind,
+           relfilenode::text AS "snapshotFilenode",
+           pg_relation_filenode(oid)::text AS filenode
+      FROM pg_class WHERE oid = ${relationOID}`;
+	if (Number(lockedOID) !== relationOID) throw new SchemaIncompatibilityError(bf, `Table has been renamed or replaced`);
+	if (relkind !== "r" || filenode === null) return {
+		timeline: `${relkind}:${randomCharacters(12)}`,
+		resumable: false,
+		start: void 0
+	};
+	if (filenode !== snapshotFilenode) throw new Error(`${bf.table.schema}.${bf.table.name} was rewritten after the backfill snapshot (relfilenode ${snapshotFilenode} => ${filenode})`);
+	const timeline = filenode;
+	const requested = resumePoint(bf);
+	if (requested && requested.timeline !== timeline) {
+		lc.info?.(`restarting backfill from scratch: the requested progress is on timeline ${requested.timeline} but the table is on ${timeline}`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	if (requested && !await canResume(sql, relationOID, bf)) {
+		lc.info?.(`restarting backfill from scratch: the table has TOASTed values and backfilling columns [${Object.keys(bf.columns).join(",")}] are TOAST-able`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	return {
+		timeline,
+		resumable: true,
+		start: requested
+	};
+}
+/**
+* Whether resuming a backfill (in a new snapshot) is guaranteed to deliver
+* the values of every row that the preceding snapshot(s) had not reached.
+*
+* Within a single snapshot, every row is scanned exactly once, since the
+* version visible to the snapshot stays in place while it is held. Across
+* snapshots, however, a row that had not yet been reached (i.e. beyond the
+* resume mark `M`) can be updated between the snapshots, with its new
+* version placed at or before `M` (a non-HOT update into free space in an
+* earlier page, or a HOT update into a lower line pointer of `M`'s page).
+* Neither snapshot then scans the row, and it is only delivered by the
+* replication stream. The stream's UPDATE contains the full row, _except_
+* for unchanged values stored out-of-line in TOAST, which pgoutput omits
+* (unless the table has REPLICA IDENTITY FULL). A backfilling column with
+* such a value would be published without it.
+*
+* Resuming is therefore only done if no backfilling column can be TOASTed
+* (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the
+* table has no TOAST data. The latter is safe because an omitted value must
+* still be in the TOAST relation at the new snapshot (unless its row has since
+* been deleted, which the replication stream does deliver), and conservative
+* because the TOAST relation does not shrink to empty without a heap rewrite,
+* which changes the timeline.
+*
+* Potential optimization for TOAST-able backfills: have the resumed run also
+* deliver the rows at or before `M` that were modified since the snapshot of
+* the run that started the backfill from scratch. That run's snapshot `xmin`
+* (`T_origin`, as an xid8) would be encoded in the timeline, i.e.
+* `timeline = "<relfilenode>:<T_origin>"`, and carried over (rather than
+* replaced) by resumed runs, so that:
+* * Within a chain of runs, every row version with `xmin < T_origin` stayed
+*   in place across all of the runs' snapshots and was therefore scanned
+*   exactly once, and all other rows are re-delivered by the resumed run
+*   (`ctid > M OR xmin >= T_origin`).
+* * Continuations from a different chain (i.e. with a newer `T_origin`) are
+*   not accepted by subscribers, and requests from different chains are
+*   merged into a backfill from scratch, as the timelines differ. Note that
+*   using the _latest_ `T` would be incorrect: rows that moved before `M`
+*   between the earlier snapshots would never be re-delivered.
+* The resumed run would scan the whole table (but only deliver the modified
+* rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which
+* requires falling back to a backfill from scratch once the current xid is
+* 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the
+* `xmin` system column returning the raw xmin of frozen tuples, which should
+* be verified.
+*/
+async function canResume(sql, relationOID, bf) {
+	const [{ toastable, toasted }] = await sql`
+    SELECT
+      EXISTS (
+        SELECT 1 FROM pg_attribute
+          WHERE attrelid = ${relationOID}
+            AND attname IN ${sql(Object.keys(bf.columns))}
+            AND attstorage <> 'p'
+      ) AS toastable,
+      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted
+      FROM pg_class WHERE oid = ${relationOID}`;
+	return !toastable || !toasted;
+}
 //#endregion
 export { streamBackfill };