@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAEL,KAAK,uBAAuB,EAE7B,MAAM,+BAA+B,CAAC;AAgBvC,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,iBAoBf;AAkBD,eAAO,MAAM,+BAA+B,0UAW3C,CAAC;AAEF,eAAO,MAAM,oBAAoB,8XAUhC,CAAC;AAEF,eAAO,MAAM,8BAA8B,sMAO1C,CAAC;AAKF,eAAO,MAAM,2BAA2B,0BAA0B,CAAC;AAMnE,eAAO,MAAM,4BAA4B,8aAaxC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,uBAmMvC,CAAC;AAGF,eAAO,MAAM,sBAAsB,QAEwB,CAAC"}
+{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAEL,KAAK,uBAAuB,EAE7B,MAAM,+BAA+B,CAAC;AAgBvC,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,iBAoBf;AAkBD,eAAO,MAAM,+BAA+B,0UAW3C,CAAC;AAEF,eAAO,MAAM,oBAAoB,8XAUhC,CAAC;AAEF,eAAO,MAAM,8BAA8B,sMAO1C,CAAC;AAEF,eAAO,MAAM,4BAA4B,8OAQxC,CAAC;AAKF,eAAO,MAAM,2BAA2B,0BAA0B,CAAC;AAMnE,eAAO,MAAM,4BAA4B,8aAaxC,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,uBAkNvC,CAAC;AAGF,eAAO,MAAM,sBAAsB,QAEwB,CAAC"}

package/out/zero-cache/src/services/change-source/common/replica-schema.js

@@ -1,6 +1,6 @@
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { listTables } from "../../../db/lite-tables.js";
-import { CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
+import { BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
 import { deleteLiteDB } from "../../../db/delete-lite-db.js";
@@ -70,6 +70,15 @@ var CREATE_V9_TABLE_METADATA_TABLE = `
     PRIMARY KEY ("schema", "table")
   );
 `;
+var CREATE_V17_BACKFILLING_TABLE = `
+  CREATE TABLE "${BACKFILLING_TABLE}" (
+    "schema"   TEXT NOT NULL,
+    "table"    TEXT NOT NULL,
+    "column"   TEXT NOT NULL,
+    "backfill" TEXT NOT NULL,
+    PRIMARY KEY ("schema", "table", "column")
+  );
+`;
 var V14_CHANGE_LOG_STREAM_TABLE = "_zero.changeLogStream";
 var V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX = "_zero.changeLogStream_writeTimeMs";
 `${V14_CHANGE_LOG_STREAM_TABLE}${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}${V14_CHANGE_LOG_STREAM_TABLE}`;
@@ -181,12 +190,15 @@ var schemaVersionMigrationMap = {
 	} },
 	17: {
 		migrateSchema: (_, db) => {
-			db.exec(CREATE_BACKFILLING_TABLE);
+			db.exec(CREATE_V17_BACKFILLING_TABLE);
 		},
 		migrateData: (lc, db) => {
 			populateBackfillingFromColumnMetadata(lc, db);
 		}
-	}
+	},
+	18: { migrateSchema: (_, db) => {
+		db.exec(`ALTER TABLE "${BACKFILLING_TABLE}" ADD COLUMN "progress" TEXT`);
+	} }
 };
 Object.keys(schemaVersionMigrationMap).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);
 //#endregion

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import {existsSync, renameSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../../db/delete-lite-db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  CREATE_BACKFILLING_TABLE,\n  populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const isInitialSync = !existsSync(dbPath);\n  const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;\n  if (isInitialSync) {\n    deleteLiteDB(migrationPath);\n  }\n\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      migrationPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n    if (isInitialSync) {\n      renameSync(migrationPath, dbPath);\n    }\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, migrationPath, e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nfunction throwAutoResetForCorruption(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  e: unknown,\n): never {\n  if (isSQLiteCorruption(e)) {\n    logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// describe replica files created by pre-release v14 builds, while that module\n// describes the separate change-log database. The two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table created by pre-release v14 builds and dropped by\n// migration 16. Retained at its old shape for migration tests only.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /*sql*/ `\n  CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"   TEXT NOT NULL,\n    \"pos\"         INTEGER NOT NULL,\n    \"change\"      TEXT NOT NULL,\n    \"precommit\"   TEXT,\n    \"writeTimeMs\" INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n\n  // The replica-local change log moved to its own database before v14 shipped.\n  14: {},\n\n  15: {\n    migrateSchema: (_, db) => {\n      // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n      // create a new table, copy the data over, delete the old table, and\n      // rename. This is cheap because it's a single row table.\n      db.exec(/*sql*/ `\n        CREATE TABLE \"_zero.replicationState2\" (\n          stateVersion TEXT NOT NULL,\n          writeTimeMs INTEGER NOT NULL,\n          lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n        );\n\n        INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n          SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n        DROP TABLE \"_zero.replicationState\";\n\n        ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n      `);\n    },\n  },\n\n  // The change log moved out of the replica and into its own database\n  // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n  // every file the backup ships and every follower downloads. Nothing reads it\n  // as of this version.\n  //\n  // `IF EXISTS` because fresh and post-v13 replicas never create this table.\n  // Pre-release v14 builds did, so keep their cleanup idempotent.\n  //\n  // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n  // trigger in `workers/replicator.ts`; no special handling here.\n  //\n  // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n  // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n  // table. Bumping the rollback limit would turn every rollback into a hard\n  // startup failure to protect a configuration that is not enabled anywhere.\n  16: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n        DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n      `);\n    },\n  },\n\n  // `_zero.column_metadata.backfill` records which columns are backfilling but\n  // is keyed by lite table name, which has no inverse: it cannot be turned back\n  // into the `BackfillRequest`s that a change log initialized from this replica\n  // would have to send. This table carries the upstream identity alongside the\n  // same state. See `replicator/schema/backfilling.ts`.\n  //\n  // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n  // since nothing at v16 reads the new table. Its writes to\n  // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n  // forward again, so a rollback costs nothing but the re-seed.\n  17: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_BACKFILLING_TABLE);\n    },\n\n    migrateData: (lc, db) => {\n      populateBackfillingFromColumnMetadata(lc, db);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;AAyBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,gBAAgB,CAAC,WAAW,MAAM;CACxC,MAAM,gBAAgB,gBAAgB,GAAG,OAAO,QAAQ;CACxD,IAAI,eACF,aAAa,aAAa;CAG5B,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,eACA,gBACA,yBACF;EACA,IAAI,eACF,WAAW,eAAe,MAAM;CAEpC,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,eAAe,CAAC;CAC9D;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAYtD,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAIkD,GAClC,4BADkC,EAUlC,uCAVkC,EAW1C,4BAX0C;AAepD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAGA,IAAI,CAAC;CAEL,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAiBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,wBAAwB;EAClC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
+{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import {existsSync, renameSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../../db/delete-lite-db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  BACKFILLING_TABLE,\n  populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const isInitialSync = !existsSync(dbPath);\n  const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;\n  if (isInitialSync) {\n    deleteLiteDB(migrationPath);\n  }\n\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      migrationPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n    if (isInitialSync) {\n      renameSync(migrationPath, dbPath);\n    }\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, migrationPath, e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nfunction throwAutoResetForCorruption(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  e: unknown,\n): never {\n  if (isSQLiteCorruption(e)) {\n    logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const CREATE_V17_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// describe replica files created by pre-release v14 builds, while that module\n// describes the separate change-log database. The two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table created by pre-release v14 builds and dropped by\n// migration 16. Retained at its old shape for migration tests only.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /*sql*/ `\n  CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"   TEXT NOT NULL,\n    \"pos\"         INTEGER NOT NULL,\n    \"change\"      TEXT NOT NULL,\n    \"precommit\"   TEXT,\n    \"writeTimeMs\" INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n\n  // The replica-local change log moved to its own database before v14 shipped.\n  14: {},\n\n  15: {\n    migrateSchema: (_, db) => {\n      // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n      // create a new table, copy the data over, delete the old table, and\n      // rename. This is cheap because it's a single row table.\n      db.exec(/*sql*/ `\n        CREATE TABLE \"_zero.replicationState2\" (\n          stateVersion TEXT NOT NULL,\n          writeTimeMs INTEGER NOT NULL,\n          lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n        );\n\n        INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n          SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n        DROP TABLE \"_zero.replicationState\";\n\n        ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n      `);\n    },\n  },\n\n  // The change log moved out of the replica and into its own database\n  // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n  // every file the backup ships and every follower downloads. Nothing reads it\n  // as of this version.\n  //\n  // `IF EXISTS` because fresh and post-v13 replicas never create this table.\n  // Pre-release v14 builds did, so keep their cleanup idempotent.\n  //\n  // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n  // trigger in `workers/replicator.ts`; no special handling here.\n  //\n  // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n  // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n  // table. Bumping the rollback limit would turn every rollback into a hard\n  // startup failure to protect a configuration that is not enabled anywhere.\n  16: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n        DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n      `);\n    },\n  },\n\n  // `_zero.column_metadata.backfill` records which columns are backfilling but\n  // is keyed by lite table name, which has no inverse: it cannot be turned back\n  // into the `BackfillRequest`s that a change log initialized from this replica\n  // would have to send. This table carries the upstream identity alongside the\n  // same state. See `replicator/schema/backfilling.ts`.\n  //\n  // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n  // since nothing at v16 reads the new table. Its writes to\n  // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n  // forward again, so a rollback costs nothing but the re-seed.\n  17: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_V17_BACKFILLING_TABLE);\n    },\n\n    migrateData: (lc, db) => {\n      populateBackfillingFromColumnMetadata(lc, db);\n    },\n  },\n\n  // Adds `_zero.backfilling.progress`, with which the replica tracks the\n  // progress of its pending backfills and reports it to the change-streamer\n  // (protocol v8).\n  //\n  // No `minSafeVersion`: an older zero-cache ignores the column (its writes\n  // name their columns explicitly), and a `progress` left stale by one is at\n  // or before the actual progress, which only results in redundant backfill.\n  18: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `ALTER TABLE \"${BACKFILLING_TABLE}\" ADD COLUMN \"progress\" TEXT`,\n      );\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;AAyBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,gBAAgB,CAAC,WAAW,MAAM;CACxC,MAAM,gBAAgB,gBAAgB,GAAG,OAAO,QAAQ;CACxD,IAAI,eACF,aAAa,aAAa;CAG5B,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,eACA,gBACA,yBACF;EACA,IAAI,eACF,WAAW,eAAe,MAAM;CAEpC,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,eAAe,CAAC;CAC9D;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,+BAAuC;kBAClC,kBAAkB;;;;;;;;AAYpC,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAIkD,GAClC,4BADkC,EAUlC,uCAVkC,EAW1C,4BAX0C;AAepD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAGA,IAAI,CAAC;CAEL,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAiBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,4BAA4B;EACtC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;CASA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO,gBAAgB,kBAAkB,6BAC5C;CACF,EACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts

@@ -2,6 +2,9 @@ import type { LogContext } from '@rocicorp/logger';
 import { type BackfillMessage } from '../common/backfill-manager.ts';
 import type { BackfillRequest } from '../protocol/current.ts';
 import type { Replica } from './schema/shard.ts';
+export declare function ctidToProgressMark(ctid: string): string;
+/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */
+export declare function progressMarkToCtid(mark: string): string;
 type StreamOptions = {
     /**
      * The number of bytes at which to flush a batch of rows in a
@@ -15,6 +18,12 @@ type StreamOptions = {
      * revert to the old code path if needed.
      */
     textCopy?: boolean | undefined;
+    /**
+     * @visibleForTesting
+     * Called after the snapshot transaction is opened, before the table is
+     * locked, e.g. to run a concurrent heap rewrite in between.
+     */
+    afterSnapshotForTesting?: (() => Promise<void>) | undefined;
 };
 /**
  * Streams a series of `backfill` messages (ending with `backfill-complete`)

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAkBjD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAEV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAChC,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,CAAC,CAsGjC"}
+{"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAqBjD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,EAGV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAoC/C,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIvD;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIvD;AAmCD,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE/B;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;CAC7D,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,CAAC,CAwIjC"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -1,11 +1,15 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { parse } from "../../../../../shared/src/valita.js";
+import { must } from "../../../../../shared/src/must.js";
+import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
+import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { Queue } from "../../../../../shared/src/queue.js";
+import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
 import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
 import { TsvParser } from "../../../db/pg-copy.js";
@@ -17,6 +21,52 @@ import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js
 import postgres from "postgres";
 import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
+/**
+* Encodes a `ctid` text representation, e.g. `(123,4)`, as a
+* fixed-width string that sorts lexicographically in `ctid` order.
+*/
+var CTID_RE = /^\((\d+),(\d+)\)$/;
+var CTID_MARK_RE = /^(\d{10})\.(\d{5})$/;
+function ctidToProgressMark(ctid) {
+	const match = CTID_RE.exec(ctid);
+	assert(match, () => `invalid ctid: ${ctid}`);
+	return `${match[1].padStart(10, "0")}.${match[2].padStart(5, "0")}`;
+}
+/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */
+function progressMarkToCtid(mark) {
+	const match = CTID_MARK_RE.exec(mark);
+	assert(match, () => `invalid ctid progress mark: ${mark}`);
+	return `(${Number(match[1])},${Number(match[2])})`;
+}
+function progressMarks(previous, current) {
+	return previous ? {
+		previous,
+		current
+	} : { current };
+}
+/** The `ctid` of each row is appended as the last column of the download. */
+var CTID_SELECT = "ctid::text";
+/**
+* Settings that ensure that the COPY scans the table in physical (i.e.
+* `ctid`) order, which is necessary for the progress marks of a resumable
+* backfill to be meaningful. (Monotonicity is nonetheless verified as rows
+* are streamed.)
+*
+* * Synchronized seqscans can start in the middle of the table.
+* * Parallel scans interleave the rows of different workers.
+* * Index scans follow index order. (A publication row filter could
+*   otherwise be satisfied with an index.)
+*/
+var PHYSICAL_ORDER_SETTINGS = [
+	"synchronize_seqscans = off",
+	"max_parallel_workers_per_gather = 0",
+	"enable_indexscan = off",
+	"enable_indexonlyscan = off",
+	"enable_bitmapscan = off"
+];
+function rowCountMark(rows) {
+	return String(rows).padStart(16, "0");
+}
 var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
 var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 /**
@@ -27,7 +77,7 @@ var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 */
 async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
 	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
-	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
+	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false, afterSnapshotForTesting } = opts;
 	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", {
 		["max_lifetime"]: 1440 * 60,
 		liveness: "keepalive"
@@ -36,23 +86,26 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
 	let watermark;
 	try {
 		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
-		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
+		await afterSnapshotForTesting?.();
+		const { tableSpec, backfill, progress } = await validateSchema(lc, tx, publications, bf, watermark);
 		const { relation, columns } = backfill;
 		const cols = [...relation.rowKey.columns, ...columns];
-		const stmts = makeDownloadStatements(tableSpec, cols);
+		const order = progress.resumable && progress.start ? { after: `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid` } : void 0;
+		const stmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...cols.map((col) => id(col)), CTID_SELECT], order);
+		if (progress.start) lc.info?.(`resuming backfill from ${progress.start.progressMark} (timeline ${progress.timeline})`);
 		if (textCopy) {
 			const types = await getTypeParsers(db, { returnJsonAsString: true });
-			yield* stream(lc, tx, backfill, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
 				const parser = types.getTypeParser(tableSpec.columns[col].typeOID);
 				return (text) => parser(text);
-			}), flushThresholdBytes);
+			}), (text) => text, flushThresholdBytes);
 		} else {
-			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, makeBinarySelectExprs(tableSpec, cols));
-			yield* stream(lc, tx, backfill, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
+			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT], order);
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
 				const spec = tableSpec.columns[col];
 				const decoder = hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder;
 				return (buf) => decoder(buf);
-			}), flushThresholdBytes);
+			}), (buf) => buf.toString("utf8"), flushThresholdBytes);
 		}
 	} catch (e) {
 		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
@@ -62,7 +115,7 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
 		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
 	}
 }
-async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, flushThresholdBytes) {
+async function* stream(lc, tx, backfill, { timeline, resumable, start: startMark }, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, ctidDecoder, flushThresholdBytes) {
 	assert(!SAMPLE_OR_LIMIT_RE.test(copyCommand), `backfill COPY must not sample or limit: ${copyCommand}`);
 	const start = performance.now();
 	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
@@ -76,6 +129,7 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const chunks = new Queue();
 	const acks = new Queue();
 	const copyDone = tx.processReadTask(async (sql) => {
+		for (const setting of PHYSICAL_ORDER_SETTINGS) await sql.unsafe(`SET LOCAL ${setting}`);
 		const readable = await sql.unsafe(copyCommand).readable();
 		for await (const chunk of readable) {
 			chunks.enqueue(chunk);
@@ -91,14 +145,26 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const logFlushed = () => {
 		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
 	};
+	let current = startMark;
+	let previous = startMark;
+	const markOf = (progressMark) => ({
+		progressMark,
+		timeline
+	});
+	const numCols = decoders.length + 1;
 	let row = Array.from({ length: decoders.length });
 	let col = 0;
 	for (;;) {
 		const chunk = await chunks.dequeue();
 		if (chunk === "done") break;
 		for (const field of parser.parse(chunk)) {
-			row[col] = field === null ? null : decoders[col](field);
-			if (++col === decoders.length) {
+			if (col < decoders.length) row[col] = field === null ? null : decoders[col](field);
+			else {
+				const mark = resumable ? ctidToProgressMark(ctidDecoder(field)) : rowCountMark(status.rows + 1);
+				if (current !== void 0 && mark <= current.progressMark) throw new Error(`backfill rows are not in ctid order (${mark} after ${current.progressMark})`);
+				current = markOf(mark);
+			}
+			if (++col === numCols) {
 				rowValues.push(row);
 				status.rows++;
 				row = Array.from({ length: decoders.length });
@@ -107,16 +173,18 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		}
 		bufferedBytes += chunk.byteLength;
 		totalBytes += chunk.byteLength;
-		if (bufferedBytes >= flushThresholdBytes) {
+		if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {
 			yield {
 				message: {
 					tag: "backfill",
 					...backfill,
 					rowValues,
-					status
+					status,
+					progressMarks: progressMarks(previous, must(current))
 				},
 				byteSize: bufferedBytes
 			};
+			previous = current;
 			totalMsgs++;
 			logFlushed();
 			rowValues = [];
@@ -131,10 +199,12 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 				tag: "backfill",
 				...backfill,
 				rowValues,
-				status
+				status,
+				progressMarks: progressMarks(previous, must(current))
 			},
 			byteSize: bufferedBytes
 		};
+		previous = current;
 		totalMsgs++;
 		logFlushed();
 	}
@@ -142,7 +212,8 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		message: {
 			tag: "backfill-completed",
 			...backfill,
-			status
+			status,
+			progressMarks: previous ? { previous } : {}
 		},
 		byteSize: 0
 	};
@@ -185,7 +256,7 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 		await replicationSession.end();
 	}
 }
-function validateSchema(tx, publications, bf, watermark) {
+function validateSchema(lc, tx, publications, bf, watermark) {
 	return tx.processReadTask(async (sql) => {
 		const { tables } = await getPublicationInfo(sql, publications);
 		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
@@ -194,7 +265,7 @@ function validateSchema(tx, publications, bf, watermark) {
 		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
 		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
 		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
-		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
+		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns).map(([col, { id }]) => [col, id])];
 		for (const [col, val] of allCols) {
 			const colSpec = spec.columns[col];
 			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
@@ -210,10 +281,125 @@ function validateSchema(tx, publications, bf, watermark) {
 				},
 				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
 				watermark
-			}
+			},
+			progress: await getProgress(lc, sql, spec.oid, bf)
 		};
 	});
 }
+/**
+* Determines the timeline of the table's progress marks (as of the snapshot),
+* and whether the backfill can be resumed from the requested progress.
+*
+* The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest
+* of the transaction) so that its storage cannot be rewritten (by `VACUUM
+* FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which
+* require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note
+* that `pg_relation_filenode()` reflects the table's current storage (only
+* refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at
+* the transaction snapshot. If they differ, the storage was rewritten after the
+* snapshot, and the COPY would scan the new storage with the old snapshot,
+* which is not MVCC-safe for all rewrites (e.g. the table appears empty after
+* a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.
+*/
+async function getProgress(lc, sql, relationOID, bf) {
+	const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;
+	await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);
+	const [{ lockedOID, relkind, snapshotFilenode, filenode }] = await sql`
+    SELECT to_regclass(${table})::oid::int8 AS "lockedOID",
+           relkind,
+           relfilenode::text AS "snapshotFilenode",
+           pg_relation_filenode(oid)::text AS filenode
+      FROM pg_class WHERE oid = ${relationOID}`;
+	if (Number(lockedOID) !== relationOID) throw new SchemaIncompatibilityError(bf, `Table has been renamed or replaced`);
+	if (relkind !== "r" || filenode === null) return {
+		timeline: `${relkind}:${randomCharacters(12)}`,
+		resumable: false,
+		start: void 0
+	};
+	if (filenode !== snapshotFilenode) throw new Error(`${bf.table.schema}.${bf.table.name} was rewritten after the backfill snapshot (relfilenode ${snapshotFilenode} => ${filenode})`);
+	const timeline = filenode;
+	const requested = resumePoint(bf);
+	if (requested && requested.timeline !== timeline) {
+		lc.info?.(`restarting backfill from scratch: the requested progress is on timeline ${requested.timeline} but the table is on ${timeline}`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	if (requested && !await canResume(sql, relationOID, bf)) {
+		lc.info?.(`restarting backfill from scratch: the table has TOASTed values and backfilling columns [${Object.keys(bf.columns).join(",")}] are TOAST-able`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	return {
+		timeline,
+		resumable: true,
+		start: requested
+	};
+}
+/**
+* Whether resuming a backfill (in a new snapshot) is guaranteed to deliver
+* the values of every row that the preceding snapshot(s) had not reached.
+*
+* Within a single snapshot, every row is scanned exactly once, since the
+* version visible to the snapshot stays in place while it is held. Across
+* snapshots, however, a row that had not yet been reached (i.e. beyond the
+* resume mark `M`) can be updated between the snapshots, with its new
+* version placed at or before `M` (a non-HOT update into free space in an
+* earlier page, or a HOT update into a lower line pointer of `M`'s page).
+* Neither snapshot then scans the row, and it is only delivered by the
+* replication stream. The stream's UPDATE contains the full row, _except_
+* for unchanged values stored out-of-line in TOAST, which pgoutput omits
+* (unless the table has REPLICA IDENTITY FULL). A backfilling column with
+* such a value would be published without it.
+*
+* Resuming is therefore only done if no backfilling column can be TOASTed
+* (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the
+* table has no TOAST data. The latter is safe because an omitted value must
+* still be in the TOAST relation at the new snapshot (unless its row has since
+* been deleted, which the replication stream does deliver), and conservative
+* because the TOAST relation does not shrink to empty without a heap rewrite,
+* which changes the timeline.
+*
+* Potential optimization for TOAST-able backfills: have the resumed run also
+* deliver the rows at or before `M` that were modified since the snapshot of
+* the run that started the backfill from scratch. That run's snapshot `xmin`
+* (`T_origin`, as an xid8) would be encoded in the timeline, i.e.
+* `timeline = "<relfilenode>:<T_origin>"`, and carried over (rather than
+* replaced) by resumed runs, so that:
+* * Within a chain of runs, every row version with `xmin < T_origin` stayed
+*   in place across all of the runs' snapshots and was therefore scanned
+*   exactly once, and all other rows are re-delivered by the resumed run
+*   (`ctid > M OR xmin >= T_origin`).
+* * Continuations from a different chain (i.e. with a newer `T_origin`) are
+*   not accepted by subscribers, and requests from different chains are
+*   merged into a backfill from scratch, as the timelines differ. Note that
+*   using the _latest_ `T` would be incorrect: rows that moved before `M`
+*   between the earlier snapshots would never be re-delivered.
+* The resumed run would scan the whole table (but only deliver the modified
+* rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which
+* requires falling back to a backfill from scratch once the current xid is
+* 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the
+* `xmin` system column returning the raw xmin of frozen tuples, which should
+* be verified.
+*/
+async function canResume(sql, relationOID, bf) {
+	const [{ toastable, toasted }] = await sql`
+    SELECT
+      EXISTS (
+        SELECT 1 FROM pg_attribute
+          WHERE attrelid = ${relationOID}
+            AND attname IN ${sql(Object.keys(bf.columns))}
+            AND attstorage <> 'p'
+      ) AS toastable,
+      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted
+      FROM pg_class WHERE oid = ${relationOID}`;
+	return !toastable || !toasted;
+}
 //#endregion
 export { streamBackfill };