@rocicorp/zero 1.8.0 → 1.9.0-canary.1

package/out/zero-cache/src/services/change-source/common/replica-schema.js

@@ -1,9 +1,9 @@
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { listTables } from "../../../db/lite-tables.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
+import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
 import { runSchemaMigrations } from "../../../db/migration-lite.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
-import { SqliteError } from "@rocicorp/zero-sqlite3";
 //#region ../zero-cache/src/services/change-source/common/replica-schema.ts
 async function initReplica(log, debugName, dbPath, initialSync) {
 	const setupMigration = {
@@ -13,7 +13,7 @@ async function initReplica(log, debugName, dbPath, initialSync) {
 	try {
 		await runSchemaMigrations(log, debugName, dbPath, setupMigration, schemaVersionMigrationMap);
 	} catch (e) {
-		throwAutoResetForCorruption(e);
+		throwAutoResetForCorruption(log, debugName, dbPath, e);
 	}
 }
 async function upgradeReplica(log, debugName, dbPath) {
@@ -22,11 +22,14 @@ async function upgradeReplica(log, debugName, dbPath) {
 			throw new Error("This should only be called for already synced replicas");
 		} }, schemaVersionMigrationMap);
 	} catch (e) {
-		throwAutoResetForCorruption(e);
+		throwAutoResetForCorruption(log, debugName, dbPath, e);
 	}
 }
-function throwAutoResetForCorruption(e) {
-	if (e instanceof SqliteError && e.code === "SQLITE_CORRUPT") throw new AutoResetSignal(`replica database appears corrupt: ${String(e)}`, { cause: e });
+function throwAutoResetForCorruption(log, debugName, dbPath, e) {
+	if (isSQLiteCorruption(e)) {
+		logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);
+		throw new AutoResetSignal(`replica database appears corrupt: ${String(e)}`, { cause: e });
+	}
 	throw e;
 }
 var CREATE_V6_COLUMN_METADATA_TABLE = `

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(e);\n  }\n}\n\nfunction throwAutoResetForCorruption(e: unknown): never {\n  if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAgBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,CAAC;CAC/B;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,CAAC;CAC/B;AACF;AAEA,SAAS,4BAA4B,GAAmB;CACtD,IAAI,aAAa,eAAe,EAAE,SAAS,kBACzC,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CAEF,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
+{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nfunction throwAutoResetForCorruption(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  e: unknown,\n): never {\n  if (isSQLiteCorruption(e)) {\n    logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAmBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts

@@ -2,14 +2,16 @@ import type { LogContext } from '@rocicorp/logger';
 import * as v from '../../../../../shared/src/valita.ts';
 import type { PublishedTableSpec } from '../../../db/specs.ts';
 import { type LexiVersion } from '../../../types/lexi-version.ts';
-import { type ShardConfig } from '../../../types/shards.ts';
+import { type PostgresDB } from '../../../types/pg.ts';
+import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
 import type { Sink } from '../../../types/streams.ts';
 import { type SubscriptionState } from '../../replicator/schema/replication-state.ts';
 import type { ChangeSource } from '../change-source.ts';
 import { type Listener } from '../common/change-stream-multiplexer.ts';
+import type { DownstreamStatusMessage } from '../protocol/current.ts';
 import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
 import { type InitialSyncOptions, type ServerContext } from './initial-sync.ts';
-import type { MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
+import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
 /**
  * Initializes a Postgres change source, including the initial sync of the
  * replica, before streaming changes from the corresponding logical replication
@@ -31,6 +33,38 @@ declare const lagReportSchema: v.ObjectType<{
     commitTimeMs: v.Type<number>;
 }, undefined>;
 export type LagReport = v.Infer<typeof lagReportSchema>;
+export declare class LagReporter {
+    #private;
+    static readonly MESSAGE_SUFFIX = "/lag-report/v1";
+    readonly messagePrefix: string;
+    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
+    get pgVersion(): number | undefined;
+    initiateLagReport(log?: boolean): Promise<{
+        nextSendTimeMs: number;
+    }>;
+    /**
+     * In Postgres < 17, the pg_logical_emit_message lacks an immediate "flush"
+     * option, which can cause messages to be missed when the replication stream
+     * starts up:
+     *
+     * ```
+     * * emit message → WAL write (buffered, not flushed)
+     * * walsender reads up to current flush LSN
+     * * emitted message's LSN is beyond flush LSN → not yet visible
+     * * stream feedback/acknowledgment advances slot
+     * * WAL eventually flushes → but slot has already moved past it
+     * ```
+     *
+     * This has been seen to happen for the initial `wal_writer_delay` interval
+     * of a replication session.
+     *
+     * To account for this, the last emitted lag report is considered "received"
+     * if the stream has advanced beyond the LSN of the report.
+     */
+    checkCurrentLSN(lsn: bigint): DownstreamStatusMessage | undefined;
+    stop(): void;
+    processLagReportMessage(msg: MessageMessage): DownstreamStatusMessage;
+}
 export declare function relationDifferent(a: PublishedTableSpec, b: PostgresRelation): boolean;
 export {};
 //# sourceMappingURL=change-source.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAQhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA0BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AAuaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAqzBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAEhD,OAAO,KAAK,EAEV,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA0BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AAuaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;gBAqB7B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAiBvB,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;IA0EnC;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS;IAcjE,IAAI;IA2CJ,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB;CAiDtE;AAonBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -9,6 +9,7 @@ import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
+import { getOrCreateCounter } from "../../../observability/metrics.js";
 import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../../db/statements.js";
 import { connectPgClient, isPostgresError, pgClient } from "../../../types/pg.js";
@@ -314,6 +315,10 @@ var LagReporter = class LagReporter {
 	messagePrefix;
 	#db;
 	#lagIntervalMs;
+	#lagReportRetries = getOrCreateCounter("replication", "lag_report_retries", {
+		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
+		unit: "{report}"
+	});
 	#pgVersion;
 	#expectingLagReport = null;
 	#timer;
@@ -347,33 +352,39 @@ var LagReporter = class LagReporter {
 		this.#expectingLagReport = lagReport;
 		let commitTimeMs;
 		let lsn;
-		if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
-        WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
-        SELECT "commitTimeMs", pg_logical_emit_message(
-          false,
-          ${this.messagePrefix},
-          json_build_object(
-            'id', ${id}::text,
-            'sendTimeMs', ${now}::int8,
-            'commitTimeMs', "commitTimeMs"
-          )::text,
-          true
-        ) as lsn FROM CTE;
-    `;
-		else [{commitTimeMs, lsn}] = await this.#db`
-        WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
-        SELECT "commitTimeMs", pg_logical_emit_message(
-          false,
-          ${this.messagePrefix},
-          json_build_object(
-            'id', ${id}::text,
-            'sendTimeMs', ${now}::int8,
-            'commitTimeMs', "commitTimeMs"
-          )::text
-        ) as lsn FROM CTE;
-    `;
+		try {
+			if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
+          WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
+          SELECT "commitTimeMs", pg_logical_emit_message(
+            false,
+            ${this.messagePrefix},
+            json_build_object(
+              'id', ${id}::text,
+              'sendTimeMs', ${now}::int8,
+              'commitTimeMs', "commitTimeMs"
+            )::text,
+            true
+          ) as lsn FROM CTE;
+      `;
+			else [{commitTimeMs, lsn}] = await this.#db`
+          WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
+          SELECT "commitTimeMs", pg_logical_emit_message(
+            false,
+            ${this.messagePrefix},
+            json_build_object(
+              'id', ${id}::text,
+              'sendTimeMs', ${now}::int8,
+              'commitTimeMs', "commitTimeMs"
+            )::text
+          ) as lsn FROM CTE;
+      `;
+		} catch (e) {
+			if (this.#expectingLagReport?.id === id) this.#expectingLagReport = null;
+			throw e;
+		}
 		lagReport.lsn = toBigInt(lsn);
 		lagReport.commitTimeMs = commitTimeMs;
+		if (this.#expectingLagReport?.id === id) this.#scheduleMissingReportRetry(id);
 		if (log) this.#lc.info?.(`initiated lag report at lsn ${lsn}`, {
 			id,
 			lsn,
@@ -415,6 +426,7 @@ var LagReporter = class LagReporter {
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);
 		this.#timer = setTimeout(async () => {
+			this.#timer = void 0;
 			try {
 				await this.initiateLagReport();
 			} catch (e) {
@@ -423,6 +435,26 @@ var LagReporter = class LagReporter {
 			}
 		}, delayMs);
 	}
+	#scheduleMissingReportRetry(reportID) {
+		clearTimeout(this.#timer);
+		this.#timer = setTimeout(async () => {
+			this.#timer = void 0;
+			const missingReport = this.#expectingLagReport;
+			if (missingReport?.id !== reportID) return;
+			this.#lagReportRetries.add(1);
+			this.#lc.warn?.(`retrying missing lag report`, {
+				id: missingReport.id,
+				lsn: fromBigInt(missingReport.lsn),
+				sendTimeMs: missingReport.sendTimeMs
+			});
+			try {
+				await this.initiateLagReport();
+			} catch (e) {
+				this.#lc.warn?.(`error retrying lag report`, e);
+				this.#scheduleNextReport(this.#lagIntervalMs);
+			}
+		}, this.#lagIntervalMs);
+	}
 	processLagReportMessage(msg) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
 		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
@@ -430,9 +462,15 @@ var LagReporter = class LagReporter {
 	}
 	#processLagReport(report, watermark) {
 		const now = Date.now();
-		const nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-		if (report.id === this.#expectingLagReport?.id) this.#scheduleNextReport(nextSendTimeMs - now);
-		else this.#lc.debug?.(`received extraneous lag report`, { report });
+		const expectedReport = this.#expectingLagReport;
+		let nextSendTimeMs;
+		if (report.id === expectedReport?.id) {
+			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
+			this.#scheduleNextReport(nextSendTimeMs - now);
+		} else {
+			this.#lc.debug?.(`received extraneous lag report`, { report });
+			nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
+		}
 		const { sendTimeMs, commitTimeMs } = report;
 		return [
 			"status",
@@ -581,6 +619,11 @@ var ChangeMaker = class {
 		}
 		const prevEvent = this.#lastReplicationEvent;
 		this.#lastReplicationEvent = event;
+		const { schema } = event;
+		if (schema === void 0) {
+			lc.debug?.(`received context-only ${msg.prefix}/${type} event`, { event: summarizeReplicationEventForLog(event) });
+			return [];
+		}
 		const prevSchema = event.previousSchema === void 0 ? prevEvent?.schema : event.previousSchema;
 		if (!prevSchema) {
 			lc.info?.(`received ${msg.prefix}/${type} event`, { event: summarizeReplicationEventForLog(event) });
@@ -588,9 +631,9 @@ var ChangeMaker = class {
 		}
 		const effectiveTag = prevEvent?.type === "ddlStart" ? prevEvent.event.tag : event.type === "ddlUpdate" ? event.event.tag : "UNKNOWN";
 		lc.info?.(`processing ${effectiveTag} command from ${msg.prefix}/${type}`, { event: summarizeReplicationEventForLog(event) });
-		const changes = this.#makeSchemaChanges(lc, prevSchema, event, effectiveTag).map((change) => ["data", change]);
+		const changes = this.#makeSchemaChanges(lc, prevSchema, schema, event, effectiveTag).map((change) => ["data", change]);
 		lc.info?.(`${changes.length} schema change(s)`, { changes });
-		const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(event.schema);
+		const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(schema);
 		if (replicaIdentities) this.#replicaIdentityTimer = setTimeout(async () => {
 			try {
 				await replicaIdentities.apply(lc, this.#db);
@@ -627,10 +670,10 @@ var ChangeMaker = class {
 	* columns. This, for example, would be needed to properly support changing
 	* the type of a column that's indexed.
 	*/
-	#makeSchemaChanges(lc, preSchema, event, tag) {
+	#makeSchemaChanges(lc, preSchema, nextSchema, event, tag) {
 		try {
 			const [prevTbl, prevIdx] = specsByID(preSchema);
-			const [nextTbl, nextIdx] = specsByID(event.schema);
+			const [nextTbl, nextIdx] = specsByID(nextSchema);
 			const changes = [];
 			for (const table of nextTbl.values()) validate(lc, table);
 			const [droppedIdx, createdIdx] = symmetricDifferences(prevIdx, nextIdx);
@@ -903,7 +946,7 @@ function summarizeReplicationEventForLog(event) {
 	const { previousSchema, ...eventWithoutSchemas } = rest;
 	return {
 		...eventWithoutSchemas,
-		schema: summarizeSchemaForLog(schema),
+		...schema !== void 0 && { schema: summarizeSchemaForLog(schema) },
 		...previousSchema !== void 0 && { previousSchema: previousSchema === null ? null : summarizeSchemaForLog(previousSchema) }
 	};
 }