@rocicorp/zero 1.2.0-canary.0 → 1.2.0-canary.10

package/out/zero-cache/src/services/change-source/common/replica-schema.js

@@ -1,8 +1,8 @@
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
 import { listTables } from "../../../db/lite-tables.js";
-import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { runSchemaMigrations } from "../../../db/migration-lite.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
 //#region ../zero-cache/src/services/change-source/common/replica-schema.ts
 async function initReplica(log, debugName, dbPath, initialSync) {
@@ -123,6 +123,18 @@ var schemaVersionMigrationMap = {
               "metadata" = NULL;
       `);
 		}
+	},
+	13: {
+		migrateSchema: (_, db) => {
+			db.exec(`
+        ALTER TABLE "_zero.replicationState" ADD COLUMN writeTimeMs INTEGER;
+      `);
+		},
+		migrateData: (_, db) => {
+			db.exec(`
+        UPDATE "_zero.replicationState" 
+          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);
+		}
 	}
 };
 Object.keys(schemaVersionMigrationMap).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n      throw new AutoResetSignal(e.message);\n    }\n    throw e;\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  await runSchemaMigrations(\n    log,\n    debugName,\n    dbPath,\n    // setupMigration should never be invoked\n    {\n      migrateSchema: () => {\n        throw new Error(\n          'This should only be called for already synced replicas',\n        );\n      },\n    },\n    schemaVersionMigrationMap,\n  );\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAgBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,GAAG;EAChD,gBAAgB;EACjB;AAED,KAAI;AACF,QAAM,oBACJ,KACA,WACA,QACA,gBACA,0BACD;UACM,GAAG;AACV,MAAI,aAAa,eAAe,EAAE,SAAS,iBACzC,OAAM,IAAI,gBAAgB,EAAE,QAAQ;AAEtC,QAAM;;;AAIV,eAAsB,eACpB,KACA,WACA,QACA;AACA,OAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;AACnB,QAAM,IAAI,MACR,yDACD;IAEJ,EACD,0BACD;;AAGH,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;AACnB,SAAM,IAAI,gBAAgB,kCAAkC;;EAE9D,gBAAgB;EACjB;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AACxB,MAAG,KAAK,4BAA4B;;EAEtC,cAAc,GAAG,OAAO;AACtB,eAAY,IAAI,UAAU;;EAE7B;CAGD,GAAG,EAAE;CAEL,GAAG,EACD,gBAAgB,GAAG,OAAO;AAIxB,KAAG,KAAK,gCAAgC;AAExC,KAAG,KAAK,qBAAqB;IAEhC;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AAOxB,OAAI,CANgB,GACjB,QACC,sFACD,CACA,KAAK,CAGN,IAAG,KAAK,gCAAgC;;EAG5C,cAAc,GAAG,OAAO;AAItB,MAAG,KAAa,sCAAsC;AAGtD,8BAA2B,IADZ,WAAW,IAAI,OAAO,MAAM,CACL;;EAEzC;CAED,GAAG,EACD,gBAAgB,GAAG,OAAO;AACxB,KAAG,KACO;;;;;UAKN,+BACH;IAEJ;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;QAGd;IAEL;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;;;;;;;;QAUd;IAEL;CAED,IAAI;EACF,gBAAgB,GAAG,OAAO;AAGxB,MAAG,KAAa;;;QAGd;;EAGJ,cAAc,GAAG,OAAO;AAEtB,MAAG,KAAa;;;;QAId;;EAEL;CACF;AAGqC,OAAO,KAC3C,0BACD,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK,CAAC,EAAE,EAAE"}
+{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    if (e instanceof SqliteError && e.code === 'SQLITE_CORRUPT') {\n      throw new AutoResetSignal(e.message);\n    }\n    throw e;\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  await runSchemaMigrations(\n    log,\n    debugName,\n    dbPath,\n    // setupMigration should never be invoked\n    {\n      migrateSchema: () => {\n        throw new Error(\n          'This should only be called for already synced replicas',\n        );\n      },\n    },\n    schemaVersionMigrationMap,\n  );\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAgBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,GAAG;EAChD,gBAAgB;EACjB;AAED,KAAI;AACF,QAAM,oBACJ,KACA,WACA,QACA,gBACA,0BACD;UACM,GAAG;AACV,MAAI,aAAa,eAAe,EAAE,SAAS,iBACzC,OAAM,IAAI,gBAAgB,EAAE,QAAQ;AAEtC,QAAM;;;AAIV,eAAsB,eACpB,KACA,WACA,QACA;AACA,OAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;AACnB,QAAM,IAAI,MACR,yDACD;IAEJ,EACD,0BACD;;AAGH,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;AACnB,SAAM,IAAI,gBAAgB,kCAAkC;;EAE9D,gBAAgB;EACjB;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AACxB,MAAG,KAAK,4BAA4B;;EAEtC,cAAc,GAAG,OAAO;AACtB,eAAY,IAAI,UAAU;;EAE7B;CAGD,GAAG,EAAE;CAEL,GAAG,EACD,gBAAgB,GAAG,OAAO;AAIxB,KAAG,KAAK,gCAAgC;AAExC,KAAG,KAAK,qBAAqB;IAEhC;CAED,GAAG;EACD,gBAAgB,GAAG,OAAO;AAOxB,OAAI,CANgB,GACjB,QACC,sFACD,CACA,KAAK,CAGN,IAAG,KAAK,gCAAgC;;EAG5C,cAAc,GAAG,OAAO;AAItB,MAAG,KAAa,sCAAsC;AAGtD,8BAA2B,IADZ,WAAW,IAAI,OAAO,MAAM,CACL;;EAEzC;CAED,GAAG,EACD,gBAAgB,GAAG,OAAO;AACxB,KAAG,KACO;;;;;UAKN,+BACH;IAEJ;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;QAGd;IAEL;CAED,IAAI,EACF,gBAAgB,GAAG,OAAO;AACxB,KAAG,KAAa;;;;;;;;;;QAUd;IAEL;CAED,IAAI;EACF,gBAAgB,GAAG,OAAO;AAGxB,MAAG,KAAa;;;QAGd;;EAGJ,cAAc,GAAG,OAAO;AAEtB,MAAG,KAAa;;;;QAId;;EAEL;CAED,IAAI;EACF,gBAAgB,GAAG,OAAO;AACxB,MAAG,KAAa;;QAEd;;EAGJ,cAAc,GAAG,OAAO;AACtB,MAAG,KAAa;;+EAEyD;;EAE5E;CACF;AAGqC,OAAO,KAC3C,0BACD,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK,CAAC,EAAE,EAAE"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -7,12 +7,12 @@ import { Database } from "../../../../../zqlite/src/db.js";
 import { StatementRunner } from "../../../db/statements.js";
 import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
 import { computeZqlSpecs } from "../../../db/lite-tables.js";
-import { stream } from "../../../types/streams.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { initReplica } from "../common/replica-schema.js";
+import { stream } from "../../../types/streams.js";
 import { ChangeProcessor } from "../../replicator/change-processor.js";
 import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
-import { initReplica } from "../common/replica-schema.js";
-import { WebSocket as WebSocket$1 } from "ws";
+import { WebSocket } from "ws";
 //#region ../zero-cache/src/services/change-source/custom/change-source.ts
 /**
 * Initializes a Custom change source before streaming changes from the
@@ -66,7 +66,7 @@ var CustomChangeSource = class {
 			url.searchParams.set("lastWatermark", clientWatermark);
 			url.searchParams.set("replicaVersion", replicaVersion);
 		}
-		const ws = new WebSocket$1(url);
+		const ws = new WebSocket(url);
 		const { instream, outstream } = stream(this.#lc, ws, changeStreamMessageSchema, { coalesce: (curr) => curr });
 		return {
 			changes: instream,

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n  type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = [...shard.publications].sort();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            [...publications].sort(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM;EAChD,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,CAAC,GAAG,aAAa,CAAC,MAAM,EACxB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}
+{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n  type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = [...shard.publications].sort();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            [...publications].sort(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM;EAChD,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,UAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,CAAC,GAAG,aAAa,CAAC,MAAM,EACxB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA2BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;AAqaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAitBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA6BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;AA+bD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AA8vBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -10,7 +10,7 @@ import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { StatementRunner } from "../../../db/statements.js";
-import { pgClient } from "../../../types/pg.js";
+import { isPostgresError, pgClient } from "../../../types/pg.js";
 import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
 import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
 import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
@@ -30,14 +30,15 @@ import { streamBackfill } from "./backfill-stream.js";
 import { subscribe } from "./logical-replication/stream.js";
 import postgres from "postgres";
 import { nanoid } from "nanoid";
-import { PG_ADMIN_SHUTDOWN, PG_OBJECT_IN_USE } from "@drdgvhbh/postgres-error-codes";
+import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE, PG_OBJECT_IN_USE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
+var PG_17 = 17e4;
 /**
 * Initializes a Postgres change source, including the initial sync of the
 * replica, before streaming changes from the corresponding logical replication
 * stream.
 */
-async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs) {
+async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0) {
 	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => initialSync(log, shard, tx, upstreamURI, syncOptions, context));
 	const replica = new Database(lc, replicaDbFile);
 	const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
@@ -46,7 +47,7 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
 	try {
 		return {
 			subscriptionState,
-			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, await checkAndUpdateUpstream(lc, db, shard, subscriptionState), context, lagReportIntervalMs ?? null)
+			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, await checkAndUpdateUpstream(lc, db, shard, subscriptionState), context, lagReportIntervalMs)
 		};
 	} finally {
 		await db.end();
@@ -100,10 +101,18 @@ var PostgresChangeSource = class {
 		this.#shard = shard;
 		this.#replica = replica;
 		this.#context = context;
-		this.#lagReporter = lagReportIntervalMs ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
+		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
 	}
-	startLagReporter() {
-		return this.#lagReporter ? this.#lagReporter.initiateLagReport(true) : null;
+	async startLagReporter() {
+		if (this.#lagReporter) try {
+			return await this.#lagReporter.initiateLagReport(true);
+		} catch (e) {
+			if (isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
+				const functionName = (this.#lagReporter.pgVersion ?? 0) >= PG_17 ? "pg_logical_emit_message(boolean, text, text, boolean)" : "pg_logical_emit_message(boolean, text, text)";
+				this.#lc.warn?.("\n\nUnable to initiate replication lag reports due to insufficient privileges.\nTo enable replication lag reporting, run:", `\n\tGRANT EXECUTE ON FUNCTION ${functionName} TO <your_db_user>;\n\n`, e);
+			} else this.#lc.error?.(`Unexpected error while initiating lag reports. Lag reports will be disabled.`, e);
+		}
+		return null;
 	}
 	async startStream(clientWatermark, backfillRequests = []) {
 		const { slot } = this.#replica;
@@ -127,10 +136,11 @@ var PostgresChangeSource = class {
 		*/
 		const isTransactionalMessage = (lsn, msg) => {
 			if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
-				changes.pushStatus(this.#lagReporter.processLagReport(msg));
+				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg));
 				return false;
 			}
-			this.#lagReporter?.checkCurrentLSN(lsn);
+			const status = this.#lagReporter?.checkCurrentLSN(lsn);
+			if (status) changes.pushStatus(status);
 			if (msg.tag === "keepalive") {
 				changes.pushStatus([
 					"status",
@@ -328,50 +338,80 @@ var LagReporter = class LagReporter {
 		}
 		return this.#pgVersion;
 	}
+	get pgVersion() {
+		return this.#pgVersion;
+	}
 	async initiateLagReport(log = false) {
 		const pgVersion = this.#pgVersion ?? await this.#getPgVersion();
 		const now = Date.now();
 		const id = nanoid();
 		const lagReport = {
 			id,
+			sendTimeMs: now,
+			commitTimeMs: now,
 			lsn: 0n
 		};
 		this.#expectingLagReport = lagReport;
+		let commitTimeMs;
 		let lsn;
-		if (pgVersion >= 17e4) [{lsn}] = await this.#db`
-        SELECT pg_logical_emit_message(
+		if (pgVersion >= PG_17) [{commitTimeMs, lsn}] = await this.#db`
+        WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
+        SELECT "commitTimeMs", pg_logical_emit_message(
           false,
           ${this.messagePrefix},
           json_build_object(
             'id', ${id}::text,
             'sendTimeMs', ${now}::int8,
-            'commitTimeMs', extract(epoch from now()) * 1000
+            'commitTimeMs', "commitTimeMs"
           )::text,
           true
-        ) as lsn;
+        ) as lsn FROM CTE;
     `;
-		else [{lsn}] = await this.#db`
-        SELECT pg_logical_emit_message(
+		else [{commitTimeMs, lsn}] = await this.#db`
+        WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
+        SELECT "commitTimeMs", pg_logical_emit_message(
           false,
           ${this.messagePrefix},
           json_build_object(
             'id', ${id}::text,
             'sendTimeMs', ${now}::int8,
-            'commitTimeMs', extract(epoch from now()) * 1000
+            'commitTimeMs', "commitTimeMs"
           )::text
-        ) as lsn;
+        ) as lsn FROM CTE;
     `;
 		lagReport.lsn = toBigInt(lsn);
+		lagReport.commitTimeMs = commitTimeMs;
 		if (log) this.#lc.info?.(`initiated lag report at lsn ${lsn}`, {
 			id,
-			lsn
+			lsn,
+			sendTimeMs: now,
+			commitTimeMs
 		});
 		return { nextSendTimeMs: now };
 	}
+	/**
+	* In Postgres < 17, the pg_logical_emit_message lacks an immediate "flush"
+	* option, which can cause messages to be missed when the replication stream
+	* starts up:
+	*
+	* ```
+	* * emit message → WAL write (buffered, not flushed)
+	* * walsender reads up to current flush LSN
+	* * emitted message's LSN is beyond flush LSN → not yet visible
+	* * stream feedback/acknowledgment advances slot
+	* * WAL eventually flushes → but slot has already moved past it
+	* ```
+	*
+	* This has been seen to happen for the initial `wal_writer_delay` interval
+	* of a replication session.
+	*
+	* To account for this, the last emitted lag report is considered "received"
+	* if the stream has advanced beyond the LSN of the report.
+	*/
 	checkCurrentLSN(lsn) {
 		if (this.#expectingLagReport?.lsn && lsn > this.#expectingLagReport.lsn) {
-			this.#lc.warn?.(`LSN ${fromBigInt(lsn)} is passed expected lag report ${fromBigInt(this.#expectingLagReport.lsn)}. Initiating new report.`);
-			this.#scheduleNextReport(0);
+			this.#lc.info?.(`LSN ${fromBigInt(lsn)} is passed expected lag report ${fromBigInt(this.#expectingLagReport.lsn)}. Processing it as received.`);
+			return this.#processLagReport(this.#expectingLagReport, majorVersionToString(lsn));
 		}
 	}
 	#scheduleNextReport(delayMs) {
@@ -386,9 +426,12 @@ var LagReporter = class LagReporter {
 			}
 		}, delayMs);
 	}
-	processLagReport(msg) {
+	processLagReportMessage(msg) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
 		const report = parseLogicalMessageContent(msg, lagReportSchema);
+		return this.#processLagReport(report, toStateVersionString(msg.messageLsn ?? "0/0"));
+	}
+	#processLagReport(report, watermark) {
 		const now = Date.now();
 		const nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
 		if (report.id === this.#expectingLagReport?.id) this.#scheduleNextReport(nextSendTimeMs - now);
@@ -407,7 +450,7 @@ var LagReporter = class LagReporter {
 					nextSendTimeMs
 				}
 			},
-			{ watermark: toStateVersionString(msg.messageLsn ?? "0/0") }
+			{ watermark }
 		];
 	}
 };