@rocicorp/zero 1.8.0 → 1.9.0-canary.0

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts

@@ -2,14 +2,16 @@ import type { LogContext } from '@rocicorp/logger';
 import * as v from '../../../../../shared/src/valita.ts';
 import type { PublishedTableSpec } from '../../../db/specs.ts';
 import { type LexiVersion } from '../../../types/lexi-version.ts';
-import { type ShardConfig } from '../../../types/shards.ts';
+import { type PostgresDB } from '../../../types/pg.ts';
+import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
 import type { Sink } from '../../../types/streams.ts';
 import { type SubscriptionState } from '../../replicator/schema/replication-state.ts';
 import type { ChangeSource } from '../change-source.ts';
 import { type Listener } from '../common/change-stream-multiplexer.ts';
+import type { DownstreamStatusMessage } from '../protocol/current.ts';
 import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
 import { type InitialSyncOptions, type ServerContext } from './initial-sync.ts';
-import type { MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
+import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
 /**
  * Initializes a Postgres change source, including the initial sync of the
  * replica, before streaming changes from the corresponding logical replication
@@ -31,6 +33,38 @@ declare const lagReportSchema: v.ObjectType<{
     commitTimeMs: v.Type<number>;
 }, undefined>;
 export type LagReport = v.Infer<typeof lagReportSchema>;
+export declare class LagReporter {
+    #private;
+    static readonly MESSAGE_SUFFIX = "/lag-report/v1";
+    readonly messagePrefix: string;
+    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
+    get pgVersion(): number | undefined;
+    initiateLagReport(log?: boolean): Promise<{
+        nextSendTimeMs: number;
+    }>;
+    /**
+     * In Postgres < 17, the pg_logical_emit_message lacks an immediate "flush"
+     * option, which can cause messages to be missed when the replication stream
+     * starts up:
+     *
+     * ```
+     * * emit message → WAL write (buffered, not flushed)
+     * * walsender reads up to current flush LSN
+     * * emitted message's LSN is beyond flush LSN → not yet visible
+     * * stream feedback/acknowledgment advances slot
+     * * WAL eventually flushes → but slot has already moved past it
+     * ```
+     *
+     * This has been seen to happen for the initial `wal_writer_delay` interval
+     * of a replication session.
+     *
+     * To account for this, the last emitted lag report is considered "received"
+     * if the stream has advanced beyond the LSN of the report.
+     */
+    checkCurrentLSN(lsn: bigint): DownstreamStatusMessage | undefined;
+    stop(): void;
+    processLagReportMessage(msg: MessageMessage): DownstreamStatusMessage;
+}
 export declare function relationDifferent(a: PublishedTableSpec, b: PostgresRelation): boolean;
 export {};
 //# sourceMappingURL=change-source.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAQhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA0BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AAuaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAqzBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAEhD,OAAO,KAAK,EAEV,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA0BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAuC7E;AAuaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;gBAqB7B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM;IAiBvB,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;IA0EnC;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS;IAcjE,IAAI;IA2CJ,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB;CAiDtE;AAonBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -9,6 +9,7 @@ import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
+import { getOrCreateCounter } from "../../../observability/metrics.js";
 import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../../db/statements.js";
 import { connectPgClient, isPostgresError, pgClient } from "../../../types/pg.js";
@@ -314,6 +315,10 @@ var LagReporter = class LagReporter {
 	messagePrefix;
 	#db;
 	#lagIntervalMs;
+	#lagReportRetries = getOrCreateCounter("replication", "lag_report_retries", {
+		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
+		unit: "{report}"
+	});
 	#pgVersion;
 	#expectingLagReport = null;
 	#timer;
@@ -347,33 +352,39 @@ var LagReporter = class LagReporter {
 		this.#expectingLagReport = lagReport;
 		let commitTimeMs;
 		let lsn;
-		if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
-        WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
-        SELECT "commitTimeMs", pg_logical_emit_message(
-          false,
-          ${this.messagePrefix},
-          json_build_object(
-            'id', ${id}::text,
-            'sendTimeMs', ${now}::int8,
-            'commitTimeMs', "commitTimeMs"
-          )::text,
-          true
-        ) as lsn FROM CTE;
-    `;
-		else [{commitTimeMs, lsn}] = await this.#db`
-        WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
-        SELECT "commitTimeMs", pg_logical_emit_message(
-          false,
-          ${this.messagePrefix},
-          json_build_object(
-            'id', ${id}::text,
-            'sendTimeMs', ${now}::int8,
-            'commitTimeMs', "commitTimeMs"
-          )::text
-        ) as lsn FROM CTE;
-    `;
+		try {
+			if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
+          WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
+          SELECT "commitTimeMs", pg_logical_emit_message(
+            false,
+            ${this.messagePrefix},
+            json_build_object(
+              'id', ${id}::text,
+              'sendTimeMs', ${now}::int8,
+              'commitTimeMs', "commitTimeMs"
+            )::text,
+            true
+          ) as lsn FROM CTE;
+      `;
+			else [{commitTimeMs, lsn}] = await this.#db`
+          WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
+          SELECT "commitTimeMs", pg_logical_emit_message(
+            false,
+            ${this.messagePrefix},
+            json_build_object(
+              'id', ${id}::text,
+              'sendTimeMs', ${now}::int8,
+              'commitTimeMs', "commitTimeMs"
+            )::text
+          ) as lsn FROM CTE;
+      `;
+		} catch (e) {
+			if (this.#expectingLagReport?.id === id) this.#expectingLagReport = null;
+			throw e;
+		}
 		lagReport.lsn = toBigInt(lsn);
 		lagReport.commitTimeMs = commitTimeMs;
+		if (this.#expectingLagReport?.id === id) this.#scheduleMissingReportRetry(id);
 		if (log) this.#lc.info?.(`initiated lag report at lsn ${lsn}`, {
 			id,
 			lsn,
@@ -415,6 +426,7 @@ var LagReporter = class LagReporter {
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);
 		this.#timer = setTimeout(async () => {
+			this.#timer = void 0;
 			try {
 				await this.initiateLagReport();
 			} catch (e) {
@@ -423,6 +435,26 @@ var LagReporter = class LagReporter {
 			}
 		}, delayMs);
 	}
+	#scheduleMissingReportRetry(reportID) {
+		clearTimeout(this.#timer);
+		this.#timer = setTimeout(async () => {
+			this.#timer = void 0;
+			const missingReport = this.#expectingLagReport;
+			if (missingReport?.id !== reportID) return;
+			this.#lagReportRetries.add(1);
+			this.#lc.warn?.(`retrying missing lag report`, {
+				id: missingReport.id,
+				lsn: fromBigInt(missingReport.lsn),
+				sendTimeMs: missingReport.sendTimeMs
+			});
+			try {
+				await this.initiateLagReport();
+			} catch (e) {
+				this.#lc.warn?.(`error retrying lag report`, e);
+				this.#scheduleNextReport(this.#lagIntervalMs);
+			}
+		}, this.#lagIntervalMs);
+	}
 	processLagReportMessage(msg) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
 		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
@@ -430,9 +462,15 @@ var LagReporter = class LagReporter {
 	}
 	#processLagReport(report, watermark) {
 		const now = Date.now();
-		const nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-		if (report.id === this.#expectingLagReport?.id) this.#scheduleNextReport(nextSendTimeMs - now);
-		else this.#lc.debug?.(`received extraneous lag report`, { report });
+		const expectedReport = this.#expectingLagReport;
+		let nextSendTimeMs;
+		if (report.id === expectedReport?.id) {
+			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
+			this.#scheduleNextReport(nextSendTimeMs - now);
+		} else {
+			this.#lc.debug?.(`received extraneous lag report`, { report });
+			nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
+		}
 		const { sendTimeMs, commitTimeMs } = report;
 		return [
 			"status",
@@ -581,6 +619,11 @@ var ChangeMaker = class {
 		}
 		const prevEvent = this.#lastReplicationEvent;
 		this.#lastReplicationEvent = event;
+		const { schema } = event;
+		if (schema === void 0) {
+			lc.debug?.(`received context-only ${msg.prefix}/${type} event`, { event: summarizeReplicationEventForLog(event) });
+			return [];
+		}
 		const prevSchema = event.previousSchema === void 0 ? prevEvent?.schema : event.previousSchema;
 		if (!prevSchema) {
 			lc.info?.(`received ${msg.prefix}/${type} event`, { event: summarizeReplicationEventForLog(event) });
@@ -588,9 +631,9 @@ var ChangeMaker = class {
 		}
 		const effectiveTag = prevEvent?.type === "ddlStart" ? prevEvent.event.tag : event.type === "ddlUpdate" ? event.event.tag : "UNKNOWN";
 		lc.info?.(`processing ${effectiveTag} command from ${msg.prefix}/${type}`, { event: summarizeReplicationEventForLog(event) });
-		const changes = this.#makeSchemaChanges(lc, prevSchema, event, effectiveTag).map((change) => ["data", change]);
+		const changes = this.#makeSchemaChanges(lc, prevSchema, schema, event, effectiveTag).map((change) => ["data", change]);
 		lc.info?.(`${changes.length} schema change(s)`, { changes });
-		const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(event.schema);
+		const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(schema);
 		if (replicaIdentities) this.#replicaIdentityTimer = setTimeout(async () => {
 			try {
 				await replicaIdentities.apply(lc, this.#db);
@@ -627,10 +670,10 @@ var ChangeMaker = class {
 	* columns. This, for example, would be needed to properly support changing
 	* the type of a column that's indexed.
 	*/
-	#makeSchemaChanges(lc, preSchema, event, tag) {
+	#makeSchemaChanges(lc, preSchema, nextSchema, event, tag) {
 		try {
 			const [prevTbl, prevIdx] = specsByID(preSchema);
-			const [nextTbl, nextIdx] = specsByID(event.schema);
+			const [nextTbl, nextIdx] = specsByID(nextSchema);
 			const changes = [];
 			for (const table of nextTbl.values()) validate(lc, table);
 			const [droppedIdx, createdIdx] = symmetricDifferences(prevIdx, nextIdx);
@@ -903,7 +946,7 @@ function summarizeReplicationEventForLog(event) {
 	const { previousSchema, ...eventWithoutSchemas } = rest;
 	return {
 		...eventWithoutSchemas,
-		schema: summarizeSchemaForLog(schema),
+		...schema !== void 0 && { schema: summarizeSchemaForLog(schema) },
 		...previousSchema !== void 0 && { previousSchema: previousSchema === null ? null : summarizeSchemaForLog(previousSchema) }
 	};
 }