@rocicorp/zero 1.11.4 → 1.11.5

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -27,6 +27,7 @@ import "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
 import { ReplicationSlotCleanupMonitor } from "./replication-slot-cleanup-monitor.js";
 import { registerReplicationSlotHealthMetrics } from "./replication-slot-health.js";
+import { timingSafeEqual } from "node:crypto";
 import { nanoid } from "nanoid";
 import postgres from "postgres";
 import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
@@ -118,7 +119,8 @@ var PostgresChangeSource = class {
 		*/
 		const isTransactionalMessage = (lsn, msg) => {
 			if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
-				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg, lsn));
+				const status = this.#lagReporter.processLagReportMessage(msg, lsn);
+				if (status) changes.pushStatus(status);
 				return false;
 			}
 			const status = this.#lagReporter?.checkCurrentLSN(lsn);
@@ -325,6 +327,10 @@ var LagReporter = class LagReporter {
 		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
 		unit: "{report}"
 	});
+	#lagReportsIgnored = getOrCreateCounter("replication", "lag_reports_ignored", {
+		description: "Number of messages with the lag report prefix ignored because their content could not be parsed as a lag report.",
+		unit: "{message}"
+	});
 	#pgVersion;
 	#expectingLagReport = null;
 	#lastAcceptedSendTimeMs = 0;
@@ -475,10 +481,22 @@ var LagReporter = class LagReporter {
 	*     XLogData header), rather than the `messageLsn` of the message itself,
 	*     which is not guaranteed to be comparable to other stream LSNs (e.g. on
 	*     Aurora). See {@link initiateLagReport}.
+	* @returns the status to send downstream, or `undefined` if the message is
+	*     not a lag report. Any session that can call
+	*     `pg_logical_emit_message()` can emit a message with the lag report
+	*     prefix; one that cannot be parsed carries no report and is ignored,
+	*     as a message of an unknown prefix would be, rather than failing the
+	*     stream (which would replay it from the same position).
 	*/
 	processLagReportMessage(msg, lsn) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
-		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
+		let report;
+		try {
+			report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
+		} catch {
+			this.#lagReportsIgnored.add(1);
+			return;
+		}
 		return this.#processLagReport(report, majorVersionToString(lsn));
 	}
 	#processLagReport(report, watermark) {
@@ -643,9 +661,50 @@ var ChangeMaker = class {
 			default: throw new Error(`Unexpected message type ${stringify(msg)}`);
 		}
 	}
+	/**
+	* Any session that can call `pg_logical_emit_message()` can emit a message
+	* with the shard's ddl prefix. Only events that carry the shard's secret,
+	* which the upstream functions read from a table in the shard's schema
+	* (see `createEventFunctionStatements()`), are applied. Events emitted
+	* before the secret was installed (by functions that predate it) are
+	* accepted without one.
+	*
+	* @returns why the event cannot be verified, or `undefined` if it can.
+	*/
+	#ddlEventOriginProblem(lsn, json, ddlSecret, ddlSecretLsn) {
+		const secret = field(json, "secret");
+		if (typeof secret === "string") {
+			if (equalSecrets(secret, ddlSecret)) return;
+		} else if ((secret === void 0 || secret === null) && ddlSecretLsn && lsn <= toBigInt(ddlSecretLsn)) return;
+		return `the event does not carry the secret of the ${upstreamSchema(this.#shard)} schema's event functions (secret ${typeof secret === "string" ? "mismatch" : "absent"}, installed at ${ddlSecretLsn ?? "unknown"})`;
+	}
 	#lastReplicationEvent;
+	#ddlEventsIgnored = getOrCreateCounter("replication", "ddl_events_ignored", {
+		description: "Number of ddl messages ignored because they were not emitted by the shard's upstream functions (missing or wrong secret, or not valid JSON).",
+		unit: "{message}"
+	});
 	#handleDdlMessage(lc, lsn, msg) {
-		const event = parseLogicalMessageContent(lc, msg, replicationEventSchema);
+		const { ddlSecret, ddlSecretLsn } = this.#shardConfig;
+		assert(ddlSecret, "shard config has no ddl event secret");
+		const content = decodeLogicalMessage(msg);
+		let json;
+		let problem;
+		try {
+			json = JSON.parse(content);
+		} catch (e) {
+			problem = `not valid JSON: ${String(e)}`;
+		}
+		problem ??= this.#ddlEventOriginProblem(lsn, json, ddlSecret, ddlSecretLsn);
+		if (problem !== void 0) {
+			this.#ddlEventsIgnored.add(1);
+			lc.warn?.(`ignoring ${msg.prefix} message at ${fromBigInt(lsn)} (not emitted by the shard's upstream functions; replication continues): ${problem}`, describeUnverifiedMessage(json));
+			lc.debug?.(`ignored ${msg.prefix} message content`, { content: content.substring(0, 4096) });
+			return [];
+		}
+		const event = validateLogicalMessage(lc, msg, json, replicationEventSchema, () => stringify(field(json, "secret") === void 0 ? json : {
+			...json,
+			secret: "<redacted>"
+		}));
 		lc = lc.withContext("lsn", fromBigInt(lsn)).withContext("tag", event.event.tag).withContext("query", event.context.query);
 		if (event.previousSchema) warnForSkippedIndexes(lc, event.previousSchema, this.#skippedIndexWarnings);
 		if (event.schema) warnForSkippedIndexes(lc, event.schema, this.#skippedIndexWarnings);
@@ -999,8 +1058,7 @@ function summarizeSchemaForLog(schema) {
 	};
 }
 function summarizeReplicationEventForLog(event) {
-	const { schema, ...rest } = event;
-	const { previousSchema, ...eventWithoutSchemas } = rest;
+	const { schema, previousSchema, secret: _, ...eventWithoutSchemas } = event;
 	return {
 		...eventWithoutSchemas,
 		...schema !== void 0 && { schema: summarizeSchemaForLog(schema) },
@@ -1029,20 +1087,59 @@ var ShutdownSignal = class extends AbortError {
 		super("shutdown signal received (e.g. another zero-cache taking over the replication stream)", { cause });
 	}
 };
+function decodeLogicalMessage({ content }) {
+	return content instanceof Buffer ? content.toString("utf-8") : new TextDecoder().decode(content);
+}
 function parseLogicalMessageContent(lc, msg, schema) {
-	const { content } = msg;
-	const str = content instanceof Buffer ? content.toString("utf-8") : new TextDecoder().decode(content);
+	const str = decodeLogicalMessage(msg);
+	let json;
+	try {
+		json = JSON.parse(str);
+	} catch (e) {
+		return failParse(lc, msg, e, str);
+	}
+	return validateLogicalMessage(lc, msg, json, schema, () => str);
+}
+function validateLogicalMessage(lc, msg, json, schema, contentForLog) {
 	try {
-		const json = JSON.parse(str);
 		return parse(json, schema, "passthrough");
 	} catch (e) {
-		lc.error?.(`unable to parse logical message content: ${String(e)}`, { message: {
-			...msg,
-			content: str
-		} });
-		throw e;
+		return failParse(lc, msg, e, contentForLog());
 	}
 }
+/**
+* Logs and rethrows a parse failure. Any session that can call
+* `pg_logical_emit_message()` can emit a message with a prefix that the
+* zero-cache parses, so the content is untrusted: it is logged at `warn`
+* (callers skip the message, so there is nothing for an operator to fix in
+* zero-cache) without the content, which goes to `debug`, bounded.
+*/
+function failParse(lc, msg, e, content) {
+	const { content: _, ...header } = msg;
+	lc.warn?.(`unable to parse logical message content: ${String(e)}`, {
+		message: header,
+		contentLength: content.length
+	});
+	lc.debug?.(`unparseable logical message content`, { content: content.substring(0, 4096) });
+	throw e;
+}
+/** A property of untrusted JSON, or `undefined` if there is no such property. */
+function field(json, name) {
+	return typeof json === "object" && json !== null && name in json ? json[name] : void 0;
+}
+function describeUnverifiedMessage(json) {
+	const str = (value) => typeof value === "string" ? value.substring(0, 200) : null;
+	return {
+		type: str(field(json, "type")),
+		tag: str(field(field(json, "event"), "tag")),
+		query: str(field(field(json, "context"), "query"))
+	};
+}
+function equalSecrets(a, b) {
+	const bufA = Buffer.from(a);
+	const bufB = Buffer.from(b);
+	return bufA.length === bufB.length && timingSafeEqual(bufA, bufB);
+}
 //#endregion
 export { Acker, LagReporter, PostgresChangeSource, isIndexStructurallyChanged, relationDifferent };