@harperfast/harper-pro 5.0.16 → 5.0.17

package/replication/replicationConnection.ts

@@ -86,6 +86,11 @@ export const RECEIVING_STATUS_WAITING = 0;
 export const RECEIVING_STATUS_RECEIVING = 1;
 
 const MAX_PAYLOAD = env.get('replication_maxPayload') ?? 100_000_000;
+// When receiving a replication message, we apply per-record backpressure to keep a single
+// large batch from synchronously decoding thousands of records and ballooning the worker
+// heap past its limit. If the local replicator queue grows beyond this threshold we pause
+// the WS connection and wait for it to drain before continuing the decode loop.
+const RECEIVE_EVENT_HIGH_WATER_MARK = env.get('replication_receiveEventHighWaterMark') ?? 100;
 
 export const tableUpdateListeners = new Map();
 // This a map of the database name to the subscription object, for the subscriptions from our tables to the replication module
@@ -455,13 +460,37 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	const MAX_OUTSTANDING_BLOBS_BEING_SENT = env.get(CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
 	let outstandingCommits = 0;
 	let lastStructureLength = 0;
-	let replicationPaused = false;
+	// Multiple independent conditions can ask to pause receive on this WS (commit backlog,
+	// consumer queue full, blob write backpressure). We refcount the reasons so that resuming
+	// one does not race ahead of another that still wants the WS paused.
+	let pauseReasons = 0;
+	let commitBacklogPaused = false;
+	function addPauseReason(): void {
+		if (pauseReasons === 0) ws.pause();
+		pauseReasons++;
+	}
+	function removePauseReason(): void {
+		if (pauseReasons === 0) return;
+		pauseReasons--;
+		if (pauseReasons === 0) ws.resume();
+	}
 	let subscriptionRequest, auditSubscription;
 	let nodeSubscriptions;
 	let excludedNodes: string[]; // list of nodes to exclude from this subscription
 	let remoteShortIdToLocalId: Map<number, number>;
 	let subscribedNodeIds: Array<boolean | { startTime: number; endTime?: number }> | undefined; // map of node IDs to their subscription time ranges
-	ws.on('message', onWSMessage);
+	// Serialize message handling so that async backpressure inside onWSMessage doesn't allow
+	// the WS library to start processing the next frame before the current one is fully decoded.
+	// Without serialization, awaiting inside the handler would let concurrent message handlers
+	// share the consumer queue and defeat the per-record backpressure below.
+	let messageProcessing: Promise<void> = Promise.resolve();
+	let wsClosed = false;
+	ws.on('message', (body: Buffer) => {
+		messageProcessing = messageProcessing.then(
+			() => (wsClosed ? undefined : onWSMessage(body)),
+			() => (wsClosed ? undefined : onWSMessage(body))
+		);
+	});
 	let authorizationFinished = false;
 	function checkAuthorization(): boolean {
 		authorizationFinished = true;
@@ -473,29 +502,23 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		}
 		return true;
 	}
-	function onWSMessage(body: Buffer) {
+	async function onWSMessage(body: Buffer): Promise<void> {
 		if (!authorizationFinished) {
 			if (authorization?.then) {
-				return authorization.then(
-					(resolvedAuth) => {
-						authorization = resolvedAuth;
-						if (checkAuthorization()) {
-							onWSMessage(body); // continue on, now that authorization succeeded
-						}
-					},
-					(error) => {
-						authorizationFinished = true;
-						logger.error?.(connectionId, 'Authorization failed', error);
-						// don't send disconnect because we want the client to potentially retry
-						close(1008, 'Unauthorized');
-					}
-				);
-			} else {
-				if (checkAuthorization()) {
-					onWSMessage(body); // continue on, now that authorization succeeded
+				try {
+					authorization = await authorization;
+				} catch (error) {
+					authorizationFinished = true;
+					logger.error?.(connectionId, 'Authorization failed', error);
+					// don't send disconnect because we want the client to potentially retry
+					close(1008, 'Unauthorized');
+					return;
 				}
+				if (!checkAuthorization()) return;
+			} else if (!checkAuthorization()) {
+				return;
 			}
-			return; // we recursively call onWSMessage if authorization succeeded/completed
+			// fall through to handle this message now that authorization succeeded
 		}
 		if (!authorization) return;
 		// A replication header should begin with either a transaction timestamp or messagepack message of
@@ -542,9 +565,17 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 										schemaUpdateListener = forEachReplicatedDatabase(options, (database, databaseName) => {
 											if (checkDatabaseAccess(databaseName)) sendDBSchema(databaseName);
 										});
-										ws.on('close', () => {
-											schemaUpdateListener?.remove();
-										});
+										// onWSMessage is async, so the WS may have already closed by the time we get
+										// here — in that case 'close' has fired and adding the cleanup listener now
+										// would silently leak. Drop the registration immediately.
+										if (wsClosed) {
+											schemaUpdateListener.remove();
+											schemaUpdateListener = undefined;
+										} else {
+											ws.on('close', () => {
+												schemaUpdateListener?.remove();
+											});
+										}
 									}
 								} catch (error) {
 									// if this fails, we should close the connection and indicate that we should not reconnect
@@ -773,7 +804,21 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 									);
 								} else stream.end(blobBody);
 								if (stream.connectedToBlob) blobsInFlight.delete(fileId);
-							} else stream.write(blobBody);
+							} else if (!stream.write(blobBody)) {
+								// The PassThrough's internal queue is over its HWM, meaning the downstream
+								// file write (via pipeline in saveBlob) can't keep up. Pause the WS until the
+								// stream drains so blob chunks don't accumulate in memory faster than they
+								// can be flushed to disk. Also listen for 'close' so a destroyed stream
+								// (e.g. saveBlob error) doesn't strand the pause reason.
+								addPauseReason();
+								const release = () => {
+									stream.off('drain', release);
+									stream.off('close', release);
+									removePauseReason();
+								};
+								stream.on('drain', release);
+								stream.on('close', release);
+							}
 						} catch (error) {
 							logger.error?.(
 								`Error receiving blob for ${stream.recordId} from ${remoteNodeName} and streaming to storage`,
@@ -1344,6 +1389,16 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 											close();
 										}
 									});
+									// We are inside an async .then(); if the WS closed while waiting for it to
+									// resolve, attaching a 'close' handler now will not fire and the listeners
+									// above would stay subscribed on the global databaseEventsEmitter forever.
+									if (wsClosed) {
+										schemaUpdateListener.remove();
+										dbRemovalListener.remove();
+										schemaUpdateListener = undefined;
+										dbRemovalListener = undefined;
+										return;
+									}
 									ws.on('close', () => {
 										schemaUpdateListener?.remove();
 										dbRemovalListener?.remove();
@@ -1601,6 +1656,19 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						event.nodeId
 					);
 					tableSubscriptionToReplicator.send(event);
+					// Per-record backpressure: a single large WS message can synchronously decode
+					// thousands of records, each holding a decoded value object and a closure over
+					// the source buffer. Without yielding here the consumer can never drain the
+					// queue mid-message and the worker heap balloons until it OOMs.
+					const queueLength = tableSubscriptionToReplicator.queue?.length ?? 0;
+					if (queueLength > RECEIVE_EVENT_HIGH_WATER_MARK) {
+						addPauseReason();
+						try {
+							await tableSubscriptionToReplicator.waitForDrain();
+						} finally {
+							removePauseReason();
+						}
+					}
 				}
 				decoder.position = start + eventLength;
 			} while (decoder.position < body.byteLength);
@@ -1614,9 +1682,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 					'ingest'
 				);
 			}
-			if (outstandingCommits > MAX_OUTSTANDING_COMMITS && !replicationPaused) {
-				replicationPaused = true;
-				ws.pause();
+			if (outstandingCommits > MAX_OUTSTANDING_COMMITS && !commitBacklogPaused) {
+				commitBacklogPaused = true;
+				addPauseReason();
 				logger.debug?.(
 					`Commit backlog causing replication back-pressure, requesting that ${remoteNodeName} pause replication`
 				);
@@ -1639,9 +1707,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						}
 					}
 					outstandingCommits--;
-					if (replicationPaused) {
-						replicationPaused = false;
-						ws.resume();
+					if (commitBacklogPaused) {
+						commitBacklogPaused = false;
+						removePauseReason();
 						logger.debug?.(`Replication resuming ${remoteNodeName}`);
 					}
 					// if there are outstanding blobs to finish writing, delay commit receipts until they are finished (so that if we are interrupting
@@ -1689,6 +1757,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	});
 	ws.on('close', (code, reasonBuffer) => {
 		// cleanup
+		wsClosed = true;
 		clearInterval(sendPingInterval);
 		clearTimeout(receivePingTimer);
 		clearInterval(blobsTimer);
@@ -1808,15 +1877,21 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 			tableSubscriptionToReplicator.auditStore?.rootStore
 		);
 		if (finished) {
-			finished.blobId = blobId;
-			outstandingBlobsToFinish.push(finished);
-			finished
+			// We log the rejection via .catch() and also need the resulting promise — not the
+			// raw `finished` — to be what we hand to `Promise.all(outstandingBlobsToFinish)` in
+			// the end_txn onCommit path below. If we pushed `finished` directly, a save
+			// rejection would surface to that `await Promise.all(...)` as an unhandled error
+			// even though we already logged it here, and it would escape onCommit as an
+			// uncaughtException — observed in prod as ~35/sec ENOENT spam during catch-up.
+			const tracked = finished
 				.catch((err) => logger.error?.(`Blob save failed for ${blobId} from ${remoteNodeName}`, err))
 				.finally(() => {
 					logger.debug?.(`Finished receiving blob stream ${blobId}`);
-					const index = outstandingBlobsToFinish.indexOf(finished);
+					const index = outstandingBlobsToFinish.indexOf(tracked);
 					if (index > -1) outstandingBlobsToFinish.splice(index, 1);
 				});
+			tracked.blobId = blobId;
+			outstandingBlobsToFinish.push(tracked);
 		}
 		return localBlob;
 	}

package/replication/replicator.ts

@@ -595,12 +595,21 @@ export function forEachReplicatedDatabase(options, callback) {
 	for (const databaseName of Object.getOwnPropertyNames(databases)) {
 		forDatabase(databaseName);
 	}
-	onRemovedDB((databaseName) => {
+	// Both listeners must be returned through the same handle, otherwise callers that
+	// .remove() the result still leak the dropDatabase listener forever — which over time
+	// trips MaxListenersExceededWarning on the global databaseEventsEmitter.
+	const removedListener = onRemovedDB((databaseName) => {
 		forDatabase(databaseName);
 	});
-	return onUpdatedTable((Table) => {
+	const updatedListener = onUpdatedTable((Table) => {
 		forDatabase(Table.databaseName);
 	});
+	return {
+		remove() {
+			removedListener.remove();
+			updatedListener.remove();
+		},
+	};
 	function forDatabase(databaseName) {
 		const database = databases[databaseName];
 		logger.trace('Checking replication status of ', databaseName, options?.databases);

package/replication/subscriptionManager.ts

@@ -46,6 +46,13 @@ type ReplicationConnectionStatus = {
 type DBReplicationStatusMap = Map<string, ReplicationConnectionStatus> & { iterator: any };
 
 const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to other nodes, so operations can complete first
+// When a worker dies it may have been holding subscriptions for many (database, node) pairs.
+// All of those pairs fire onDatabase reassignments in the same tick, which would otherwise
+// slam a fresh worker with a burst of catchup connections and is the kind of pressure that
+// caused the OOM in the first place. We stagger the re-subscriptions in time so the new
+// worker(s) absorb them gradually.
+const WORKER_EXIT_REASSIGN_STAGGER_MS = 100;
+let nextWorkerExitReassignAt = 0;
 const connectionReplicationMap = new Map<string, DBReplicationStatusMap>();
 export let disconnectedFromNode; // this is set by thread to handle when a node is disconnected (or notify main thread so it can handle)
 export let connectedToNode; // this is set by thread to handle when a node is connected (or notify main thread so it can handle)
@@ -249,7 +256,10 @@ export async function startOnMainThread(options) {
 					if (dbReplicationWorkers.get(databaseName)?.worker === worker) {
 						// first verify it is still the worker
 						dbReplicationWorkers.delete(databaseName);
-						onDatabase(databaseName, tablesReplicateByDefault);
+						const now = Date.now();
+						nextWorkerExitReassignAt = Math.max(now, nextWorkerExitReassignAt) + WORKER_EXIT_REASSIGN_STAGGER_MS;
+						const delay = nextWorkerExitReassignAt - now;
+						setTimeout(() => onDatabase(databaseName, tablesReplicateByDefault), delay).unref();
 					}
 				});
 			}