@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/replication/replicationConnection.ts

@@ -31,10 +31,11 @@ import {
 	urlToNodeName,
 } from './replicator.ts';
 import { getThisNodeName } from '../core/server/nodeName.ts';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
 import { HAS_STRUCTURE_UPDATE, lastMetadata, lastValueEncoding, METADATA } from '../core/resources/RecordEncoder.ts';
 import { decode, encode, Packr } from 'msgpackr';
+import { createStructon } from 'structon';
 import { WebSocket } from 'ws';
 import { threadId } from 'worker_threads';
 import harperLogger from '../core/utility/logging/harper_logger.js';
@@ -43,11 +44,17 @@ import { disconnectedFromNode, connectedToNode, ensureNode } from './subscriptio
 import { EventEmitter } from 'events';
 import { createTLSSelector } from '../core/security/keys.js';
 import * as tls from 'node:tls';
-import { getHDBNodeTable, getNodeURL, getReplicationSharedStatus, type Node } from './knownNodes.ts';
+import {
+	getHDBNodeTable,
+	getNodeURL,
+	getReplicationSharedStatus,
+	getExcludedTablesForRouteEntries,
+} from './knownNodes.ts';
 import * as process from 'node:process';
 import { isIP } from 'node:net';
 import { recordAction } from '../core/resources/analytics/write.ts';
 import {
+	createBlob,
 	decodeBlobsWithWrites,
 	decodeFromDatabase,
 	decodeWithBlobCallback,
@@ -59,6 +66,14 @@ import { PassThrough } from 'node:stream';
 import { getLastVersion } from 'lmdb';
 const logger = forComponent('replication').conditional as Logger;
 
+// msgpackr v2 removed the built-in `randomAccessStructure` option; that random-access
+// struct support now lives in the `structon` package (the same wrapper core's
+// RecordEncoder uses). Replication decoders must be structon-wrapped so they install
+// the `_readStruct` hook and can decode typed-struct records — without it, struct
+// marker bytes (0x20–0x3f) decode as plain integers and the decode aborts with
+// "Data read, but end of buffer not reached".
+const StructonPackr = createStructon(Packr);
+
 // these are the codes we use for the different commands
 const SUBSCRIPTION_REQUEST = 129;
 const NODE_NAME = 140;
@@ -75,6 +90,8 @@ const COMMITTED_UPDATE = 144;
 const DB_SCHEMA = 145;
 const BLOB_CHUNK = 146;
 const SUBSCRIPTION_UPDATE = 147;
+const COPY_START = 148; // leader -> follower: a bulk table copy is starting; carries copyStartTime
+const COPY_COMPLETE = 149; // leader -> follower: the bulk table copy finished; follower clears its resume cursor
 export const CONFIRMATION_STATUS_POSITION = 0;
 export const RECEIVED_VERSION_POSITION = 1;
 export const RECEIVED_TIME_POSITION = 2;
@@ -91,6 +108,16 @@ const MAX_PAYLOAD = env.get('replication_maxPayload') ?? 100_000_000;
 // heap past its limit. If the local replicator queue grows beyond this threshold we pause
 // the WS connection and wait for it to drain before continuing the decode loop.
 const RECEIVE_EVENT_HIGH_WATER_MARK = env.get('replication_receiveEventHighWaterMark') ?? 100;
+// Even when the consumer keeps up (queue below the high-water mark), a single large inbound message
+// would otherwise decode thousands of records in one synchronous turn — pegging the worker, blocking
+// replication ping responses, and tripping core's "JavaScript execution has taken too long" monitor
+// (MAX_EVENT_DELAY_TIME = 3 s). Yield the event loop at least this often (ms) while decoding so the
+// worker stays responsive during a bulk copy/clone.
+const RECEIVE_YIELD_INTERVAL = env.get('replication_receiveYieldInterval') ?? 100;
+// During a bulk clone copy the leader flushes a checkpoint transaction every this many records so the
+// follower commits incrementally and persists a resume cursor. On reconnect the copy resumes from that
+// cursor instead of restarting from zero. Larger = less overhead but coarser resume granularity.
+const COPY_CHECKPOINT_RECORDS = env.get('replication_copyCheckpointRecords') ?? 1000;
 
 export const tableUpdateListeners = new Map();
 // This a map of the database name to the subscription object, for the subscriptions from our tables to the replication module
@@ -104,7 +131,95 @@ const SKIPPED_MESSAGE_SEQUENCE_UPDATE_DELAY = 300;
 // We want it be fairly quick so we can let the sending node know that we have received and committed the update.
 // (but still allow for batching so we aren't sending out a message for every update under load)
 const COMMITTED_UPDATE_DELAY = 2;
-const PING_INTERVAL = 30000;
+const PING_INTERVAL = env.get(CONFIG_PARAMS.REPLICATION_PINGINTERVAL) ?? 30000;
+// Time (ms) without any socket activity before a connection is treated as dead.
+const PING_TIMEOUT = env.get(CONFIG_PARAMS.REPLICATION_PINGTIMEOUT) ?? PING_INTERVAL * 2;
+// The receive-side watchdog fires after this much silence on a replication WS. Both client and
+// server arm it: the client also runs an active 30s sendPing tick that should normally catch a
+// silent peer first, but if that tick is missed (event-loop stall, ws.terminate() not propagating
+// to 'close', etc.) this timer-based watchdog is the belt-and-suspenders that forces the
+// reconnect — see harper-pro#233.
+const RECEIVE_SILENCE_THRESHOLD_MS = PING_TIMEOUT;
+
+/**
+ * Decide whether an idle replication connection should be terminated as dead.
+ *
+ * Liveness is measured from the last observed socket byte movement (in either direction), not from a
+ * single ping interval. A bulk transfer — notably the initial clone copy of a large table — makes
+ * slow but real progress: the sender's socket buffer drains in bursts as the peer consumes, so bytes
+ * keep moving within the timeout window even while it is otherwise stalled. A genuinely dead or
+ * unreachable peer moves no bytes at all, so it still trips the timeout. We deliberately do NOT exempt
+ * the sender's `isPausedForBackPressure` drain-wait here: if the peer dies after we have filled our
+ * socket buffer, the drain event never fires, and exempting it would hang the connection forever.
+ *
+ * The one exemption is `pauseReasons > 0`: the receiver has intentionally stopped reading to drain its
+ * own queue. That stall is local and self-clearing (it does not depend on the peer), so it is never a
+ * death signal; the caller keeps liveness fresh while paused and resumes normal detection afterward.
+ */
+export function shouldTerminateIdlePing(idleMs: number, pingTimeout: number, pauseReasons: number): boolean {
+	return pauseReasons === 0 && idleMs >= pingTimeout;
+}
+
+/**
+ * Receive-side silence watchdog. Arm with `reset()` whenever incoming activity is observed
+ * (peer ping, pong, message). If `intervalMs` elapses with the underlying socket's `bytesRead`
+ * unchanged, `onSilence` is invoked exactly once.
+ *
+ * Only `bytesRead` is checked — `bytesWritten` reflects our own outbound traffic (pings, blob
+ * sends) and is not proof that the peer is alive. Including it would let our own keepalive
+ * pings suppress the watchdog in the exact missed-sendPing scenario this is meant to recover.
+ *
+ * Callers should suspend the watchdog (via `stop()`) when the underlying WS is intentionally
+ * paused for backpressure: `bytesRead` is frozen by design while paused and would otherwise
+ * cause a spurious termination of a healthy connection.
+ *
+ * Exported so the timer logic can be exercised in isolation by `unitTests/replication/
+ * receiveWatchdog.test.mjs` — production callers go through `replicateOverWS`.
+ */
+export function createReceiveWatchdog(opts: {
+	intervalMs: number;
+	getBytesRead: () => number;
+	onSilence: () => void;
+}): { reset: () => void; stop: () => void } {
+	let timer: NodeJS.Timeout | undefined;
+	let bytesReadAtArm = 0;
+	let lastResetAt = 0;
+	// Coalesce rapid reset() calls (e.g. message frames arriving thousands of times per second
+	// during a large copy) so we do not churn setTimeout/clearTimeout per frame. Granularity loss
+	// is small relative to intervalMs — at worst the watchdog fires this much earlier or later.
+	const throttleMs = Math.min(1000, Math.max(100, opts.intervalMs / 30));
+	function check() {
+		const current = opts.getBytesRead();
+		if (current === bytesReadAtArm) {
+			timer = undefined;
+			opts.onSilence();
+			return;
+		}
+		// Bytes advanced since the last arm — but the activity may have been swallowed by the
+		// reset() throttle, so we cannot rely on an external caller to re-arm us. Re-arm from
+		// the new baseline; otherwise a throttled-reset-then-silence sequence would leave the
+		// watchdog permanently inactive (see PR #234 review).
+		bytesReadAtArm = current;
+		lastResetAt = Date.now();
+		timer = setTimeout(check, opts.intervalMs).unref();
+	}
+	return {
+		reset() {
+			const now = Date.now();
+			if (timer && now - lastResetAt < throttleMs) return;
+			lastResetAt = now;
+			if (timer) clearTimeout(timer);
+			bytesReadAtArm = opts.getBytesRead();
+			timer = setTimeout(check, opts.intervalMs).unref();
+		},
+		stop() {
+			if (timer) {
+				clearTimeout(timer);
+				timer = undefined;
+			}
+		},
+	};
+}
 let secureContexts: Map<string, tls.SecureContext>;
 /**
  * Handles reconnection, and requesting catch-up
@@ -161,7 +276,7 @@ export async function createWebSocket(
 			);
 		}
 	}
-	const headers = {};
+	const headers: Record<string, string> = {};
 	if (authorization) {
 		headers.Authorization = authorization;
 	}
@@ -222,6 +337,7 @@ export class NodeReplicationConnection extends EventEmitter {
 	databaseName: string;
 	nodeName?: string;
 	authorization?: string;
+	tentativeNode?: any;
 	constructor(url: string, subscription: any, databaseName: string, nodeName?: string, authorization?: string) {
 		super();
 		this.url = url;
@@ -232,6 +348,7 @@ export class NodeReplicationConnection extends EventEmitter {
 	}
 
 	async connect() {
+		if (this.intentionallyUnsubscribed) return;
 		if (!this.session) this.resetSession();
 		// TODO: Need to do this specifically for each node
 		this.socket = await createWebSocket(this.url, { serverName: this.nodeName, authorization: this.authorization });
@@ -357,7 +474,7 @@ export class NodeReplicationConnection extends EventEmitter {
 	}
 	unsubscribe() {
 		this.intentionallyUnsubscribed = true;
-		this.socket.close(1008, 'No longer subscribed');
+		this.socket?.close(1008, 'No longer subscribed');
 	}
 
 	getRecord(request) {
@@ -370,7 +487,7 @@ export class NodeReplicationConnection extends EventEmitter {
 /**
  * This handles both incoming and outgoing WS allowing either one to issue a subscription and get replication and/or handle subscription requests
  */
-export function replicateOverWS(ws: WebSocket, options: any, authorization: Promise<Node | undefined>) {
+export function replicateOverWS(ws: WebSocket, options: any, authorization: any) {
 	const p = options.port || options.securePort;
 	const connectionId =
 		(process.pid % 1000) +
@@ -387,7 +504,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	let databaseName = options.database;
 	const dbSubscriptions = options.databaseSubscriptions || databaseSubscriptions;
 	let auditStore: any;
-	let auditLogIterable: Iterable<AuditRecord>; // reusable iterator for a subscription
+	let auditLogIterable: Iterable<AuditRecord> & { removeLog?: (name: string) => void; addLog?: (name: string) => void }; // reusable iterator for a subscription
 	let replicationSharedStatus: Float64Array;
 	// this is the subscription that the local table makes to this replicator, and incoming messages
 	// are sent to this subscription queue:
@@ -405,7 +522,33 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	remoteNodeName = authorization.name;
 	if (remoteNodeName && options.connection) options.connection.nodeName = remoteNodeName;
 	let lastSequenceIdReceived, lastSequenceIdCommitted;
-	let sendPingInterval, receivePingTimer, lastPingTime, skippedMessageSequenceUpdateTimer;
+	// Bulk-copy resume state (receiver side). While inCopyMode, each committed batch persists a cursor
+	// (copyStartTime + last committed table/key) so an interrupted copy can resume instead of restarting.
+	let inCopyMode = false;
+	let copyModeStartTime = 0;
+	let copyFromNodeId; // local id of the node we are copying from — the key for the persisted cursor
+	let copyCompleteReceived = false;
+	// Finish the copy — leave copy mode and remove the resume cursor — only once COPY_COMPLETE has been
+	// received AND every copied batch has committed (outstandingCommits drained, which includes the final
+	// end_txn that advances the resume seqId to copyStartTime). We deliberately stay in copy mode until
+	// then so batches still committing keep advancing the cursor (onCommit). Finishing earlier — e.g.
+	// synchronously when COPY_COMPLETE is decoded while batches are still queued — would freeze the cursor
+	// and risk a crash that loses both the cursor and the not-yet-durable rows, leaving the next start to
+	// resume from seqId with gaps.
+	function maybeFinishCopy() {
+		if (copyCompleteReceived && outstandingCommits === 0) {
+			// guard only the cursor removal on a known node id; ALWAYS exit copy mode, otherwise a
+			// COPY_START whose getIdOfRemoteNode returned undefined would strand the node in copy mode
+			// (received-version watermark suppressed) and it could never reach Available.
+			if (copyFromNodeId !== undefined)
+				tableSubscriptionToReplicator?.dbisDB?.remove([Symbol.for('copyCursor'), copyFromNodeId]);
+			inCopyMode = false;
+			copyCompleteReceived = false;
+			copyFromNodeId = undefined;
+		}
+	}
+	let sendPingInterval, lastPingTime, skippedMessageSequenceUpdateTimer;
+	let receiveWatchdog: { reset: () => void; stop: () => void } | undefined;
 	let blobsTimer;
 	const DELAY_CLOSE_TIME = 60000; // amount of time to wait before closing the connection if we haven't any activity and there are no subscriptions
 	let delayedClose: NodeJS.Timeout;
@@ -413,41 +556,78 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	// track bytes read and written so we can verify if a connection is really dead on pings
 	let bytesRead = 0;
 	let bytesWritten = 0;
+	// wall-clock time of the last observed socket activity (bytes moved in either direction); the
+	// keep-alive timeout is measured from this so a legitimately slow-but-progressing transfer stays
+	// alive while a truly idle/dead peer is still terminated.
+	let lastByteActivity = performance.now();
+	// Multiple independent conditions can ask to pause receive on this WS (commit backlog, consumer
+	// queue full, blob write backpressure). We refcount the reasons so that resuming one does not race
+	// ahead of another that still wants the WS paused. Declared before the ping setup below because the
+	// immediate sendPing() reads it.
+	let pauseReasons = 0;
 	const blobTimeout = env.get(CONFIG_PARAMS.REPLICATION_BLOBTIMEOUT) ?? 120000;
 	const blobsInFlight = new Map();
 	const outstandingBlobsToFinish: Promise<void>[] = [];
 	let outstandingBlobsBeingSent = 0;
 	let blobSentCallback: (v?: any) => void;
+	// Refresh the keep-alive liveness clock from observed socket byte movement. If the underlying
+	// _socket isn't observable (test mocks, pre-connect, or a change in the ws library internals),
+	// bytesRead/bytesWritten read as undefined; we can't measure activity, so treat the connection as
+	// live rather than let the keep-alive falsely terminate a healthy peer.
+	function noteByteActivity(): void {
+		const read = ws._socket?.bytesRead;
+		const written = ws._socket?.bytesWritten;
+		if (read === undefined || written === undefined || read !== bytesRead || written !== bytesWritten) {
+			lastByteActivity = performance.now();
+		}
+	}
 	if (options.url) {
 		const sendPing = () => {
-			// if we have not received a message in the last ping interval, we should terminate the connection (but check to make sure we aren't just waiting for other data to flow)
-			if (lastPingTime && bytesRead === ws._socket?.bytesRead && bytesWritten === ws._socket?.bytesWritten)
-				ws.terminate(); // timeout
-			else {
-				lastPingTime = performance.now();
-				ws.ping();
-				bytesRead = ws._socket?.bytesRead;
-				bytesWritten = ws._socket?.bytesWritten;
+			// Note any socket activity since the last interval (incoming pong/data or our send buffer
+			// draining as the peer consumes) — either proves the peer is still alive.
+			noteByteActivity();
+			if (shouldTerminateIdlePing(performance.now() - lastByteActivity, PING_TIMEOUT, pauseReasons)) {
+				ws.terminate(); // no socket activity within the timeout — peer is gone
+				return;
 			}
+			// While paused for receiver backpressure, keep our own liveness fresh: the stall is local and
+			// self-clearing (it doesn't depend on the peer), so we must not time the peer out for it.
+			if (pauseReasons > 0) lastByteActivity = performance.now();
+			// Always send the keep-alive ping. ws.pause() only stops reads, not writes, and the accepted
+			// peer relies on our pings to keep its own receive timer alive even when it has no data to send
+			// us. Record byte counts AFTER the ping so the ping's own bytes aren't later mistaken for peer
+			// activity.
+			lastPingTime = performance.now();
+			ws.ping();
+			bytesRead = ws._socket?.bytesRead;
+			bytesWritten = ws._socket?.bytesWritten;
 		};
 		sendPingInterval = setInterval(sendPing, PING_INTERVAL).unref();
 		sendPing(); // send the first ping immediately so we can measure latency
-	} else {
-		resetPingTimer();
 	}
+	// Both client and server arm the receive watchdog. On the client this is independent of the
+	// sendPing tick above: if that tick is missed or its ws.terminate() does not propagate a
+	// 'close' event, the watchdog forces the reconnect path. See harper-pro#233 for the failure
+	// modes observed in the field.
+	receiveWatchdog = createReceiveWatchdog({
+		intervalMs: RECEIVE_SILENCE_THRESHOLD_MS,
+		getBytesRead: () => ws._socket?.bytesRead ?? 0,
+		onSilence: () => {
+			// Warn-level: if the active sendPing was healthy this watchdog should not have fired,
+			// so it is a signal that something is wrong upstream (event-loop stall, keepalive timer
+			// misbehaving, peer accepting bytes but not progressing the protocol). Surface it so
+			// operators have something to grep for.
+			const dbContext = databaseName ? ` (db: "${databaseName}")` : '';
+			const direction = options.url ? 'no activity from' : 'no ping from';
+			logger.warn?.(
+				`Receive watchdog: ${direction} ${remoteNodeName}${dbContext} for ${RECEIVE_SILENCE_THRESHOLD_MS}ms — terminating connection and reconnecting`
+			);
+			ws.terminate();
+		},
+	});
+	const resetPingTimer = receiveWatchdog.reset;
+	resetPingTimer();
 	ws._socket?.setMaxListeners(200); // we should allow a lot of drain listeners for concurrent blob streams
-	function resetPingTimer() {
-		clearTimeout(receivePingTimer);
-		bytesRead = ws._socket?.bytesRead;
-		bytesWritten = ws._socket?.bytesWritten;
-		receivePingTimer = setTimeout(() => {
-			// double check to make sure we aren't just waiting for other data to flow
-			if (bytesRead === ws._socket?.bytesRead && bytesWritten === ws._socket?.bytesWritten) {
-				logger.warn?.(`Timeout waiting for ping from ${remoteNodeName}, terminating connection and reconnecting`);
-				ws.terminate();
-			}
-		}, PING_INTERVAL * 2).unref();
-	}
 	let ratioOfBackPressureTime = 0;
 	let lastBackPressureCheck = 0;
 	let isPausedForBackPressure = false;
@@ -489,23 +669,32 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	const MAX_OUTSTANDING_BLOBS_BEING_SENT = env.get(CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
 	let outstandingCommits = 0;
 	let lastStructureLength = 0;
-	// Multiple independent conditions can ask to pause receive on this WS (commit backlog,
-	// consumer queue full, blob write backpressure). We refcount the reasons so that resuming
-	// one does not race ahead of another that still wants the WS paused.
-	let pauseReasons = 0;
 	let commitBacklogPaused = false;
 	function addPauseReason(): void {
-		if (pauseReasons === 0) ws.pause();
+		if (pauseReasons === 0) {
+			ws.pause();
+			// Suspend the receive watchdog while the socket is intentionally paused — `bytesRead`
+			// is frozen by `ws.pause()` so the byte check cannot tell legitimate backpressure
+			// from peer silence, and firing here would terminate a healthy mid-ingest connection.
+			receiveWatchdog?.stop();
+		}
 		pauseReasons++;
 	}
 	function removePauseReason(): void {
 		if (pauseReasons === 0) return;
 		pauseReasons--;
-		if (pauseReasons === 0) ws.resume();
+		if (pauseReasons === 0) {
+			ws.resume();
+			// Restart the silence window from the resume point — we deliberately do not penalize
+			// the connection for the time it spent paused.
+			receiveWatchdog?.reset();
+		}
 	}
 	let subscriptionRequest, auditSubscription;
 	let nodeSubscriptions;
 	let excludedNodes: string[]; // list of nodes to exclude from this subscription
+	// undefined = not yet computed; null = computed, no exclusions; Set = tables to drop on receive
+	let receiveExcludedTables: Set<string> | null | undefined;
 	let remoteShortIdToLocalId: Map<number, number>;
 	let subscribedNodeIds: Array<boolean | { startTime: number; endTime?: number }> | undefined; // map of node IDs to their subscription time ranges
 	// Serialize message handling so that async backpressure inside onWSMessage doesn't allow
@@ -515,6 +704,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	let messageProcessing: Promise<void> = Promise.resolve();
 	let wsClosed = false;
 	ws.on('message', (body: Buffer) => {
+		// Reset the receive watchdog synchronously on every frame — async processing below may
+		// take a long time and we want a single late frame to count as proof of life immediately.
+		resetPingTimer();
 		messageProcessing = messageProcessing.then(
 			() => (wsClosed ? undefined : onWSMessage(body)),
 			() => (wsClosed ? undefined : onWSMessage(body))
@@ -554,7 +746,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		// of an array that begins with the command code
 		lastMessageTime = performance.now();
 		try {
-			const decoder = (body.dataView = new Decoder(body.buffer, body.byteOffset, body.byteLength));
+			const decoder = ((body as any).dataView = new Decoder(body.buffer, body.byteOffset, body.byteLength));
 			if (body[0] > 127) {
 				// not a transaction, special message
 				const message = decode(body);
@@ -720,13 +912,12 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						// of the table id to the decoder so we can decode the binary data for each table.
 						tableDecoders[tableId] = {
 							name: tableName,
-							decoder: new Packr({
+							decoder: new StructonPackr({
 								useBigIntExtension: true,
-								randomAccessStructure: true,
 								freezeData: true,
 								typedStructs: data.typedStructs,
 								structures: data.structures,
-							}),
+							} as any),
 							getEntry(id) {
 								return table.primaryStore.getEntry(id);
 							},
@@ -765,7 +956,21 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 							'replication',
 							'committed-update'
 						);
-						getSharedStatus().buffer.notify();
+						(getSharedStatus().buffer as any).notify();
+						break;
+					case COPY_START:
+						// the leader is (re)starting a bulk copy; track a resume cursor for it
+						inCopyMode = true;
+						copyModeStartTime = data; // copyStartTime anchor chosen by the leader
+						copyFromNodeId = getIdOfRemoteNode(remoteNodeName, auditStore);
+						logger.debug?.(connectionId, 'bulk copy starting from', remoteNodeName, new Date(copyModeStartTime));
+						break;
+					case COPY_COMPLETE:
+						// Copy signalled complete. Stay in copy mode so batches still committing keep advancing the
+						// cursor; maybeFinishCopy exits copy mode and clears the cursor once those commits drain.
+						copyCompleteReceived = true;
+						maybeFinishCopy();
+						logger.debug?.(connectionId, 'bulk copy complete from', remoteNodeName);
 						break;
 					case SEQUENCE_ID_UPDATE:
 						// we need to record the sequence number that the remote node has received
@@ -833,20 +1038,36 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 									);
 								} else stream.end(blobBody);
 								if (stream.connectedToBlob) blobsInFlight.delete(fileId);
+							} else if (stream.destroyed || stream.writableEnded) {
+								// The stream was already torn down before this mid-blob chunk arrived —
+								// typically because saveBlob's pipeline failed (e.g. ENOENT on
+								// createWriteStream) and destroyed the PassThrough source, firing 'close'.
+								// We must NOT fall into the backpressure branch below: writing to a dead
+								// stream returns false, and pausing to wait for a 'drain'/'close' that has
+								// already fired strands the pause reason forever, wedging the entire
+								// receive loop (observed in prod: receiver goes silent, sender stuck
+								// reconnecting, replication never recovers). Drop the orphaned chunk and
+								// forget the stream instead.
+								blobsInFlight.delete(fileId);
 							} else if (!stream.write(blobBody)) {
 								// The PassThrough's internal queue is over its HWM, meaning the downstream
 								// file write (via pipeline in saveBlob) can't keep up. Pause the WS until the
 								// stream drains so blob chunks don't accumulate in memory faster than they
-								// can be flushed to disk. Also listen for 'close' so a destroyed stream
-								// (e.g. saveBlob error) doesn't strand the pause reason.
-								addPauseReason();
-								const release = () => {
-									stream.off('drain', release);
-									stream.off('close', release);
-									removePauseReason();
-								};
-								stream.on('drain', release);
-								stream.on('close', release);
+								// can be flushed to disk.
+								if (stream.destroyed || stream.writableEnded) {
+									// write() itself may have torn the stream down (e.g. a late error). If so,
+									// 'drain'/'close' won't arrive — skip pausing rather than strand the reason.
+									blobsInFlight.delete(fileId);
+								} else {
+									addPauseReason();
+									const release = () => {
+										stream.off('drain', release);
+										stream.off('close', release);
+										removePauseReason();
+									};
+									stream.on('drain', release);
+									stream.on('close', release);
+								}
 							}
 						} catch (error) {
 							logger.error?.(
@@ -1097,6 +1318,11 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						let tableById;
 						let currentSequenceId = Infinity; // the last sequence number in the audit log that we have processed, set this with a finite number from the subscriptions
 						let sentSequenceId; // the last sequence number we have sent
+						// Tables excluded from outgoing replication to this peer+database (from sendsTo config)
+						const sendExcludedTables =
+							authorization?.replicates && typeof authorization.replicates === 'object'
+								? getExcludedTablesForRouteEntries(authorization.replicates.sendsTo, remoteNodeName, databaseName)
+								: null;
 						const sendAuditRecord = (auditRecord, localTime) => {
 							if (auditRecord.type === 'end_txn') {
 								if (currentTransaction.txnTime) {
@@ -1127,6 +1353,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 								}
 							}
 							const table = tableEntry.table;
+							if (sendExcludedTables?.has(table.tableName)) {
+								return skipAuditRecord();
+							}
 							const primaryStore = table.primaryStore;
 							const encoder = primaryStore.encoder;
 							// Force a reload the first time this connection touches each table:
@@ -1154,7 +1383,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 							const matchesSubscription =
 								(excludedNodes && timeRange === undefined) ||
 								// if it is in the list, we check the timestamps to verify it matches
-								(timeRange && timeRange.startTime < localTime && (!timeRange.endTime || timeRange.endTime > localTime));
+								(timeRange &&
+									(timeRange as any).startTime < localTime &&
+									(!(timeRange as any).endTime || (timeRange as any).endTime > localTime));
 							if (!matchesSubscription) {
 								if (DEBUG_MODE)
 									logger.trace?.(
@@ -1386,8 +1617,11 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 							subscriptionToHdbNodes?.end();
 						});
 						// find the earliest start time of the subscriptions
-						for (const { startTime } of nodeSubscriptions) {
-							if (startTime < currentSequenceId) currentSequenceId = startTime;
+						let copyResume: { copyStartTime: number; currentTable: string; afterKey: any } | undefined;
+						for (const subscription of nodeSubscriptions) {
+							if (subscription.startTime < currentSequenceId) currentSequenceId = subscription.startTime;
+							// a follower resuming an interrupted bulk copy sends back where it left off
+							if (subscription.copyResume) copyResume = subscription.copyResume;
 						}
 
 						// wait for internal subscription, might be waiting for a table to be registered
@@ -1457,16 +1691,51 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 										isFirst = false;
 										if (currentSequenceId === 0) {
 											logger.info?.('Replicating all tables to', remoteNodeName);
-											let lastSequenceId = Date.now(); // we need at least a default time point in case there are no records
+											// Capture the resume point BEFORE iterating. The bulk copy walks the primary store in
+											// key order (snapshot: false), but the follower resumes replication from the audit log in
+											// time order. Using copyStartTime — not max(localTime) of the copied records — guarantees
+											// the post-copy audit replay re-delivers every write committed during the copy, including
+											// ones to keys we already passed; resuming from max(localTime) would skip those (data loss).
+											// When the follower is resuming an interrupted copy, keep the original copy start time so
+											// the post-copy resume point stays anchored to when the copy first began (see safety note).
+											const copyStartTime = copyResume?.copyStartTime ?? Date.now();
 											const nodeId = getThisNodeId(auditStore);
+											// Tell the follower a bulk copy is starting and its anchor time, so it tracks a resume cursor.
+											ws.send(encode([COPY_START, copyStartTime]));
+											let recordsSinceCheckpoint = 0;
+											// If resuming, the follower already committed every table before currentTable (records commit
+											// in stable iteration order), so skip to currentTable and continue after its last committed key.
+											let reachedResumeTable = !copyResume;
+											// currentTable must be one the loop below will actually visit (present in `tables` AND passing
+											// the same replication filter); otherwise the skip loop never reaches it and would omit every
+											// later table. Mirror the loop's own check so a dropped/unreplicated cursor table forces a restart.
+											if (copyResume && !tableToTableEntry(tables[copyResume.currentTable])) {
+												// cursor table is gone or no longer replicated — the skip loop would never reach it and
+												// would omit every later table, so recopy from scratch (idempotent puts, copyStartTime reset)
+												logger.warn?.(
+													'Copy-resume table missing or unreplicated, restarting full copy',
+													copyResume.currentTable
+												);
+												copyResume = undefined;
+												reachedResumeTable = true;
+											}
+											const resumeCurrentTable = copyResume?.currentTable;
+											const resumeAfterKey = copyResume?.afterKey;
 											for (const tableName in tables) {
-												if (!tableToTableEntry(tableName)) continue; // if we aren't replicating this table, skip it
 												const table = tables[tableName];
-												for (const entry of table.primaryStore.getRange({
-													snapshot: false,
-													versions: true,
-													// values: false, // TODO: eventually, we don't want to decode, we want to use fast binary transfer
-												})) {
+												if (!tableToTableEntry(table)) continue; // if we aren't replicating this table, skip it
+												if (!reachedResumeTable) {
+													if (tableName !== resumeCurrentTable) continue; // already committed on the follower
+													reachedResumeTable = true;
+												}
+												const rangeOptions: any = { snapshot: false, versions: true };
+												// values: false, // TODO: eventually, we don't want to decode, we want to use fast binary transfer
+												if (tableName === resumeCurrentTable) {
+													// resume this table after the last key the follower committed
+													rangeOptions.start = resumeAfterKey;
+													rangeOptions.exclusiveStart = true;
+												}
+												for (const entry of table.primaryStore.getRange(rangeOptions)) {
 													if (closed) return;
 													logger.trace?.(
 														connectionId,
@@ -1476,7 +1745,6 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 														entry.key,
 														entry.localTime
 													);
-													lastSequenceId = Math.max(entry.localTime ?? 1, lastSequenceId);
 													getSharedStatus()[SENDING_TIME_POSITION] = 1;
 													const encoded = createAuditEntry({
 														version: entry.version,
@@ -1496,7 +1764,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 														residencyId: entry.residencyId,
 														previousResidencyId: null,
 														expiresAt: entry.expiresAt,
-													});
+													} as any);
 													await sendAuditRecord(
 														{
 															// make it look like an audit record
@@ -1521,26 +1789,46 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 														encoded.length,
 														encoded.slice(0, 10)
 													);
+													// Periodically flush the accumulated records as a message so the follower commits this
+													// batch and advances its resume cursor. This is a plain flush with NO sequence update:
+													// emitting an end_txn here would advance the follower's received-version watermark to
+													// copyStartTime mid-copy, which monitorSync could read as "caught up" and mark the clone
+													// Available/cloned with rows still uncopied. (Records with differing versions already flush
+													// naturally above; this also bounds same-version bulk data into committable batches.) The
+													// watermark is only advanced to copyStartTime by the single end_txn after the whole copy.
+													if (++recordsSinceCheckpoint >= COPY_CHECKPOINT_RECORDS && position - encodingStart > 8) {
+														recordsSinceCheckpoint = 0;
+														sendQueuedData();
+														encodingStart = position;
+														currentTransaction.txnTime = 0;
+													}
 												}
 												logger.info?.('Finished copy table', tableName, remoteNodeName);
 											}
-											currentSequenceId = lastSequenceId;
+											currentSequenceId = copyStartTime;
 											if (!currentTransaction.txnTime) {
-												// if we haven't sent any records, force a txn start and end so the subscribers records a timestamp
-												currentTransaction.txnTime = lastSequenceId;
-												writeFloat64(lastSequenceId);
-											}
-											if (position - encodingStart > 8) {
-												// if we have any queued transactions to send, send them now
-												sendAuditRecord(
-													{
-														type: 'end_txn',
-													},
-													currentSequenceId
-												);
+												// no records pending (none sent, or the last batch landed on a checkpoint flush):
+												// force a txn so the end_txn below still carries the sequence update
+												currentTransaction.txnTime = copyStartTime;
+												encodingStart = position;
+												writeFloat64(copyStartTime);
 											}
+											// ALWAYS emit the final end_txn at copyStartTime. It carries the REMOTE_SEQUENCE_UPDATE
+											// that advances the follower's seqId and received-version watermark to copyStartTime —
+											// the sole signal that the copy is synced (per-record watermark advance is suppressed
+											// during the copy). Skipping it when the last rows landed exactly on a checkpoint flush
+											// would leave the clone unable to ever reach Available.
+											sendAuditRecord(
+												{
+													type: 'end_txn',
+												},
+												currentSequenceId
+											);
+											// The full copy is done — tell the follower to clear its resume cursor and fall back to
+											// normal audit-log replication from the persisted seqId (which is copyStartTime).
+											ws.send(encode([COPY_COMPLETE]));
 											getSharedStatus()[SENDING_TIME_POSITION] = 0;
-											currentSequenceId = lastSequenceId;
+											currentSequenceId = copyStartTime;
 										}
 									}
 									const logName = subscribedNodeName === getThisNodeName() ? 'local' : subscribedNodeName;
@@ -1599,6 +1887,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 			let beginTxn = true;
 			let event; // could also get txnTime from decoder.getFloat64(0);
 			let sequenceIdReceived;
+			let lastYieldTime = performance.now();
 			do {
 				getSharedStatus();
 				const eventLength = decoder.readInt();
@@ -1627,6 +1916,30 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 				if (!tableDecoder) {
 					logger.error?.(`No table found with an id of ${auditRecord.tableId}`);
 				}
+				// Lazily compute receive-side exclusions once remoteNodeName is known.
+				// Prefer routeReplicates from the subscriber-side connection; fall back to
+				// authorization.replicates when this is the server-side handler.
+				if (receiveExcludedTables === undefined) {
+					const firstNode = options.connection?.nodeSubscriptions?.[0];
+					const receivesFromEntries =
+						firstNode?.routeReplicates?.receivesFrom ??
+						(authorization?.replicates && typeof authorization.replicates === 'object'
+							? authorization.replicates.receivesFrom
+							: undefined);
+					receiveExcludedTables =
+						getExcludedTablesForRouteEntries(receivesFromEntries, remoteNodeName, databaseName) ?? null;
+				}
+				if (tableDecoder && receiveExcludedTables?.has(tableDecoder.name)) {
+					logger.trace?.(
+						connectionId,
+						'dropping incoming replication for excluded table',
+						databaseName + '.' + tableDecoder.name,
+						'from',
+						remoteNodeName
+					);
+					decoder.position = start + eventLength;
+					continue;
+				}
 				let residencyList;
 				if (auditRecord.residencyId) {
 					residencyList = receivedResidencyLists[auditRecord.residencyId];
@@ -1640,6 +1953,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 				}
 				const id = auditRecord.recordId;
 				event = undefined; // reset before each decode attempt
+				let receivedBlobs: any[] | undefined;
 				try {
 					decodeBlobsWithWrites(
 						() => {
@@ -1658,7 +1972,11 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 							};
 						},
 						auditStore?.rootStore,
-						(blob) => receiveBlobs(blob, id)
+						(blob) => {
+							const localBlob = receiveBlobs(blob, id);
+							(receivedBlobs ??= []).push(localBlob);
+							return localBlob;
+						}
 					);
 				} catch (error) {
 					logger.error?.(
@@ -1671,11 +1989,23 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						error
 					);
 				}
-				replicationSharedStatus[RECEIVED_VERSION_POSITION] = Math.max(
-					// ensure monotonicity
-					auditRecord.version,
-					replicationSharedStatus[RECEIVED_VERSION_POSITION]
-				);
+				if (!event && receivedBlobs) {
+					// decode failed mid-message; the blobs that were already accepted will never be referenced. Give in-flight reads
+					// a window to complete, then unlink the files. (mirrors the pattern at the relocate path above.)
+					setTimeout(() => receivedBlobs.forEach(deleteBlob), 60000).unref();
+				}
+				// During a bulk copy, do NOT advance the received-version watermark per copied record:
+				// records arrive in primary-key order carrying their original (possibly newest) versions, so a
+				// single record at the leader's latest timestamp would otherwise let checkSyncStatus mark the
+				// clone Available with rows still uncopied. The watermark is advanced to copyStartTime by the
+				// single end_txn after the whole copy (the REMOTE_SEQUENCE_UPDATE branch above).
+				if (!inCopyMode) {
+					replicationSharedStatus[RECEIVED_VERSION_POSITION] = Math.max(
+						// ensure monotonicity
+						auditRecord.version,
+						replicationSharedStatus[RECEIVED_VERSION_POSITION]
+					);
+				}
 				replicationSharedStatus[RECEIVED_TIME_POSITION] = Date.now();
 				replicationSharedStatus[RECEIVING_STATUS_POSITION] = RECEIVING_STATUS_RECEIVING;
 
@@ -1706,6 +2036,13 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						} finally {
 							removePauseReason();
 						}
+						lastYieldTime = performance.now();
+					} else if (performance.now() - lastYieldTime >= RECEIVE_YIELD_INTERVAL) {
+						// The high-water-mark pause only fires under heap pressure. When the consumer keeps
+						// up, yield on a time budget anyway so a large message doesn't decode in one
+						// synchronous turn and stall ping responses (see RECEIVE_YIELD_INTERVAL).
+						await new Promise(setImmediate);
+						lastYieldTime = performance.now();
 					}
 				}
 				decoder.position = start + eventLength;
@@ -1727,6 +2064,9 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 					`Commit backlog causing replication back-pressure, requesting that ${remoteNodeName} pause replication`
 				);
 			}
+			// Is this a bulk-copy frame? Only frames received before COPY_COMPLETE are part of the
+			// primary-key copy; later audit-replay frames must not be recorded as the resume cursor.
+			const isCopyFrame = inCopyMode && !copyCompleteReceived;
 			tableSubscriptionToReplicator.send({
 				type: 'end_txn',
 				localTime: lastSequenceIdReceived,
@@ -1754,6 +2094,18 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 					// we correctly resend the blobs)
 					if (outstandingBlobsToFinish.length > 0) await Promise.all(outstandingBlobsToFinish);
 					logger.trace?.('All blobs finished');
+					// Persist/clear the resume cursor only AFTER this batch's blobs are durable too. The cursor
+					// means "fully copied through this key"; advancing it before blob writes finish would let a
+					// crash skip re-requesting an unfinished blob, leaving the record pointing at missing data.
+					if (isCopyFrame && event && copyFromNodeId !== undefined) {
+						tableSubscriptionToReplicator?.dbisDB?.put([Symbol.for('copyCursor'), copyFromNodeId], {
+							copyStartTime: copyModeStartTime,
+							currentTable: event.table,
+							afterKey: event.id,
+						});
+					}
+					// once the last copied batch (incl. its blobs) is durable, it's safe to drop the cursor
+					maybeFinishCopy();
 					if (!lastSequenceIdCommitted && sequenceIdReceived) {
 						logger.trace?.(connectionId, 'queuing confirmation of a commit at', sequenceIdReceived);
 						setTimeout(() => {
@@ -1773,6 +2125,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		}
 	}
 	ws.on('ping', resetPingTimer);
+	ws.on('pong', resetPingTimer);
 	ws.on('pong', () => {
 		if (options.connection) {
 			// every pong we can use to update our connection information (and latency)
@@ -1797,7 +2150,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		// cleanup
 		wsClosed = true;
 		clearInterval(sendPingInterval);
-		clearTimeout(receivePingTimer);
+		receiveWatchdog?.stop();
 		clearInterval(blobsTimer);
 		if (auditSubscription) auditSubscription.emit('close');
 		if (subscriptionRequest) subscriptionRequest.end();
@@ -1855,7 +2208,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						])
 					);
 				}
-				lastBuffer = buffer;
+				lastBuffer = buffer as Buffer;
 				if (ws._socket.writableNeedDrain) {
 					logger.debug?.('draining', id);
 					await new Promise((resolve) => ws._socket.once('drain', resolve));
@@ -1911,7 +2264,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		stream.connectedToBlob = true;
 		stream.lastChunk = Date.now();
 		stream.recordId = id;
-		if (remoteBlob.size === undefined && stream.expectedSize) remoteBlob.size = stream.expectedSize;
+		if (remoteBlob.size === undefined && stream.expectedSize) (remoteBlob as any).size = stream.expectedSize;
 		const localBlob = stream.blob ?? createBlob(stream, remoteBlob);
 		stream.blob = localBlob; // record the blob so we can reuse it if another request uses the same blob
 
@@ -1935,7 +2288,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 					const index = outstandingBlobsToFinish.indexOf(tracked);
 					if (index > -1) outstandingBlobsToFinish.splice(index, 1);
 				});
-			tracked.blobId = blobId;
+			(tracked as any).blobId = blobId;
 			outstandingBlobsToFinish.push(tracked);
 		}
 		return localBlob;
@@ -2027,14 +2380,20 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		const nodeSubscriptions = options.connection?.nodeSubscriptions.map((node: any) => {
 			const tableSubs = [];
 			let { replicateByDefault } = node;
+			// Tables excluded by this node's receivesFrom config for this peer+database
+			const receiverExcludedTables = getExcludedTablesForRouteEntries(
+				node.routeReplicates?.receivesFrom,
+				node.name,
+				databaseName
+			);
 			if (node.subscriptions) {
 				// if the node has explicit subscriptions, we need to use that to determine subscriptions
 				for (const subscription of node.subscriptions) {
 					// if there is an explicit subscription listed
 					if (subscription.subscribe && (subscription.schema || subscription.database) === databaseName) {
 						const tableName = subscription.table;
-						if (tables?.[tableName]?.replicate !== false)
-							// if replication is enabled for this table
+						if (tables?.[tableName]?.replicate !== false && !receiverExcludedTables?.has(tableName))
+							// if replication is enabled for this table and not excluded
 							tableSubs.push(tableName);
 					}
 				}
@@ -2042,15 +2401,26 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 			} else {
 				// note that if replicateByDefault is enabled, we are listing the *excluded* tables
 				for (const tableName in tables) {
-					if (replicateByDefault ? tables[tableName].replicate === false : tables[tableName].replicate) {
+					if (
+						replicateByDefault
+							? tables[tableName].replicate === false || receiverExcludedTables?.has(tableName)
+							: tables[tableName].replicate && !receiverExcludedTables?.has(tableName)
+					) {
 						tableSubs.push(tableName);
 					}
 				}
 			}
 
 			const nodeId = auditStore && getIdOfRemoteNode(node.name, auditStore);
-			auditStore.ensureLogExists?.(node.name);
+			auditStore?.ensureLogExists?.(node.name);
 			const sequenceEntry = tableSubscriptionToReplicator?.dbisDB?.get([Symbol.for('seq'), nodeId]) ?? 1;
+			// A persisted copy cursor means a bulk copy from this node was interrupted mid-stream. We must
+			// resume that copy (not treat the persisted seqId as a normal start point — the un-copied table
+			// data predates copyStartTime and would never be delivered by an audit-log resume).
+			const copyCursor =
+				nodeId === undefined
+					? undefined
+					: tableSubscriptionToReplicator?.dbisDB?.get([Symbol.for('copyCursor'), nodeId]);
 			// if we are connected directly to the node, we start from the last sequence number we received at the top level
 			let startTime = Math.max(
 				sequenceEntry?.seqId ?? 1,
@@ -2097,6 +2467,18 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 					startTime = Date.now() - 60000;
 				}
 			}
+			let copyResume;
+			if (copyCursor) {
+				startTime = 0; // request a copy; the cursor tells the leader where to resume
+				copyResume = {
+					copyStartTime: copyCursor.copyStartTime,
+					currentTable: copyCursor.currentTable,
+					afterKey: copyCursor.afterKey,
+				};
+				logger.warn?.(
+					`Resuming interrupted copy of database ${databaseName} from ${getNodeURL(node)} at table ${copyCursor.currentTable}`
+				);
+			}
 			logger.trace?.(connectionId, 'defining subscription request', node.name, databaseName, new Date(startTime));
 			return {
 				name: node.name,
@@ -2105,6 +2487,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 				startTime,
 				isLeader: node.isLeader,
 				endTime: node.endTime,
+				copyResume, // present only when resuming an interrupted bulk copy
 			};
 		});
 		let excluded: string[];
@@ -2203,7 +2586,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 		const thisNodeName = getThisNodeName();
 		if (thisNodeName === remoteNodeName) {
 			if (!thisNodeName) throw new Error('Node name not defined');
-			else throw new Error('Should not connect to self', thisNodeName);
+			else throw new Error('Should not connect to self: ' + thisNodeName);
 		}
 		sendNodeDBName(thisNodeName, databaseName);
 		return true;