@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/dist/replication/replicationConnection.js

@@ -37,6 +37,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NodeReplicationConnection = exports.databaseSubscriptions = exports.tableUpdateListeners = exports.RECEIVING_STATUS_RECEIVING = exports.RECEIVING_STATUS_WAITING = exports.BACK_PRESSURE_RATIO_POSITION = exports.RECEIVING_STATUS_POSITION = exports.LATENCY_POSITION = exports.SENDING_TIME_POSITION = exports.RECEIVED_TIME_POSITION = exports.RECEIVED_VERSION_POSITION = exports.CONFIRMATION_STATUS_POSITION = exports.OPERATION_REQUEST = void 0;
+exports.shouldTerminateIdlePing = shouldTerminateIdlePing;
+exports.createReceiveWatchdog = createReceiveWatchdog;
 exports.createWebSocket = createWebSocket;
 exports.replicateOverWS = replicateOverWS;
 const databases_ts_1 = require("../core/resources/databases.js");
@@ -45,10 +47,11 @@ const nodeIdMapping_ts_1 = require("../core/resources/nodeIdMapping.js");
 const transactionBroadcast_ts_1 = require("../core/resources/transactionBroadcast.js");
 const replicator_ts_1 = require("./replicator.js");
 const nodeName_ts_1 = require("../core/server/nodeName.js");
-const environmentManager_js_1 = __importDefault(require("../core/utility/environment/environmentManager.js"));
+const env = __importStar(require("../core/utility/environment/environmentManager.js"));
 const hdbTerms_ts_1 = require("../core/utility/hdbTerms.js");
 const RecordEncoder_ts_1 = require("../core/resources/RecordEncoder.js");
 const msgpackr_1 = require("msgpackr");
+const structon_1 = require("structon");
 const ws_1 = require("ws");
 const worker_threads_1 = require("worker_threads");
 const harper_logger_js_1 = __importDefault(require("../core/utility/logging/harper_logger.js"));
@@ -65,6 +68,13 @@ const blob_ts_1 = require("../core/resources/blob.js");
 const node_stream_1 = require("node:stream");
 const lmdb_1 = require("lmdb");
 const logger = forComponent('replication').conditional;
+// msgpackr v2 removed the built-in `randomAccessStructure` option; that random-access
+// struct support now lives in the `structon` package (the same wrapper core's
+// RecordEncoder uses). Replication decoders must be structon-wrapped so they install
+// the `_readStruct` hook and can decode typed-struct records — without it, struct
+// marker bytes (0x20–0x3f) decode as plain integers and the decode aborts with
+// "Data read, but end of buffer not reached".
+const StructonPackr = (0, structon_1.createStructon)(msgpackr_1.Packr);
 // these are the codes we use for the different commands
 const SUBSCRIPTION_REQUEST = 129;
 const NODE_NAME = 140;
@@ -81,6 +91,8 @@ const COMMITTED_UPDATE = 144;
 const DB_SCHEMA = 145;
 const BLOB_CHUNK = 146;
 const SUBSCRIPTION_UPDATE = 147;
+const COPY_START = 148; // leader -> follower: a bulk table copy is starting; carries copyStartTime
+const COPY_COMPLETE = 149; // leader -> follower: the bulk table copy finished; follower clears its resume cursor
 exports.CONFIRMATION_STATUS_POSITION = 0;
 exports.RECEIVED_VERSION_POSITION = 1;
 exports.RECEIVED_TIME_POSITION = 2;
@@ -90,12 +102,22 @@ exports.RECEIVING_STATUS_POSITION = 5;
 exports.BACK_PRESSURE_RATIO_POSITION = 6;
 exports.RECEIVING_STATUS_WAITING = 0;
 exports.RECEIVING_STATUS_RECEIVING = 1;
-const MAX_PAYLOAD = environmentManager_js_1.default.get('replication_maxPayload') ?? 100_000_000;
+const MAX_PAYLOAD = env.get('replication_maxPayload') ?? 100_000_000;
 // When receiving a replication message, we apply per-record backpressure to keep a single
 // large batch from synchronously decoding thousands of records and ballooning the worker
 // heap past its limit. If the local replicator queue grows beyond this threshold we pause
 // the WS connection and wait for it to drain before continuing the decode loop.
-const RECEIVE_EVENT_HIGH_WATER_MARK = environmentManager_js_1.default.get('replication_receiveEventHighWaterMark') ?? 100;
+const RECEIVE_EVENT_HIGH_WATER_MARK = env.get('replication_receiveEventHighWaterMark') ?? 100;
+// Even when the consumer keeps up (queue below the high-water mark), a single large inbound message
+// would otherwise decode thousands of records in one synchronous turn — pegging the worker, blocking
+// replication ping responses, and tripping core's "JavaScript execution has taken too long" monitor
+// (MAX_EVENT_DELAY_TIME = 3 s). Yield the event loop at least this often (ms) while decoding so the
+// worker stays responsive during a bulk copy/clone.
+const RECEIVE_YIELD_INTERVAL = env.get('replication_receiveYieldInterval') ?? 100;
+// During a bulk clone copy the leader flushes a checkpoint transaction every this many records so the
+// follower commits incrementally and persists a resume cursor. On reconnect the copy resumes from that
+// cursor instead of restarting from zero. Larger = less overhead but coarser resume granularity.
+const COPY_CHECKPOINT_RECORDS = env.get('replication_copyCheckpointRecords') ?? 1000;
 exports.tableUpdateListeners = new Map();
 // This a map of the database name to the subscription object, for the subscriptions from our tables to the replication module
 // when we receive messages from other nodes, we then forward them on to as a notification on these subscriptions
@@ -108,7 +130,91 @@ const SKIPPED_MESSAGE_SEQUENCE_UPDATE_DELAY = 300;
 // We want it be fairly quick so we can let the sending node know that we have received and committed the update.
 // (but still allow for batching so we aren't sending out a message for every update under load)
 const COMMITTED_UPDATE_DELAY = 2;
-const PING_INTERVAL = 30000;
+const PING_INTERVAL = env.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_PINGINTERVAL) ?? 30000;
+// Time (ms) without any socket activity before a connection is treated as dead.
+const PING_TIMEOUT = env.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_PINGTIMEOUT) ?? PING_INTERVAL * 2;
+// The receive-side watchdog fires after this much silence on a replication WS. Both client and
+// server arm it: the client also runs an active 30s sendPing tick that should normally catch a
+// silent peer first, but if that tick is missed (event-loop stall, ws.terminate() not propagating
+// to 'close', etc.) this timer-based watchdog is the belt-and-suspenders that forces the
+// reconnect — see harper-pro#233.
+const RECEIVE_SILENCE_THRESHOLD_MS = PING_TIMEOUT;
+/**
+ * Decide whether an idle replication connection should be terminated as dead.
+ *
+ * Liveness is measured from the last observed socket byte movement (in either direction), not from a
+ * single ping interval. A bulk transfer — notably the initial clone copy of a large table — makes
+ * slow but real progress: the sender's socket buffer drains in bursts as the peer consumes, so bytes
+ * keep moving within the timeout window even while it is otherwise stalled. A genuinely dead or
+ * unreachable peer moves no bytes at all, so it still trips the timeout. We deliberately do NOT exempt
+ * the sender's `isPausedForBackPressure` drain-wait here: if the peer dies after we have filled our
+ * socket buffer, the drain event never fires, and exempting it would hang the connection forever.
+ *
+ * The one exemption is `pauseReasons > 0`: the receiver has intentionally stopped reading to drain its
+ * own queue. That stall is local and self-clearing (it does not depend on the peer), so it is never a
+ * death signal; the caller keeps liveness fresh while paused and resumes normal detection afterward.
+ */
+function shouldTerminateIdlePing(idleMs, pingTimeout, pauseReasons) {
+    return pauseReasons === 0 && idleMs >= pingTimeout;
+}
+/**
+ * Receive-side silence watchdog. Arm with `reset()` whenever incoming activity is observed
+ * (peer ping, pong, message). If `intervalMs` elapses with the underlying socket's `bytesRead`
+ * unchanged, `onSilence` is invoked exactly once.
+ *
+ * Only `bytesRead` is checked — `bytesWritten` reflects our own outbound traffic (pings, blob
+ * sends) and is not proof that the peer is alive. Including it would let our own keepalive
+ * pings suppress the watchdog in the exact missed-sendPing scenario this is meant to recover.
+ *
+ * Callers should suspend the watchdog (via `stop()`) when the underlying WS is intentionally
+ * paused for backpressure: `bytesRead` is frozen by design while paused and would otherwise
+ * cause a spurious termination of a healthy connection.
+ *
+ * Exported so the timer logic can be exercised in isolation by `unitTests/replication/
+ * receiveWatchdog.test.mjs` — production callers go through `replicateOverWS`.
+ */
+function createReceiveWatchdog(opts) {
+    let timer;
+    let bytesReadAtArm = 0;
+    let lastResetAt = 0;
+    // Coalesce rapid reset() calls (e.g. message frames arriving thousands of times per second
+    // during a large copy) so we do not churn setTimeout/clearTimeout per frame. Granularity loss
+    // is small relative to intervalMs — at worst the watchdog fires this much earlier or later.
+    const throttleMs = Math.min(1000, Math.max(100, opts.intervalMs / 30));
+    function check() {
+        const current = opts.getBytesRead();
+        if (current === bytesReadAtArm) {
+            timer = undefined;
+            opts.onSilence();
+            return;
+        }
+        // Bytes advanced since the last arm — but the activity may have been swallowed by the
+        // reset() throttle, so we cannot rely on an external caller to re-arm us. Re-arm from
+        // the new baseline; otherwise a throttled-reset-then-silence sequence would leave the
+        // watchdog permanently inactive (see PR #234 review).
+        bytesReadAtArm = current;
+        lastResetAt = Date.now();
+        timer = setTimeout(check, opts.intervalMs).unref();
+    }
+    return {
+        reset() {
+            const now = Date.now();
+            if (timer && now - lastResetAt < throttleMs)
+                return;
+            lastResetAt = now;
+            if (timer)
+                clearTimeout(timer);
+            bytesReadAtArm = opts.getBytesRead();
+            timer = setTimeout(check, opts.intervalMs).unref();
+        },
+        stop() {
+            if (timer) {
+                clearTimeout(timer);
+                timer = undefined;
+            }
+        },
+    };
+}
 let secureContexts;
 let replicationSecureContext;
 async function createWebSocket(url, options) {
@@ -196,6 +302,7 @@ class NodeReplicationConnection extends events_1.EventEmitter {
     databaseName;
     nodeName;
     authorization;
+    tentativeNode;
     constructor(url, subscription, databaseName, nodeName, authorization) {
         super();
         this.url = url;
@@ -205,6 +312,8 @@ class NodeReplicationConnection extends events_1.EventEmitter {
         this.nodeName = this.nodeName ?? (0, replicator_ts_1.urlToNodeName)(url);
     }
     async connect() {
+        if (this.intentionallyUnsubscribed)
+            return;
         if (!this.session)
             this.resetSession();
         // TODO: Need to do this specifically for each node
@@ -318,7 +427,7 @@ class NodeReplicationConnection extends events_1.EventEmitter {
     }
     unsubscribe() {
         this.intentionallyUnsubscribed = true;
-        this.socket.close(1008, 'No longer subscribed');
+        this.socket?.close(1008, 'No longer subscribed');
     }
     getRecord(request) {
         return this.session.then((session) => {
@@ -366,7 +475,33 @@ function replicateOverWS(ws, options, authorization) {
     if (remoteNodeName && options.connection)
         options.connection.nodeName = remoteNodeName;
     let lastSequenceIdReceived, lastSequenceIdCommitted;
-    let sendPingInterval, receivePingTimer, lastPingTime, skippedMessageSequenceUpdateTimer;
+    // Bulk-copy resume state (receiver side). While inCopyMode, each committed batch persists a cursor
+    // (copyStartTime + last committed table/key) so an interrupted copy can resume instead of restarting.
+    let inCopyMode = false;
+    let copyModeStartTime = 0;
+    let copyFromNodeId; // local id of the node we are copying from — the key for the persisted cursor
+    let copyCompleteReceived = false;
+    // Finish the copy — leave copy mode and remove the resume cursor — only once COPY_COMPLETE has been
+    // received AND every copied batch has committed (outstandingCommits drained, which includes the final
+    // end_txn that advances the resume seqId to copyStartTime). We deliberately stay in copy mode until
+    // then so batches still committing keep advancing the cursor (onCommit). Finishing earlier — e.g.
+    // synchronously when COPY_COMPLETE is decoded while batches are still queued — would freeze the cursor
+    // and risk a crash that loses both the cursor and the not-yet-durable rows, leaving the next start to
+    // resume from seqId with gaps.
+    function maybeFinishCopy() {
+        if (copyCompleteReceived && outstandingCommits === 0) {
+            // guard only the cursor removal on a known node id; ALWAYS exit copy mode, otherwise a
+            // COPY_START whose getIdOfRemoteNode returned undefined would strand the node in copy mode
+            // (received-version watermark suppressed) and it could never reach Available.
+            if (copyFromNodeId !== undefined)
+                tableSubscriptionToReplicator?.dbisDB?.remove([Symbol.for('copyCursor'), copyFromNodeId]);
+            inCopyMode = false;
+            copyCompleteReceived = false;
+            copyFromNodeId = undefined;
+        }
+    }
+    let sendPingInterval, lastPingTime, skippedMessageSequenceUpdateTimer;
+    let receiveWatchdog;
     let blobsTimer;
     const DELAY_CLOSE_TIME = 60000; // amount of time to wait before closing the connection if we haven't any activity and there are no subscriptions
     let delayedClose;
@@ -374,42 +509,77 @@ function replicateOverWS(ws, options, authorization) {
     // track bytes read and written so we can verify if a connection is really dead on pings
     let bytesRead = 0;
     let bytesWritten = 0;
-    const blobTimeout = environmentManager_js_1.default.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_BLOBTIMEOUT) ?? 120000;
+    // wall-clock time of the last observed socket activity (bytes moved in either direction); the
+    // keep-alive timeout is measured from this so a legitimately slow-but-progressing transfer stays
+    // alive while a truly idle/dead peer is still terminated.
+    let lastByteActivity = performance.now();
+    // Multiple independent conditions can ask to pause receive on this WS (commit backlog, consumer
+    // queue full, blob write backpressure). We refcount the reasons so that resuming one does not race
+    // ahead of another that still wants the WS paused. Declared before the ping setup below because the
+    // immediate sendPing() reads it.
+    let pauseReasons = 0;
+    const blobTimeout = env.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_BLOBTIMEOUT) ?? 120000;
     const blobsInFlight = new Map();
     const outstandingBlobsToFinish = [];
     let outstandingBlobsBeingSent = 0;
     let blobSentCallback;
+    // Refresh the keep-alive liveness clock from observed socket byte movement. If the underlying
+    // _socket isn't observable (test mocks, pre-connect, or a change in the ws library internals),
+    // bytesRead/bytesWritten read as undefined; we can't measure activity, so treat the connection as
+    // live rather than let the keep-alive falsely terminate a healthy peer.
+    function noteByteActivity() {
+        const read = ws._socket?.bytesRead;
+        const written = ws._socket?.bytesWritten;
+        if (read === undefined || written === undefined || read !== bytesRead || written !== bytesWritten) {
+            lastByteActivity = performance.now();
+        }
+    }
     if (options.url) {
         const sendPing = () => {
-            // if we have not received a message in the last ping interval, we should terminate the connection (but check to make sure we aren't just waiting for other data to flow)
-            if (lastPingTime && bytesRead === ws._socket?.bytesRead && bytesWritten === ws._socket?.bytesWritten)
-                ws.terminate(); // timeout
-            else {
-                lastPingTime = performance.now();
-                ws.ping();
-                bytesRead = ws._socket?.bytesRead;
-                bytesWritten = ws._socket?.bytesWritten;
+            // Note any socket activity since the last interval (incoming pong/data or our send buffer
+            // draining as the peer consumes) — either proves the peer is still alive.
+            noteByteActivity();
+            if (shouldTerminateIdlePing(performance.now() - lastByteActivity, PING_TIMEOUT, pauseReasons)) {
+                ws.terminate(); // no socket activity within the timeout — peer is gone
+                return;
             }
+            // While paused for receiver backpressure, keep our own liveness fresh: the stall is local and
+            // self-clearing (it doesn't depend on the peer), so we must not time the peer out for it.
+            if (pauseReasons > 0)
+                lastByteActivity = performance.now();
+            // Always send the keep-alive ping. ws.pause() only stops reads, not writes, and the accepted
+            // peer relies on our pings to keep its own receive timer alive even when it has no data to send
+            // us. Record byte counts AFTER the ping so the ping's own bytes aren't later mistaken for peer
+            // activity.
+            lastPingTime = performance.now();
+            ws.ping();
+            bytesRead = ws._socket?.bytesRead;
+            bytesWritten = ws._socket?.bytesWritten;
         };
         sendPingInterval = setInterval(sendPing, PING_INTERVAL).unref();
         sendPing(); // send the first ping immediately so we can measure latency
     }
-    else {
-        resetPingTimer();
-    }
+    // Both client and server arm the receive watchdog. On the client this is independent of the
+    // sendPing tick above: if that tick is missed or its ws.terminate() does not propagate a
+    // 'close' event, the watchdog forces the reconnect path. See harper-pro#233 for the failure
+    // modes observed in the field.
+    receiveWatchdog = createReceiveWatchdog({
+        intervalMs: RECEIVE_SILENCE_THRESHOLD_MS,
+        getBytesRead: () => ws._socket?.bytesRead ?? 0,
+        onSilence: () => {
+            // Warn-level: if the active sendPing was healthy this watchdog should not have fired,
+            // so it is a signal that something is wrong upstream (event-loop stall, keepalive timer
+            // misbehaving, peer accepting bytes but not progressing the protocol). Surface it so
+            // operators have something to grep for.
+            const dbContext = databaseName ? ` (db: "${databaseName}")` : '';
+            const direction = options.url ? 'no activity from' : 'no ping from';
+            logger.warn?.(`Receive watchdog: ${direction} ${remoteNodeName}${dbContext} for ${RECEIVE_SILENCE_THRESHOLD_MS}ms — terminating connection and reconnecting`);
+            ws.terminate();
+        },
+    });
+    const resetPingTimer = receiveWatchdog.reset;
+    resetPingTimer();
     ws._socket?.setMaxListeners(200); // we should allow a lot of drain listeners for concurrent blob streams
-    function resetPingTimer() {
-        clearTimeout(receivePingTimer);
-        bytesRead = ws._socket?.bytesRead;
-        bytesWritten = ws._socket?.bytesWritten;
-        receivePingTimer = setTimeout(() => {
-            // double check to make sure we aren't just waiting for other data to flow
-            if (bytesRead === ws._socket?.bytesRead && bytesWritten === ws._socket?.bytesWritten) {
-                logger.warn?.(`Timeout waiting for ping from ${remoteNodeName}, terminating connection and reconnecting`);
-                ws.terminate();
-            }
-        }, PING_INTERVAL * 2).unref();
-    }
     let ratioOfBackPressureTime = 0;
     let lastBackPressureCheck = 0;
     let isPausedForBackPressure = false;
@@ -448,30 +618,37 @@ function replicateOverWS(ws, options, authorization) {
     const residencyMap = [];
     const sentResidencyLists = [];
     const receivedResidencyLists = [];
-    const MAX_OUTSTANDING_COMMITS = environmentManager_js_1.default.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_RECORDCONCURRENCY) ?? 150; // maximum before requesting that other nodes pause
-    const MAX_OUTSTANDING_BLOBS_BEING_SENT = environmentManager_js_1.default.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
+    const MAX_OUTSTANDING_COMMITS = env.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_RECORDCONCURRENCY) ?? 150; // maximum before requesting that other nodes pause
+    const MAX_OUTSTANDING_BLOBS_BEING_SENT = env.get(hdbTerms_ts_1.CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
     let outstandingCommits = 0;
     let lastStructureLength = 0;
-    // Multiple independent conditions can ask to pause receive on this WS (commit backlog,
-    // consumer queue full, blob write backpressure). We refcount the reasons so that resuming
-    // one does not race ahead of another that still wants the WS paused.
-    let pauseReasons = 0;
     let commitBacklogPaused = false;
     function addPauseReason() {
-        if (pauseReasons === 0)
+        if (pauseReasons === 0) {
             ws.pause();
+            // Suspend the receive watchdog while the socket is intentionally paused — `bytesRead`
+            // is frozen by `ws.pause()` so the byte check cannot tell legitimate backpressure
+            // from peer silence, and firing here would terminate a healthy mid-ingest connection.
+            receiveWatchdog?.stop();
+        }
         pauseReasons++;
     }
     function removePauseReason() {
         if (pauseReasons === 0)
             return;
         pauseReasons--;
-        if (pauseReasons === 0)
+        if (pauseReasons === 0) {
             ws.resume();
+            // Restart the silence window from the resume point — we deliberately do not penalize
+            // the connection for the time it spent paused.
+            receiveWatchdog?.reset();
+        }
     }
     let subscriptionRequest, auditSubscription;
     let nodeSubscriptions;
     let excludedNodes; // list of nodes to exclude from this subscription
+    // undefined = not yet computed; null = computed, no exclusions; Set = tables to drop on receive
+    let receiveExcludedTables;
     let remoteShortIdToLocalId;
     let subscribedNodeIds; // map of node IDs to their subscription time ranges
     // Serialize message handling so that async backpressure inside onWSMessage doesn't allow
@@ -481,6 +658,9 @@ function replicateOverWS(ws, options, authorization) {
     let messageProcessing = Promise.resolve();
     let wsClosed = false;
     ws.on('message', (body) => {
+        // Reset the receive watchdog synchronously on every frame — async processing below may
+        // take a long time and we want a single late frame to count as proof of life immediately.
+        resetPingTimer();
         messageProcessing = messageProcessing.then(() => (wsClosed ? undefined : onWSMessage(body)), () => (wsClosed ? undefined : onWSMessage(body)));
     });
     let authorizationFinished = false;
@@ -683,9 +863,8 @@ function replicateOverWS(ws, options, authorization) {
                         // of the table id to the decoder so we can decode the binary data for each table.
                         tableDecoders[tableId] = {
                             name: tableName,
-                            decoder: new msgpackr_1.Packr({
+                            decoder: new StructonPackr({
                                 useBigIntExtension: true,
-                                randomAccessStructure: true,
                                 freezeData: true,
                                 typedStructs: data.typedStructs,
                                 structures: data.structures,
@@ -714,6 +893,20 @@ function replicateOverWS(ws, options, authorization) {
                         (0, write_ts_1.recordAction)(body.length, 'bytes-received', `${remoteNodeName}.${databaseName}`, 'replication', 'committed-update');
                         getSharedStatus().buffer.notify();
                         break;
+                    case COPY_START:
+                        // the leader is (re)starting a bulk copy; track a resume cursor for it
+                        inCopyMode = true;
+                        copyModeStartTime = data; // copyStartTime anchor chosen by the leader
+                        copyFromNodeId = (0, nodeIdMapping_ts_1.getIdOfRemoteNode)(remoteNodeName, auditStore);
+                        logger.debug?.(connectionId, 'bulk copy starting from', remoteNodeName, new Date(copyModeStartTime));
+                        break;
+                    case COPY_COMPLETE:
+                        // Copy signalled complete. Stay in copy mode so batches still committing keep advancing the
+                        // cursor; maybeFinishCopy exits copy mode and clears the cursor once those commits drain.
+                        copyCompleteReceived = true;
+                        maybeFinishCopy();
+                        logger.debug?.(connectionId, 'bulk copy complete from', remoteNodeName);
+                        break;
                     case SEQUENCE_ID_UPDATE:
                         // we need to record the sequence number that the remote node has received
                         lastSequenceIdReceived = data;
@@ -759,20 +952,38 @@ function replicateOverWS(ws, options, authorization) {
                                 if (stream.connectedToBlob)
                                     blobsInFlight.delete(fileId);
                             }
+                            else if (stream.destroyed || stream.writableEnded) {
+                                // The stream was already torn down before this mid-blob chunk arrived —
+                                // typically because saveBlob's pipeline failed (e.g. ENOENT on
+                                // createWriteStream) and destroyed the PassThrough source, firing 'close'.
+                                // We must NOT fall into the backpressure branch below: writing to a dead
+                                // stream returns false, and pausing to wait for a 'drain'/'close' that has
+                                // already fired strands the pause reason forever, wedging the entire
+                                // receive loop (observed in prod: receiver goes silent, sender stuck
+                                // reconnecting, replication never recovers). Drop the orphaned chunk and
+                                // forget the stream instead.
+                                blobsInFlight.delete(fileId);
+                            }
                             else if (!stream.write(blobBody)) {
                                 // The PassThrough's internal queue is over its HWM, meaning the downstream
                                 // file write (via pipeline in saveBlob) can't keep up. Pause the WS until the
                                 // stream drains so blob chunks don't accumulate in memory faster than they
-                                // can be flushed to disk. Also listen for 'close' so a destroyed stream
-                                // (e.g. saveBlob error) doesn't strand the pause reason.
-                                addPauseReason();
-                                const release = () => {
-                                    stream.off('drain', release);
-                                    stream.off('close', release);
-                                    removePauseReason();
-                                };
-                                stream.on('drain', release);
-                                stream.on('close', release);
+                                // can be flushed to disk.
+                                if (stream.destroyed || stream.writableEnded) {
+                                    // write() itself may have torn the stream down (e.g. a late error). If so,
+                                    // 'drain'/'close' won't arrive — skip pausing rather than strand the reason.
+                                    blobsInFlight.delete(fileId);
+                                }
+                                else {
+                                    addPauseReason();
+                                    const release = () => {
+                                        stream.off('drain', release);
+                                        stream.off('close', release);
+                                        removePauseReason();
+                                    };
+                                    stream.on('drain', release);
+                                    stream.on('close', release);
+                                }
                             }
                         }
                         catch (error) {
@@ -994,6 +1205,10 @@ function replicateOverWS(ws, options, authorization) {
                         let tableById;
                         let currentSequenceId = Infinity; // the last sequence number in the audit log that we have processed, set this with a finite number from the subscriptions
                         let sentSequenceId; // the last sequence number we have sent
+                        // Tables excluded from outgoing replication to this peer+database (from sendsTo config)
+                        const sendExcludedTables = authorization?.replicates && typeof authorization.replicates === 'object'
+                            ? (0, knownNodes_ts_1.getExcludedTablesForRouteEntries)(authorization.replicates.sendsTo, remoteNodeName, databaseName)
+                            : null;
                         const sendAuditRecord = (auditRecord, localTime) => {
                             if (auditRecord.type === 'end_txn') {
                                 if (currentTransaction.txnTime) {
@@ -1019,6 +1234,9 @@ function replicateOverWS(ws, options, authorization) {
                                 }
                             }
                             const table = tableEntry.table;
+                            if (sendExcludedTables?.has(table.tableName)) {
+                                return skipAuditRecord();
+                            }
                             const primaryStore = table.primaryStore;
                             const encoder = primaryStore.encoder;
                             // Force a reload the first time this connection touches each table:
@@ -1044,7 +1262,9 @@ function replicateOverWS(ws, options, authorization) {
                             // in the subscribedNodeIds list, than it matches the subscription
                             const matchesSubscription = (excludedNodes && timeRange === undefined) ||
                                 // if it is in the list, we check the timestamps to verify it matches
-                                (timeRange && timeRange.startTime < localTime && (!timeRange.endTime || timeRange.endTime > localTime));
+                                (timeRange &&
+                                    timeRange.startTime < localTime &&
+                                    (!timeRange.endTime || timeRange.endTime > localTime));
                             if (!matchesSubscription) {
                                 if (DEBUG_MODE)
                                     logger.trace?.(connectionId, 'skipping replication update', auditRecord.recordId, 'to:', remoteNodeName, 'from:', nodeId, 'subscribed:', subscribedNodeIds);
@@ -1232,9 +1452,13 @@ function replicateOverWS(ws, options, authorization) {
                             subscriptionToHdbNodes?.end();
                         });
                         // find the earliest start time of the subscriptions
-                        for (const { startTime } of nodeSubscriptions) {
-                            if (startTime < currentSequenceId)
-                                currentSequenceId = startTime;
+                        let copyResume;
+                        for (const subscription of nodeSubscriptions) {
+                            if (subscription.startTime < currentSequenceId)
+                                currentSequenceId = subscription.startTime;
+                            // a follower resuming an interrupted bulk copy sends back where it left off
+                            if (subscription.copyResume)
+                                copyResume = subscription.copyResume;
                         }
                         // wait for internal subscription, might be waiting for a table to be registered
                         (whenSubscribedToHdbNodes || Promise.resolve())
@@ -1301,21 +1525,53 @@ function replicateOverWS(ws, options, authorization) {
                                     isFirst = false;
                                     if (currentSequenceId === 0) {
                                         logger.info?.('Replicating all tables to', remoteNodeName);
-                                        let lastSequenceId = Date.now(); // we need at least a default time point in case there are no records
+                                        // Capture the resume point BEFORE iterating. The bulk copy walks the primary store in
+                                        // key order (snapshot: false), but the follower resumes replication from the audit log in
+                                        // time order. Using copyStartTime — not max(localTime) of the copied records — guarantees
+                                        // the post-copy audit replay re-delivers every write committed during the copy, including
+                                        // ones to keys we already passed; resuming from max(localTime) would skip those (data loss).
+                                        // When the follower is resuming an interrupted copy, keep the original copy start time so
+                                        // the post-copy resume point stays anchored to when the copy first began (see safety note).
+                                        const copyStartTime = copyResume?.copyStartTime ?? Date.now();
                                         const nodeId = (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore);
+                                        // Tell the follower a bulk copy is starting and its anchor time, so it tracks a resume cursor.
+                                        ws.send((0, msgpackr_1.encode)([COPY_START, copyStartTime]));
+                                        let recordsSinceCheckpoint = 0;
+                                        // If resuming, the follower already committed every table before currentTable (records commit
+                                        // in stable iteration order), so skip to currentTable and continue after its last committed key.
+                                        let reachedResumeTable = !copyResume;
+                                        // currentTable must be one the loop below will actually visit (present in `tables` AND passing
+                                        // the same replication filter); otherwise the skip loop never reaches it and would omit every
+                                        // later table. Mirror the loop's own check so a dropped/unreplicated cursor table forces a restart.
+                                        if (copyResume && !tableToTableEntry(tables[copyResume.currentTable])) {
+                                            // cursor table is gone or no longer replicated — the skip loop would never reach it and
+                                            // would omit every later table, so recopy from scratch (idempotent puts, copyStartTime reset)
+                                            logger.warn?.('Copy-resume table missing or unreplicated, restarting full copy', copyResume.currentTable);
+                                            copyResume = undefined;
+                                            reachedResumeTable = true;
+                                        }
+                                        const resumeCurrentTable = copyResume?.currentTable;
+                                        const resumeAfterKey = copyResume?.afterKey;
                                         for (const tableName in tables) {
-                                            if (!tableToTableEntry(tableName))
-                                                continue; // if we aren't replicating this table, skip it
                                             const table = tables[tableName];
-                                            for (const entry of table.primaryStore.getRange({
-                                                snapshot: false,
-                                                versions: true,
-                                                // values: false, // TODO: eventually, we don't want to decode, we want to use fast binary transfer
-                                            })) {
+                                            if (!tableToTableEntry(table))
+                                                continue; // if we aren't replicating this table, skip it
+                                            if (!reachedResumeTable) {
+                                                if (tableName !== resumeCurrentTable)
+                                                    continue; // already committed on the follower
+                                                reachedResumeTable = true;
+                                            }
+                                            const rangeOptions = { snapshot: false, versions: true };
+                                            // values: false, // TODO: eventually, we don't want to decode, we want to use fast binary transfer
+                                            if (tableName === resumeCurrentTable) {
+                                                // resume this table after the last key the follower committed
+                                                rangeOptions.start = resumeAfterKey;
+                                                rangeOptions.exclusiveStart = true;
+                                            }
+                                            for (const entry of table.primaryStore.getRange(rangeOptions)) {
                                                 if (closed)
                                                     return;
                                                 logger.trace?.(connectionId, 'Copying record from', databaseName, tableName, entry.key, entry.localTime);
-                                                lastSequenceId = Math.max(entry.localTime ?? 1, lastSequenceId);
                                                 getSharedStatus()[exports.SENDING_TIME_POSITION] = 1;
                                                 const encoded = (0, auditStore_ts_1.createAuditEntry)({
                                                     version: entry.version,
@@ -1348,23 +1604,43 @@ function replicateOverWS(ws, options, authorization) {
                                                     extendedType: entry.metadataFlags,
                                                 }, entry.localTime);
                                                 logger.debug?.('sent record from table', entry.key, 'length:', encoded.length, encoded.slice(0, 10));
+                                                // Periodically flush the accumulated records as a message so the follower commits this
+                                                // batch and advances its resume cursor. This is a plain flush with NO sequence update:
+                                                // emitting an end_txn here would advance the follower's received-version watermark to
+                                                // copyStartTime mid-copy, which monitorSync could read as "caught up" and mark the clone
+                                                // Available/cloned with rows still uncopied. (Records with differing versions already flush
+                                                // naturally above; this also bounds same-version bulk data into committable batches.) The
+                                                // watermark is only advanced to copyStartTime by the single end_txn after the whole copy.
+                                                if (++recordsSinceCheckpoint >= COPY_CHECKPOINT_RECORDS && position - encodingStart > 8) {
+                                                    recordsSinceCheckpoint = 0;
+                                                    sendQueuedData();
+                                                    encodingStart = position;
+                                                    currentTransaction.txnTime = 0;
+                                                }
                                             }
                                             logger.info?.('Finished copy table', tableName, remoteNodeName);
                                         }
-                                        currentSequenceId = lastSequenceId;
+                                        currentSequenceId = copyStartTime;
                                         if (!currentTransaction.txnTime) {
-                                            // if we haven't sent any records, force a txn start and end so the subscribers records a timestamp
-                                            currentTransaction.txnTime = lastSequenceId;
-                                            writeFloat64(lastSequenceId);
-                                        }
-                                        if (position - encodingStart > 8) {
-                                            // if we have any queued transactions to send, send them now
-                                            sendAuditRecord({
-                                                type: 'end_txn',
-                                            }, currentSequenceId);
+                                            // no records pending (none sent, or the last batch landed on a checkpoint flush):
+                                            // force a txn so the end_txn below still carries the sequence update
+                                            currentTransaction.txnTime = copyStartTime;
+                                            encodingStart = position;
+                                            writeFloat64(copyStartTime);
                                         }
+                                        // ALWAYS emit the final end_txn at copyStartTime. It carries the REMOTE_SEQUENCE_UPDATE
+                                        // that advances the follower's seqId and received-version watermark to copyStartTime —
+                                        // the sole signal that the copy is synced (per-record watermark advance is suppressed
+                                        // during the copy). Skipping it when the last rows landed exactly on a checkpoint flush
+                                        // would leave the clone unable to ever reach Available.
+                                        sendAuditRecord({
+                                            type: 'end_txn',
+                                        }, currentSequenceId);
+                                        // The full copy is done — tell the follower to clear its resume cursor and fall back to
+                                        // normal audit-log replication from the persisted seqId (which is copyStartTime).
+                                        ws.send((0, msgpackr_1.encode)([COPY_COMPLETE]));
                                         getSharedStatus()[exports.SENDING_TIME_POSITION] = 0;
-                                        currentSequenceId = lastSequenceId;
+                                        currentSequenceId = copyStartTime;
                                     }
                                 }
                                 const logName = subscribedNodeName === (0, nodeName_ts_1.getThisNodeName)() ? 'local' : subscribedNodeName;
@@ -1415,6 +1691,7 @@ function replicateOverWS(ws, options, authorization) {
             let beginTxn = true;
             let event; // could also get txnTime from decoder.getFloat64(0);
             let sequenceIdReceived;
+            let lastYieldTime = performance.now();
             do {
                 getSharedStatus();
                 const eventLength = decoder.readInt();
@@ -1441,6 +1718,23 @@ function replicateOverWS(ws, options, authorization) {
                 if (!tableDecoder) {
                     logger.error?.(`No table found with an id of ${auditRecord.tableId}`);
                 }
+                // Lazily compute receive-side exclusions once remoteNodeName is known.
+                // Prefer routeReplicates from the subscriber-side connection; fall back to
+                // authorization.replicates when this is the server-side handler.
+                if (receiveExcludedTables === undefined) {
+                    const firstNode = options.connection?.nodeSubscriptions?.[0];
+                    const receivesFromEntries = firstNode?.routeReplicates?.receivesFrom ??
+                        (authorization?.replicates && typeof authorization.replicates === 'object'
+                            ? authorization.replicates.receivesFrom
+                            : undefined);
+                    receiveExcludedTables =
+                        (0, knownNodes_ts_1.getExcludedTablesForRouteEntries)(receivesFromEntries, remoteNodeName, databaseName) ?? null;
+                }
+                if (tableDecoder && receiveExcludedTables?.has(tableDecoder.name)) {
+                    logger.trace?.(connectionId, 'dropping incoming replication for excluded table', databaseName + '.' + tableDecoder.name, 'from', remoteNodeName);
+                    decoder.position = start + eventLength;
+                    continue;
+                }
                 let residencyList;
                 if (auditRecord.residencyId) {
                     residencyList = receivedResidencyLists[auditRecord.residencyId];
@@ -1448,6 +1742,7 @@ function replicateOverWS(ws, options, authorization) {
                 }
                 const id = auditRecord.recordId;
                 event = undefined; // reset before each decode attempt
+                let receivedBlobs;
                 try {
                     (0, blob_ts_1.decodeBlobsWithWrites)(() => {
                         event = {
@@ -1463,14 +1758,30 @@ function replicateOverWS(ws, options, authorization) {
                             beginTxn,
                             expiresAt: auditRecord.expiresAt,
                         };
-                    }, auditStore?.rootStore, (blob) => receiveBlobs(blob, id));
+                    }, auditStore?.rootStore, (blob) => {
+                        const localBlob = receiveBlobs(blob, id);
+                        (receivedBlobs ??= []).push(localBlob);
+                        return localBlob;
+                    });
                 }
                 catch (error) {
                     logger.error?.('Error decoding replication message, record id: ' + id, ' typed structures for current decoder' + JSON.stringify(tableDecoder.decoder.typedStructs), ' structures for current decoder' + JSON.stringify(tableDecoder.decoder.structures), 'encoded message', auditRecord.encoded.subarray(0, 1000), auditRecord, error);
                 }
-                replicationSharedStatus[exports.RECEIVED_VERSION_POSITION] = Math.max(
-                // ensure monotonicity
-                auditRecord.version, replicationSharedStatus[exports.RECEIVED_VERSION_POSITION]);
+                if (!event && receivedBlobs) {
+                    // decode failed mid-message; the blobs that were already accepted will never be referenced. Give in-flight reads
+                    // a window to complete, then unlink the files. (mirrors the pattern at the relocate path above.)
+                    setTimeout(() => receivedBlobs.forEach(blob_ts_1.deleteBlob), 60000).unref();
+                }
+                // During a bulk copy, do NOT advance the received-version watermark per copied record:
+                // records arrive in primary-key order carrying their original (possibly newest) versions, so a
+                // single record at the leader's latest timestamp would otherwise let checkSyncStatus mark the
+                // clone Available with rows still uncopied. The watermark is advanced to copyStartTime by the
+                // single end_txn after the whole copy (the REMOTE_SEQUENCE_UPDATE branch above).
+                if (!inCopyMode) {
+                    replicationSharedStatus[exports.RECEIVED_VERSION_POSITION] = Math.max(
+                    // ensure monotonicity
+                    auditRecord.version, replicationSharedStatus[exports.RECEIVED_VERSION_POSITION]);
+                }
                 replicationSharedStatus[exports.RECEIVED_TIME_POSITION] = Date.now();
                 replicationSharedStatus[exports.RECEIVING_STATUS_POSITION] = exports.RECEIVING_STATUS_RECEIVING;
                 if (event) {
@@ -1491,6 +1802,14 @@ function replicateOverWS(ws, options, authorization) {
                         finally {
                             removePauseReason();
                         }
+                        lastYieldTime = performance.now();
+                    }
+                    else if (performance.now() - lastYieldTime >= RECEIVE_YIELD_INTERVAL) {
+                        // The high-water-mark pause only fires under heap pressure. When the consumer keeps
+                        // up, yield on a time budget anyway so a large message doesn't decode in one
+                        // synchronous turn and stall ping responses (see RECEIVE_YIELD_INTERVAL).
+                        await new Promise(setImmediate);
+                        lastYieldTime = performance.now();
                     }
                 }
                 decoder.position = start + eventLength;
@@ -1504,6 +1823,9 @@ function replicateOverWS(ws, options, authorization) {
                 addPauseReason();
                 logger.debug?.(`Commit backlog causing replication back-pressure, requesting that ${remoteNodeName} pause replication`);
             }
+            // Is this a bulk-copy frame? Only frames received before COPY_COMPLETE are part of the
+            // primary-key copy; later audit-replay frames must not be recorded as the resume cursor.
+            const isCopyFrame = inCopyMode && !copyCompleteReceived;
             tableSubscriptionToReplicator.send({
                 type: 'end_txn',
                 localTime: lastSequenceIdReceived,
@@ -1526,6 +1848,18 @@ function replicateOverWS(ws, options, authorization) {
                     if (outstandingBlobsToFinish.length > 0)
                         await Promise.all(outstandingBlobsToFinish);
                     logger.trace?.('All blobs finished');
+                    // Persist/clear the resume cursor only AFTER this batch's blobs are durable too. The cursor
+                    // means "fully copied through this key"; advancing it before blob writes finish would let a
+                    // crash skip re-requesting an unfinished blob, leaving the record pointing at missing data.
+                    if (isCopyFrame && event && copyFromNodeId !== undefined) {
+                        tableSubscriptionToReplicator?.dbisDB?.put([Symbol.for('copyCursor'), copyFromNodeId], {
+                            copyStartTime: copyModeStartTime,
+                            currentTable: event.table,
+                            afterKey: event.id,
+                        });
+                    }
+                    // once the last copied batch (incl. its blobs) is durable, it's safe to drop the cursor
+                    maybeFinishCopy();
                     if (!lastSequenceIdCommitted && sequenceIdReceived) {
                         logger.trace?.(connectionId, 'queuing confirmation of a commit at', sequenceIdReceived);
                         setTimeout(() => {
@@ -1546,6 +1880,7 @@ function replicateOverWS(ws, options, authorization) {
         }
     }
     ws.on('ping', resetPingTimer);
+    ws.on('pong', resetPingTimer);
     ws.on('pong', () => {
         if (options.connection) {
             // every pong we can use to update our connection information (and latency)
@@ -1570,7 +1905,7 @@ function replicateOverWS(ws, options, authorization) {
         // cleanup
         wsClosed = true;
         clearInterval(sendPingInterval);
-        clearTimeout(receivePingTimer);
+        receiveWatchdog?.stop();
         clearInterval(blobsTimer);
         if (auditSubscription)
             auditSubscription.emit('close');
@@ -1690,7 +2025,7 @@ function replicateOverWS(ws, options, authorization) {
         stream.recordId = id;
         if (remoteBlob.size === undefined && stream.expectedSize)
             remoteBlob.size = stream.expectedSize;
-        const localBlob = stream.blob ?? createBlob(stream, remoteBlob);
+        const localBlob = stream.blob ?? (0, blob_ts_1.createBlob)(stream, remoteBlob);
         stream.blob = localBlob; // record the blob so we can reuse it if another request uses the same blob
         // start the save immediately. TODO: If we could add support for blobs to directly pass on a stream to the consumer
         // we would skip this
@@ -1798,14 +2133,16 @@ function replicateOverWS(ws, options, authorization) {
         const nodeSubscriptions = options.connection?.nodeSubscriptions.map((node) => {
             const tableSubs = [];
             let { replicateByDefault } = node;
+            // Tables excluded by this node's receivesFrom config for this peer+database
+            const receiverExcludedTables = (0, knownNodes_ts_1.getExcludedTablesForRouteEntries)(node.routeReplicates?.receivesFrom, node.name, databaseName);
             if (node.subscriptions) {
                 // if the node has explicit subscriptions, we need to use that to determine subscriptions
                 for (const subscription of node.subscriptions) {
                     // if there is an explicit subscription listed
                     if (subscription.subscribe && (subscription.schema || subscription.database) === databaseName) {
                         const tableName = subscription.table;
-                        if (tables?.[tableName]?.replicate !== false)
-                            // if replication is enabled for this table
+                        if (tables?.[tableName]?.replicate !== false && !receiverExcludedTables?.has(tableName))
+                            // if replication is enabled for this table and not excluded
                             tableSubs.push(tableName);
                     }
                 }
@@ -1814,14 +2151,22 @@ function replicateOverWS(ws, options, authorization) {
             else {
                 // note that if replicateByDefault is enabled, we are listing the *excluded* tables
                 for (const tableName in tables) {
-                    if (replicateByDefault ? tables[tableName].replicate === false : tables[tableName].replicate) {
+                    if (replicateByDefault
+                        ? tables[tableName].replicate === false || receiverExcludedTables?.has(tableName)
+                        : tables[tableName].replicate && !receiverExcludedTables?.has(tableName)) {
                         tableSubs.push(tableName);
                     }
                 }
             }
             const nodeId = auditStore && (0, nodeIdMapping_ts_1.getIdOfRemoteNode)(node.name, auditStore);
-            auditStore.ensureLogExists?.(node.name);
+            auditStore?.ensureLogExists?.(node.name);
             const sequenceEntry = tableSubscriptionToReplicator?.dbisDB?.get([Symbol.for('seq'), nodeId]) ?? 1;
+            // A persisted copy cursor means a bulk copy from this node was interrupted mid-stream. We must
+            // resume that copy (not treat the persisted seqId as a normal start point — the un-copied table
+            // data predates copyStartTime and would never be delivered by an audit-log resume).
+            const copyCursor = nodeId === undefined
+                ? undefined
+                : tableSubscriptionToReplicator?.dbisDB?.get([Symbol.for('copyCursor'), nodeId]);
             // if we are connected directly to the node, we start from the last sequence number we received at the top level
             let startTime = Math.max(sequenceEntry?.seqId ?? 1, (typeof node.startTime === 'string' ? new Date(node.startTime).getTime() : node.startTime) ?? 1);
             logger.debug?.('Starting time recorded in db', node.name, nodeId, databaseName, sequenceEntry?.seqId, 'start time:', startTime, new Date(startTime));
@@ -1859,6 +2204,16 @@ function replicateOverWS(ws, options, authorization) {
                     startTime = Date.now() - 60000;
                 }
             }
+            let copyResume;
+            if (copyCursor) {
+                startTime = 0; // request a copy; the cursor tells the leader where to resume
+                copyResume = {
+                    copyStartTime: copyCursor.copyStartTime,
+                    currentTable: copyCursor.currentTable,
+                    afterKey: copyCursor.afterKey,
+                };
+                logger.warn?.(`Resuming interrupted copy of database ${databaseName} from ${(0, knownNodes_ts_1.getNodeURL)(node)} at table ${copyCursor.currentTable}`);
+            }
             logger.trace?.(connectionId, 'defining subscription request', node.name, databaseName, new Date(startTime));
             return {
                 name: node.name,
@@ -1867,6 +2222,7 @@ function replicateOverWS(ws, options, authorization) {
                 startTime,
                 isLeader: node.isLeader,
                 endTime: node.endTime,
+                copyResume, // present only when resuming an interrupted bulk copy
             };
         });
         let excluded;
@@ -1953,7 +2309,7 @@ function replicateOverWS(ws, options, authorization) {
             if (!thisNodeName)
                 throw new Error('Node name not defined');
             else
-                throw new Error('Should not connect to self', thisNodeName);
+                throw new Error('Should not connect to self: ' + thisNodeName);
         }
         sendNodeDBName(thisNodeName, databaseName);
         return true;