@harperfast/harper-pro 5.0.30 → 5.0.31

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
 	"name": "@harperfast/harper-pro",
-	"version": "5.0.30",
+	"version": "5.0.31",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "@harperfast/harper-pro",
-			"version": "5.0.30",
+			"version": "5.0.31",
 			"license": "Elastic-2.0",
 			"dependencies": {
 				"@aws-sdk/client-s3": "^3.1012.0",
@@ -2220,9 +2220,9 @@
 			"license": "Apache-2.0"
 		},
 		"node_modules/@harperfast/rocksdb-js": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-1.4.2.tgz",
-			"integrity": "sha512-wQ0buhmNVcw6jPn5VOoYDsGmO1IAmtithcSgaKrnBRicf+ATvQSCB1I7ljEBuqqzWG6HcJXeCgPhFpas3kRwOw==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-1.5.0.tgz",
+			"integrity": "sha512-NyH8Nr+jOclzzFzPmN9aC58lvl7X7Gf94E/laJqIeWP7/dsrdbDgNPHBueBdh1JnAq5Fxd+qvEDf/PCGaA/cgw==",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@harperfast/extended-iterable": "1.0.3",
@@ -2236,20 +2236,20 @@
 				"node": ">=18"
 			},
 			"optionalDependencies": {
-				"@harperfast/rocksdb-js-darwin-arm64": "1.4.2",
-				"@harperfast/rocksdb-js-darwin-x64": "1.4.2",
-				"@harperfast/rocksdb-js-linux-arm64-glibc": "1.4.2",
-				"@harperfast/rocksdb-js-linux-arm64-musl": "1.4.2",
-				"@harperfast/rocksdb-js-linux-x64-glibc": "1.4.2",
-				"@harperfast/rocksdb-js-linux-x64-musl": "1.4.2",
-				"@harperfast/rocksdb-js-win32-arm64": "1.4.2",
-				"@harperfast/rocksdb-js-win32-x64": "1.4.2"
+				"@harperfast/rocksdb-js-darwin-arm64": "1.5.0",
+				"@harperfast/rocksdb-js-darwin-x64": "1.5.0",
+				"@harperfast/rocksdb-js-linux-arm64-glibc": "1.5.0",
+				"@harperfast/rocksdb-js-linux-arm64-musl": "1.5.0",
+				"@harperfast/rocksdb-js-linux-x64-glibc": "1.5.0",
+				"@harperfast/rocksdb-js-linux-x64-musl": "1.5.0",
+				"@harperfast/rocksdb-js-win32-arm64": "1.5.0",
+				"@harperfast/rocksdb-js-win32-x64": "1.5.0"
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-darwin-arm64": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-1.4.2.tgz",
-			"integrity": "sha512-qQyv8BNCjvZQayhoTd7iqt1CLNEFHfqe8/SDZk8ztAR0ZAIyHIFkmNnaQ/Izn6p3HrNCRp6lNdwO4TKPWHj4SQ==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-1.5.0.tgz",
+			"integrity": "sha512-g8P4H/imnxn/AoGp+R5TZjHbwezdcMJCyIqo+2F1bLm3lGGaqTMrrmXK7vXxLj/COUMbooJ04QlGei7+nGShhA==",
 			"cpu": [
 				"arm64"
 			],
@@ -2263,9 +2263,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-darwin-x64": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-1.4.2.tgz",
-			"integrity": "sha512-cPCsB7IVeuDhNjFcb/nqUkwtBI9BlaFNecJ/OjkG8hUpYAhd5rdNHFMt1vpu4sAYFaorFrMZoNKGbq43v14+hQ==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-1.5.0.tgz",
+			"integrity": "sha512-62g7+n0G/QV2iBPFGV8ihCi6ZFW/4cxfe+SqQAOQ7p+YtA8/68o3X7b9j2+/fEixAC0ktLp/5mUie5hQVSmZrg==",
 			"cpu": [
 				"x64"
 			],
@@ -2279,9 +2279,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-arm64-glibc": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-1.4.2.tgz",
-			"integrity": "sha512-NquwD+7Gxu3BYAIQhvSThjIRlZRBieDZKXGEmrc0gfcxcORbgOjMCdxDzhjF6l1nR4ODlfJfbKqGyF8JtmnIjA==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-1.5.0.tgz",
+			"integrity": "sha512-hls3Hg8tHIAwBLL+3VHsHHpZO/D1BWoPkN8cjq8vnVSLEAAYS1qyJpePY1hdwEsL8TD2jBPs5YZwMGQ2EQCiWQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -2298,9 +2298,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-arm64-musl": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-1.4.2.tgz",
-			"integrity": "sha512-hp2IOXYBvloJEkZ/+bCI/AsQPtzza51Pkz+U+JkaAaayZ6+zsXdVp5GuGYPnCiH4O6QnOihKkgBW8msFhR0ogA==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-1.5.0.tgz",
+			"integrity": "sha512-aS+/ZzoOAGiEcIfhpS0xPMMBlBKkEjZAVE9Y92yhVFaMIMhE8UhRPwu0tFyWos6BdyLQ7/Ud5ncR42ZIUh8Y1g==",
 			"cpu": [
 				"arm64"
 			],
@@ -2317,9 +2317,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-x64-glibc": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-1.4.2.tgz",
-			"integrity": "sha512-zEkBHRoLanN9MTVY1mFRGDdAGyXBUrk962xvtf0sGj/wIK2M3c2KL39FftWHgtm/BHA3uGHssWk6B7O2O/Dlig==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-1.5.0.tgz",
+			"integrity": "sha512-70kPD6KKwtRJMsBBnKm8JYP+QncIYl7xb02/dVswdv1I5AqN1UNeqJBU4+K4q6QP6FeNpK25oruz8S1iy8QkLQ==",
 			"cpu": [
 				"x64"
 			],
@@ -2336,9 +2336,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-x64-musl": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-1.4.2.tgz",
-			"integrity": "sha512-K7y1CC+LJma2E+wEqFq67jk7+hZW+oPdKcqxyDxrIRKO3jyn0Dtr/fwQf3cFuU8cRnZBmZTAcnr5LeNJrUuYRw==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-1.5.0.tgz",
+			"integrity": "sha512-jnGa96c1w9L98ioGAzAezlni6Nls9LQ3A5Z4HYnPrY+Dz7Q9SeYANVv+edLR/Xt8fEQn9Yz6DkpYpns3ykUzGA==",
 			"cpu": [
 				"x64"
 			],
@@ -2355,9 +2355,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-win32-arm64": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-1.4.2.tgz",
-			"integrity": "sha512-xc4oav0zTLfnjBIan1wlgB3g0NeWrzUPxR0Vq5HkLc4BGyAlPql+v1FwtQ4X6AwcAOHnS/QeYSGDQOZn6Ac1Ow==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-1.5.0.tgz",
+			"integrity": "sha512-a3A68UqztdRS1Ind5UK9sytOJPoE9UJKlGt3623HVIaiQPfP3me9mx6YpnddqBshK0Lojzr5+cc6G1ewSFGEBw==",
 			"cpu": [
 				"arm64"
 			],
@@ -2371,9 +2371,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-win32-x64": {
-			"version": "1.4.2",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-1.4.2.tgz",
-			"integrity": "sha512-60vqTYJdw0ysvHeFHoxZ/sHNesdfvri+jWIyU5TTua2s20LmLZ4hKLglnsa0LP7/IA5JutZsxJiyUUMCoG3PDw==",
+			"version": "1.5.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-1.5.0.tgz",
+			"integrity": "sha512-0PKUTon/o5TUT3XBMPGPJl6Ry4aP/1D134HDP9XGSYWqtcQwzDb0LsRQdJhF9ixJQzVPrXw9mDGMQFzasVmvcw==",
 			"cpu": [
 				"x64"
 			],

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/harper-pro",
-	"version": "5.0.30",
+	"version": "5.0.31",
 	"description": "Harper is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
 	"keywords": [
 		"database",

package/replication/knownNodes.ts

@@ -74,6 +74,20 @@ export function getReplicationSharedStatus(
 		)
 	);
 }
+/**
+ * Decide whether a change-stream event that carried *no usable decoded value* represents a
+ * genuine node deletion. (Callers only reach this for a nullish value; an event with a
+ * decoded value is always an upsert.) The ambiguity is only for `put`/`patch` events, where
+ * a decode failure can produce a null value while the record still physically exists.
+ * A `delete` event from the change stream is already the reliable signal that the record is
+ * truly gone — the physical-existence check was overly conservative and caused genuine
+ * `remove_node` deletes to be suppressed because LMDB/RocksDB reads open against an older
+ * snapshot that predates the delete commit (harper#1163 regression).
+ */
+export function isGenuineNodeDeletion(eventType: string): boolean {
+	return eventType === 'delete'; // put/patch with null value = decode failure; delete events are genuine
+}
+
 export function subscribeToNodeUpdates(listener: (node: any, id: string) => void) {
 	getHDBNodeTable()
 		.subscribe({})
@@ -103,7 +117,23 @@ export function subscribeToNodeUpdates(listener: (node: any, id: string) => void
 				}
 				server.shards = shards;
 				if (event.type === 'put' || event.type === 'delete') {
-					listener(event.value, event.id);
+					if (event.value != null) {
+						// normal upsert with a decoded value
+						listener(event.value, event.id);
+					} else if (isGenuineNodeDeletion(event.type)) {
+						// genuine deletion — forward the nullish value so the listener tears the node down
+						listener(event.value, event.id);
+					} else {
+						// A put/patch with no decodable value is a transient decode failure (e.g. stale
+						// msgpackr shared-structures, harper#1163) — NOT a node removal. Forwarding it here
+						// would make onNodeUpdate treat the nullish value as a deletion and unsubscribe the
+						// peer from every database. Drop it instead; the next decodable event self-heals.
+						logger.warn?.(
+							'hdb_nodes change event for',
+							event.id,
+							'had no decodable value; treating as a transient decode failure (see harper#1163), not a node deletion'
+						);
+					}
 				}
 			}
 		});

package/replication/replicationConnection.ts

@@ -91,6 +91,12 @@ const MAX_PAYLOAD = env.get('replication_maxPayload') ?? 100_000_000;
 // heap past its limit. If the local replicator queue grows beyond this threshold we pause
 // the WS connection and wait for it to drain before continuing the decode loop.
 const RECEIVE_EVENT_HIGH_WATER_MARK = env.get('replication_receiveEventHighWaterMark') ?? 100;
+// Even when the consumer keeps up (queue below the high-water mark), a single large inbound message
+// would otherwise decode thousands of records in one synchronous turn — pegging the worker, blocking
+// replication ping responses, and tripping core's "JavaScript execution has taken too long" monitor
+// (MAX_EVENT_DELAY_TIME = 3 s). Yield the event loop at least this often (ms) while decoding so the
+// worker stays responsive during a bulk copy/clone.
+const RECEIVE_YIELD_INTERVAL = env.get('replication_receiveYieldInterval') ?? 100;
 
 export const tableUpdateListeners = new Map();
 // This a map of the database name to the subscription object, for the subscriptions from our tables to the replication module
@@ -105,6 +111,28 @@ const SKIPPED_MESSAGE_SEQUENCE_UPDATE_DELAY = 300;
 // (but still allow for batching so we aren't sending out a message for every update under load)
 const COMMITTED_UPDATE_DELAY = 2;
 const PING_INTERVAL = 30000;
+// Time (ms) without any socket activity before a connection is treated as dead.
+// On v5.0 this is hardcoded; main exposes it via `replication.pingTimeout` (v5.1+).
+const PING_TIMEOUT = PING_INTERVAL * 2; // 60 s; configurable via replication.pingTimeout in v5.1+
+
+/**
+ * Decide whether an idle replication connection should be terminated as dead.
+ *
+ * Liveness is measured from the last observed socket byte movement (in either direction), not from a
+ * single ping interval. A bulk transfer — notably the initial clone copy of a large table — makes
+ * slow but real progress: the sender's socket buffer drains in bursts as the peer consumes, so bytes
+ * keep moving within the timeout window even while it is otherwise stalled. A genuinely dead or
+ * unreachable peer moves no bytes at all, so it still trips the timeout. We deliberately do NOT exempt
+ * the sender's `isPausedForBackPressure` drain-wait here: if the peer dies after we have filled our
+ * socket buffer, the drain event never fires, and exempting it would hang the connection forever.
+ *
+ * The one exemption is `pauseReasons > 0`: the receiver has intentionally stopped reading to drain its
+ * own queue. That stall is local and self-clearing (it does not depend on the peer), so it is never a
+ * death signal; the caller keeps liveness fresh while paused and resumes normal detection afterward.
+ */
+export function shouldTerminateIdlePing(idleMs: number, pingTimeout: number, pauseReasons: number): boolean {
+	return pauseReasons === 0 && idleMs >= pingTimeout;
+}
 let secureContexts: Map<string, tls.SecureContext>;
 /**
  * Handles reconnection, and requesting catch-up
@@ -413,22 +441,51 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	// track bytes read and written so we can verify if a connection is really dead on pings
 	let bytesRead = 0;
 	let bytesWritten = 0;
+	// wall-clock time of the last observed socket activity (bytes moved in either direction); the
+	// keep-alive timeout is measured from this so a legitimately slow-but-progressing transfer stays
+	// alive while a truly idle/dead peer is still terminated.
+	let lastByteActivity = performance.now();
+	// Multiple independent conditions can ask to pause receive on this WS (commit backlog, consumer
+	// queue full, blob write backpressure). We refcount the reasons so that resuming one does not race
+	// ahead of another that still wants the WS paused. Declared before the ping setup below because the
+	// immediate sendPing() reads it.
+	let pauseReasons = 0;
 	const blobTimeout = env.get(CONFIG_PARAMS.REPLICATION_BLOBTIMEOUT) ?? 120000;
 	const blobsInFlight = new Map();
 	const outstandingBlobsToFinish: Promise<void>[] = [];
 	let outstandingBlobsBeingSent = 0;
 	let blobSentCallback: (v?: any) => void;
+	// Refresh the keep-alive liveness clock from observed socket byte movement. If the underlying
+	// _socket isn't observable (test mocks, pre-connect, or a change in the ws library internals),
+	// bytesRead/bytesWritten read as undefined; we can't measure activity, so treat the connection as
+	// live rather than let the keep-alive falsely terminate a healthy peer.
+	function noteByteActivity(): void {
+		const read = ws._socket?.bytesRead;
+		const written = ws._socket?.bytesWritten;
+		if (read === undefined || written === undefined || read !== bytesRead || written !== bytesWritten) {
+			lastByteActivity = performance.now();
+		}
+	}
 	if (options.url) {
 		const sendPing = () => {
-			// if we have not received a message in the last ping interval, we should terminate the connection (but check to make sure we aren't just waiting for other data to flow)
-			if (lastPingTime && bytesRead === ws._socket?.bytesRead && bytesWritten === ws._socket?.bytesWritten)
-				ws.terminate(); // timeout
-			else {
-				lastPingTime = performance.now();
-				ws.ping();
-				bytesRead = ws._socket?.bytesRead;
-				bytesWritten = ws._socket?.bytesWritten;
+			// Note any socket activity since the last interval (incoming pong/data or our send buffer
+			// draining as the peer consumes) — either proves the peer is still alive.
+			noteByteActivity();
+			if (shouldTerminateIdlePing(performance.now() - lastByteActivity, PING_TIMEOUT, pauseReasons)) {
+				ws.terminate(); // no socket activity within the timeout — peer is gone
+				return;
 			}
+			// While paused for receiver backpressure, keep our own liveness fresh: the stall is local and
+			// self-clearing (it doesn't depend on the peer), so we must not time the peer out for it.
+			if (pauseReasons > 0) lastByteActivity = performance.now();
+			// Always send the keep-alive ping. ws.pause() only stops reads, not writes, and the accepted
+			// peer relies on our pings to keep its own receive timer alive even when it has no data to send
+			// us. Record byte counts AFTER the ping so the ping's own bytes aren't later mistaken for peer
+			// activity.
+			lastPingTime = performance.now();
+			ws.ping();
+			bytesRead = ws._socket?.bytesRead;
+			bytesWritten = ws._socket?.bytesWritten;
 		};
 		sendPingInterval = setInterval(sendPing, PING_INTERVAL).unref();
 		sendPing(); // send the first ping immediately so we can measure latency
@@ -489,10 +546,6 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 	const MAX_OUTSTANDING_BLOBS_BEING_SENT = env.get(CONFIG_PARAMS.REPLICATION_BLOBCONCURRENCY) ?? 5;
 	let outstandingCommits = 0;
 	let lastStructureLength = 0;
-	// Multiple independent conditions can ask to pause receive on this WS (commit backlog,
-	// consumer queue full, blob write backpressure). We refcount the reasons so that resuming
-	// one does not race ahead of another that still wants the WS paused.
-	let pauseReasons = 0;
 	let commitBacklogPaused = false;
 	function addPauseReason(): void {
 		if (pauseReasons === 0) ws.pause();
@@ -1599,6 +1652,7 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 			let beginTxn = true;
 			let event; // could also get txnTime from decoder.getFloat64(0);
 			let sequenceIdReceived;
+			let lastYieldTime = performance.now();
 			do {
 				getSharedStatus();
 				const eventLength = decoder.readInt();
@@ -1706,6 +1760,13 @@ export function replicateOverWS(ws: WebSocket, options: any, authorization: Prom
 						} finally {
 							removePauseReason();
 						}
+						lastYieldTime = performance.now();
+					} else if (performance.now() - lastYieldTime >= RECEIVE_YIELD_INTERVAL) {
+						// The high-water-mark pause only fires under heap pressure. When the consumer keeps
+						// up, yield on a time budget anyway so a large message doesn't decode in one
+						// synchronous turn and stall ping responses (see RECEIVE_YIELD_INTERVAL).
+						await new Promise(setImmediate);
+						lastYieldTime = performance.now();
 					}
 				}
 				decoder.position = start + eventLength;

package/replication/replicator.ts

@@ -447,6 +447,16 @@ export function setReplicator(dbName: string, table: any, options: any) {
 }
 const connections = new Map<string, Map<string, NodeReplicationConnection>>();
 
+// A connection that was intentionally torn down — the empty-subscription delayed close marks it
+// intentionallyUnsubscribed/isFinished — must not be handed back out of the cache. connect()
+// early-returns forever on intentionallyUnsubscribed, so a peer whose subscription set transiently
+// emptied (e.g. during a peer restart) and was then restored would otherwise stay permanently dead
+// on a connected:false connection that never retries. Callers drop a non-reusable cached connection
+// and create a fresh one through the normal connect path. See harper-pro#233 / #289.
+export function isReusableConnection(connection?: NodeReplicationConnection): boolean {
+	return !!connection && !connection.isFinished && !connection.intentionallyUnsubscribed;
+}
+
 /**
  * Get or create a connection to the specified node
  * @param url
@@ -468,7 +478,7 @@ function getSubscriptionConnection(
 		connections.set(connectionKey, dbConnections);
 	}
 	let connection = dbConnections.get(dbName);
-	if (connection) return connection;
+	if (isReusableConnection(connection)) return connection;
 	if (subscription) {
 		dbConnections.set(
 			dbName,
@@ -490,7 +500,7 @@ function getRetrievalConnectionByName(nodeName, subscription, dbName): NodeRepli
 		nodeNameToRetrievalConnections.set(nodeName, dbConnections);
 	}
 	let connection = dbConnections.get(dbName);
-	if (connection) return connection;
+	if (isReusableConnection(connection)) return connection;
 	const node = getHDBNodeTable().primaryStore.get(nodeName);
 	if (node?.url) {
 		connection = new NodeReplicationConnection(getNodeURL(node), subscription, dbName, nodeName, node.authorization);

package/replication/subscriptionManager.ts

@@ -31,6 +31,9 @@ type ConnectedWorkerStatus = {
 	worker: Worker | null;
 	connected?: boolean;
 	latency?: number;
+	// Timestamp (ms) of the most recent transition to connected:false, used by the reconcile to
+	// distinguish a connection that is briefly retrying from one that is wedged. Cleared on connect.
+	disconnectedAt?: number;
 };
 type ReplicationConnectionStatus = {
 	nodes: ({
@@ -52,8 +55,79 @@ const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to othe
 // caused the OOM in the first place. We stagger the re-subscriptions in time so the new
 // worker(s) absorb them gradually.
 const WORKER_EXIT_REASSIGN_STAGGER_MS = 100;
+// Cadence of the per-process safety-net reconcile that rebinds subscriptions whose
+// worker no longer exists. Pure read-side filter against `workers` and
+// `connectionReplicationMap` on each tick when nothing is wrong, so a short interval
+// is cheap. Sized for the deploy-time rapid-restart-storm pattern (stacked
+// `restart_http_workers` at ~1.5s spacing under live write traffic), where the per-
+// worker exit chain races against shutdown and silently drops half the subscription
+// assignments — this is the user-visible recovery latency for the resulting drift.
+const RECONCILE_INTERVAL_MS = 5_000;
+// A connection that is connected:false but still actively retrying reconnects within seconds (the
+// retry backoff starts at 500ms). Only re-drive a connection that has stayed disconnected well
+// beyond that, so the reconcile targets genuinely wedged connections (e.g. an intentionally-closed
+// connection with no pending retry) rather than churning connections that are mid-reconnect.
+const WEDGE_RECONCILE_THRESHOLD_MS = 30_000;
 let nextWorkerExitReassignAt = 0;
 const connectionReplicationMap = new Map<string, DBReplicationStatusMap>();
+// Returns the set of node URLs whose replication entries either point at a worker no longer
+// in the supplied http pool, OR have no worker assigned at all while live workers exist.
+// The second case covers "all workers were down at registration time" — onDatabase stores
+// `worker: undefined` when httpWorkers is empty, and without this the entry would never
+// get reassigned once workers came back. Pure helper so the reconcile pass below — and its
+// unit tests — can verify the broken-chain detection without spinning up real workers.
+export function findStaleNodeUrls(connectionMap: Map<string, DBReplicationStatusMap>, httpWorkers: any[]): Set<string> {
+	const staleNodeUrls = new Set<string>();
+	// No live workers to reassign to — flagging here would cause endless no-op reassignments.
+	if (httpWorkers.length === 0) return staleNodeUrls;
+	for (const [url, dbReplicationWorkers] of connectionMap) {
+		for (const entry of dbReplicationWorkers.values()) {
+			if (!entry.worker || !httpWorkers.includes(entry.worker)) {
+				staleNodeUrls.add(url);
+				break;
+			}
+		}
+	}
+	return staleNodeUrls;
+}
+// Returns the set of node URLs that have a desired replication subscription but have been
+// connected:false on a *live* worker for longer than `thresholdMs`. This is the recovery path for a
+// connection that wedged without a pending retry — most notably the empty-subscription delayed close
+// (intentionallyUnsubscribed) firing during a peer restart and then never re-establishing even though
+// the peer is reachable and still subscribed. findStaleNodeUrls does not catch this because the
+// worker is alive. Re-driving these through onNodeUpdate creates a fresh connection (the prior one is
+// no longer reusable — see replicator.isReusableConnection). A threshold well above the normal
+// reconnect backoff keeps this from firing on connections that are merely mid-retry.
+// `isDesired` must be the same predicate onDatabase uses to decide shouldSubscribe
+// (shouldReplicateFromNode), so a connection intentionally unsubscribed because this node should NOT
+// subscribe (replication off, or a sendsTo/subscription targeting another database) is not flagged
+// and re-driven forever. See harper-pro#233 / #289.
+export function findWedgedNodeUrls(
+	connectionMap: Map<string, DBReplicationStatusMap>,
+	httpWorkers: any[],
+	now: number,
+	thresholdMs: number,
+	isDesired: (node: any, database: string) => boolean
+): Set<string> {
+	const wedgedNodeUrls = new Set<string>();
+	if (httpWorkers.length === 0) return wedgedNodeUrls;
+	for (const [url, dbReplicationWorkers] of connectionMap) {
+		for (const [database, entry] of dbReplicationWorkers) {
+			if (
+				entry.connected === false &&
+				entry.worker &&
+				httpWorkers.includes(entry.worker) &&
+				entry.disconnectedAt != null &&
+				now - entry.disconnectedAt >= thresholdMs &&
+				isDesired(entry.nodes?.[0], database)
+			) {
+				wedgedNodeUrls.add(url);
+				break;
+			}
+		}
+	}
+	return wedgedNodeUrls;
+}
 export let disconnectedFromNode; // this is set by thread to handle when a node is disconnected (or notify main thread so it can handle)
 export let connectedToNode; // this is set by thread to handle when a node is connected (or notify main thread so it can handle)
 const nodeMap = new Map(); // this is a map of all nodes that are available to connect to
@@ -126,7 +200,7 @@ export async function startOnMainThread(options) {
 	 * This is called when a new node is added to the hdbNodes table
 	 * @param node
 	 */
-	function onNodeUpdate(node, hostname = node?.name) {
+	function onNodeUpdate(node, hostname = node?.name, forceResubscribe = false) {
 		const isSelf =
 			(getThisNodeName() && hostname === getThisNodeName()) || (getThisNodeUrl() && node?.url === getThisNodeUrl());
 		if (isSelf) {
@@ -194,9 +268,9 @@ export async function startOnMainThread(options) {
 		}
 		dbReplicationWorkers.iterator = forEachReplicatedDatabase(options, (database, databaseName, replicateByDefault) => {
 			if (replicateByDefault) {
-				onDatabase(databaseName, true);
+				onDatabase(databaseName, true, forceResubscribe);
 			} else {
-				onDatabase(databaseName, false);
+				onDatabase(databaseName, false, forceResubscribe);
 			}
 		});
 		// check to see if there are any explicit subscriptions to databases that don't exist yet
@@ -207,12 +281,12 @@ export async function startOnMainThread(options) {
 				const databaseName = sub.database || sub.schema;
 				if (!databases[databaseName]) {
 					logger.warn(`Database ${databaseName} not found for node ${node.name}, making a subscription anyway`);
-					onDatabase(databaseName, false);
+					onDatabase(databaseName, false, forceResubscribe);
 				}
 			}
 		}
 
-		function onDatabase(databaseName, tablesReplicateByDefault) {
+		function onDatabase(databaseName, tablesReplicateByDefault, forceResubscribe = false) {
 			logger.trace('Setting up replication for database', databaseName, 'on node', node.name);
 			const existingEntry = dbReplicationWorkers.get(databaseName);
 			let worker;
@@ -236,8 +310,13 @@ export async function startOnMainThread(options) {
 			if (existingEntry) {
 				worker = existingEntry.worker;
 				existingEntry.nodes = nodes;
-				if (shouldSubscribe) {
-					// already subscribed, don't need to do anything
+				// Normally an existing subscribed entry is left alone. Only the wedge reconcile passes
+				// forceResubscribe for a connection that has been connected:false past the threshold: that
+				// falls through to re-post subscribe-to-node on the same worker (the worker then reuses a
+				// still-retrying connection or builds a fresh one — replicator.isReusableConnection). We
+				// deliberately do NOT re-subscribe every connected:false entry on an ordinary onNodeUpdate —
+				// doing so disrupts in-flight replication (e.g. an active legacy-node base copy).
+				if (shouldSubscribe && !(forceResubscribe && existingEntry.connected === false)) {
 					return;
 				}
 			} else if (shouldSubscribe) {
@@ -345,6 +424,9 @@ export async function startOnMainThread(options) {
 				logger.warn('Disconnected node not found in replication map', connection.database, dbReplicationWorkers);
 				return;
 			}
+			// Record the first transition to disconnected so the reconcile can tell a wedged connection
+			// from one that is briefly mid-retry; don't reset it on repeated disconnect notifications.
+			if (existingWorkerEntry.connected !== false) existingWorkerEntry.disconnectedAt = Date.now();
 			existingWorkerEntry.connected = false;
 			if (connection.finished) {
 				return;
@@ -413,6 +495,7 @@ export async function startOnMainThread(options) {
 			return;
 		}
 		mainWorkerEntry.connected = true;
+		mainWorkerEntry.disconnectedAt = undefined;
 		mainWorkerEntry.latency = connection.latency;
 		const restoredNode = mainWorkerEntry.nodes[0];
 		if (!restoredNode) {
@@ -478,6 +561,59 @@ export async function startOnMainThread(options) {
 			});
 		} else subscribeToNode({ url: getNodeURL(connectingNode), name: connectingNode.name, database, nodes: [node] });
 	}
+	// Periodic safety net for stale subscription entries. The existing per-database
+	// worker.on('exit') chain reassigns to a healthy worker after a worker dies, but a
+	// single broken link in that chain (identity check failing, setTimeout retry being
+	// lost under load, shouldSubscribe early-return pinning to a dead worker before
+	// the defensive check was added) used to leave the entry permanently pointing at
+	// an exited worker, silently breaking outbound replication for the lifetime of the
+	// process. This reconciles independently of the chain so the broken-state node
+	// can never get stuck.
+	function reconcileWorkers() {
+		const httpWorkers = workers.filter((worker) => worker.name === 'http');
+		const staleNodeUrls = findStaleNodeUrls(connectionReplicationMap, httpWorkers);
+		const wedgedNodeUrls = findWedgedNodeUrls(
+			connectionReplicationMap,
+			httpWorkers,
+			Date.now(),
+			WEDGE_RECONCILE_THRESHOLD_MS,
+			shouldReplicateFromNode
+		);
+		if (staleNodeUrls.size === 0 && wedgedNodeUrls.size === 0) return;
+		if (staleNodeUrls.size > 0)
+			logger.warn(
+				'Reconciling replication subscriptions for nodes pointing at exited workers:',
+				Array.from(staleNodeUrls)
+			);
+		if (wedgedNodeUrls.size > 0)
+			logger.warn(
+				'Reconciling replication subscriptions for nodes wedged disconnected on a live worker:',
+				Array.from(wedgedNodeUrls)
+			);
+		for (const node of nodeMap.values()) {
+			const url = getNodeURL(node);
+			const isWedged = wedgedNodeUrls.has(url);
+			if (!staleNodeUrls.has(url) && !isWedged) continue;
+			if (isWedged) {
+				// Restart the disconnect clock before re-driving so a peer that is still unreachable after
+				// the re-subscribe (it stays connected:false, which does not re-stamp disconnectedAt) is
+				// retried at most once per threshold window rather than on every reconcile tick.
+				// onNodeUpdate -> onDatabase then re-posts subscribe-to-node for the connected:false entry
+				// (it no longer early-returns), reusing the existing worker — so no entry/listener churn.
+				const entries = connectionReplicationMap.get(url);
+				if (entries)
+					for (const entry of entries.values()) if (entry.connected === false) entry.disconnectedAt = Date.now();
+			}
+			try {
+				// forceResubscribe only for wedged entries, so a normal stale-worker reconcile keeps its
+				// original behavior and ordinary onNodeUpdate calls never re-subscribe live subscriptions.
+				onNodeUpdate(node, undefined, isWedged);
+			} catch (error) {
+				logger.error('Error reconciling node', node?.name, error);
+			}
+		}
+	}
+	setInterval(reconcileWorkers, RECONCILE_INTERVAL_MS).unref();
 	onMessageByType('disconnected-from-node', disconnectedFromNode);
 	onMessageByType('connected-to-node', connectedToNode);
 	onMessageByType('request-cluster-status', requestClusterStatus);
@@ -526,6 +662,18 @@ export function requestClusterStatus(message, port) {
 	return { connections };
 }
 
+// threadServer.js starts servers at import time on non-main workers, and job workers import this
+// module (replication is a HARPER_BUILTIN_COMPONENT), so importing it at module load would spuriously
+// start servers / keep job workers alive. Lazily import it only when a subscribe/unsubscribe actually
+// arrives — which only happens on HTTP/replication workers, where threadServer is already loaded, so
+// the dynamic import resolves to the cached module with no side effect. Cached after first use.
+let componentsLoadedPromise: Promise<unknown> | undefined;
+function whenWorkerComponentsLoaded(): Promise<unknown> {
+	return (componentsLoadedPromise ??= import('../core/server/threads/threadServer.js').then(
+		(threadServer) => threadServer.whenComponentsLoaded
+	));
+}
+
 if (parentPort) {
 	disconnectedFromNode = (connection) => {
 		parentPort.postMessage({ type: 'disconnected-from-node', ...connection });
@@ -534,10 +682,19 @@ if (parentPort) {
 		parentPort.postMessage({ type: 'connected-to-node', ...connection });
 	};
 	onMessageByType('subscribe-to-node', (message) => {
-		subscribeToNode(message);
+		// Defer until this worker has finished loading components (databases/tables + persisted hdb_nodes
+		// rows). subscribeToNode re-checks shouldReplicateFromNode, which reads that thread-local state; if
+		// it runs before the state is loaded it filters the request down to empty and arms a permanent
+		// "no subscriptions" close, wedging the (peer, db) until restart (harper-pro#289 / #233). Once
+		// components are loaded the predicate is authoritative. In steady state the promise is already
+		// resolved, so this is effectively synchronous.
+		whenWorkerComponentsLoaded().then(() => subscribeToNode(message));
 	});
 	onMessageByType('unsubscribe-from-node', (message) => {
-		unsubscribeFromNode(message);
+		// Defer through the same gate as subscribe-to-node so the two stay ordered: a pre-load
+		// subscribe followed by an unsubscribe must apply in that order (else the deferred subscribe
+		// would run after the unsubscribe and re-open a connection the main thread already removed).
+		whenWorkerComponentsLoaded().then(() => unsubscribeFromNode(message));
 	});
 }