@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper-pro",
-	"version": "5.0.25",
-	"description": "Harper is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
+	"version": "5.1.0-beta.1",
+	"description": "Harper is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use. Harper Pro includes replication, certificate management, profiling.",
 	"keywords": [
 		"database",
 		"nosql",
@@ -11,8 +11,7 @@
 		"mqtt",
 		"real-time",
 		"enterprise",
-		"Fastify",
-		"NATS",
+		"vector database",
 		"HarperDB",
 		"Harper",
 		"clustering",
@@ -107,7 +106,9 @@
 		"lint": "oxlint --deny-warnings .",
 		"lint:fix": "npm run lint -- --fix",
 		"lint:required": "oxlint --quiet .",
-		"test:integration": "node integrationTests/run.mjs",
+		"test:unit": "mocha --require unitTests/unitTestSetup.cjs 'unitTests/**/*.test.mjs'",
+		"test:integration": "HARPER_INTEGRATION_TEST_INSTALL_SCRIPT=dist/bin/harper.js harper-integration-test-run",
+		"test:integration:all": "npm run test:integration -- integrationTests/**/*.test.*s",
 		"cluster:ip:local": "pushd utility/dev && docker compose -f docker-compose.ip.yml --project-directory ../.. build && docker compose -f docker-compose.ip.yml --project-directory ../.. up; popd",
 		"cluster:ip:latest": "pushd utility/dev && docker compose -f docker-compose.ip.yml --project-directory ../.. pull && docker compose -f docker-compose.ip.yml --project-directory ../.. up; popd",
 		"cluster:delete": "pushd utility/dev && docker compose -f docker-compose.ip.yml --project-directory ../.. down --remove-orphans; popd",
@@ -120,7 +121,7 @@
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.1012.0",
 		"@aws-sdk/lib-storage": "3.1045.0",
-		"@datadog/pprof": "^5.11.1",
+		"@datadog/pprof": "^5.14.4",
 		"@endo/static-module-record": "^1.1.2",
 		"@fastify/autoload": "^6.3.1",
 		"@fastify/compress": "^8.3.1",
@@ -137,10 +138,11 @@
 		"@turf/distance": "6.5.0",
 		"@turf/helpers": "6.5.0",
 		"@turf/length": "6.5.0",
-		"alasql": "4.6.6",
+		"alasql": "4.17.2",
 		"amaro": "^1.1.8",
 		"argon2": "0.44.0",
-		"asn1js": "3.0.7",
+		"asn1js": "3.0.10",
+		"busboy": "^1.6.0",
 		"cbor-x": "1.6.4",
 		"chalk": "4.1.2",
 		"chokidar": "^4.0.3",
@@ -169,11 +171,10 @@
 		"minimist": "1.2.8",
 		"moment": "2.30.1",
 		"mqtt-packet": "~9.0.1",
-		"msgpackr": "1.11.12",
+		"msgpackr": "^2.0.1",
 		"needle": "3.5.0",
 		"node-forge": "^1.3.1",
 		"node-stream-zip": "1.15.0",
-		"node-unix-socket": "0.2.7",
 		"normalize-path": "^3.0.0",
 		"ora": "8.2.0",
 		"ordered-binary": "1.6.1",
@@ -191,6 +192,7 @@
 		"ses": "^1.15.0",
 		"stream-chain": "2.2.5",
 		"stream-json": "1.9.1",
+		"structon": "^1.0.4",
 		"systeminformation": "^5.31.4",
 		"tar-fs": "^3.1.2",
 		"ulidx": "0.5.0",
@@ -201,6 +203,9 @@
 	},
 	"devDependencies": {
 		"@harperdb/code-guidelines": "^0.0.6",
+		"@harperfast/integration-testing": "^0.3.1",
+		"@modelcontextprotocol/sdk": "^1.29.0",
+		"@types/busboy": "^1.5.4",
 		"@types/fs-extra": "^11.0.4",
 		"@types/gunzip-maybe": "^1.4.3",
 		"@types/jsonwebtoken": "^9.0.10",
@@ -219,7 +224,7 @@
 		"intercept-stdout": "0.1.2",
 		"mkcert": "^3.2.0",
 		"mocha": "^11.7.5",
-		"mqtt": "~4.3.8",
+		"mqtt": "^5.15.1",
 		"oxlint": "^1.31.0",
 		"prettier": "~3.8.0",
 		"rewire": "^9.0.1",

package/replication/DESIGN.md

@@ -0,0 +1,139 @@
+# replication/ — Navigation Guide
+
+Real-time, peer-to-peer replication of table data across cluster nodes via persistent WebSocket connections. Implements eventual consistency: when a local transaction commits, the audit records are forwarded asynchronously to peers.
+
+**Read this when:** you're touching cluster sync, debugging missed writes, JWT/cluster auth, latency-based node selection, or blob transfer between nodes.
+
+**Integration boundary with core:** replication hooks into core's table resource layer — a `Replicator` class is installed as a `source` of the table (`table.sourcedFrom(class Replicator extends Resource {...})`). When a local cache miss occurs, the Replicator picks the lowest-latency peer and fetches. Core's audit store (`core/resources/auditStore.ts`) and node-id mapping (`core/resources/nodeIdMapping.ts`) are the two data structures replication reads.
+
+> **Navigation convention.** Code is referenced by **symbol name** (class, function, exported const). Use your editor's go-to-symbol or `grep -n '<name>' replication/<file>` to jump. Line numbers drift; symbols don't.
+
+---
+
+## Files (6 total, ~4200 lines)
+
+| File                       | Purpose                                                                                                                                                                                                               |
+| -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `replicationConnection.ts` | The protocol engine. Defines `NodeReplicationConnection`, encodes/decodes the binary frame format, drives audit-record forwarding, manages blobs, and writes shared latency/back-pressure counters. **The big file.** |
+| `replicator.ts`            | Setup module: `start()`, per-database/per-table `Replicator` resource class, retrieval-connection pool, operation forwarding, mTLS config.                                                                            |
+| `subscriptionManager.ts`   | Main-thread orchestration. Delegates subscription work to worker threads; routes around disconnects.                                                                                                                  |
+| `setNode.ts`               | Cluster member operations — add/remove nodes, CSR signing, TLS certificate negotiation.                                                                                                                               |
+| `knownNodes.ts`            | Node registry (`hdb_nodes` system table) + shared-memory `Float64Array` status buffers (latency, confirmation, back-pressure).                                                                                        |
+| `clusterStatus.ts`         | Read-only status reporting for `cluster_status` operation.                                                                                                                                                            |
+
+---
+
+## Key abstractions
+
+### `NodeReplicationConnection` (`replicationConnection.ts`)
+
+A persistent connection to one remote node. Owns the WebSocket lifecycle, reconnection (initial delay `INITIAL_RETRY_TIME`), latency tracking, and per-subscription state. **Inspect this when debugging connection drops or auth failures** (see issue #135 lineage on JWT/cluster auth).
+
+### `replicateOverWS(ws, options, authorization)` (`replicationConnection.ts`)
+
+The protocol decoder. Reads incoming binary commands — each is a top-level named const in the same file:
+
+| Command constant                           | Value     | Meaning                                   |
+| ------------------------------------------ | --------- | ----------------------------------------- |
+| `SUBSCRIPTION_REQUEST`                     | 129       | Client wants to subscribe to a table      |
+| `RESIDENCY_LIST`                           | 130       | Negotiate which records each node holds   |
+| `TABLE_FIXED_STRUCTURE`                    | 132       | Schema sync                               |
+| `GET_RECORD` / `GET_RECORD_RESPONSE`       | 133 / 134 | Cache-miss fetch                          |
+| `OPERATION_REQUEST` / `OPERATION_RESPONSE` | 136 / 137 | Forwarded operations                      |
+| `NODE_NAME` / `NODE_NAME_TO_ID_MAP`        | 140 / 141 | Identity exchange                         |
+| `DISCONNECT`                               | 142       | Graceful close (not used on auth failure) |
+| `SEQUENCE_ID_UPDATE`                       | 143       | Audit sequence cursor                     |
+| `COMMITTED_UPDATE`                         | 144       | Confirm-on-commit                         |
+| `DB_SCHEMA`                                | 145       | Database schema replication               |
+| `BLOB_CHUNK`                               | 146       | Blob bytes                                |
+| `SUBSCRIPTION_UPDATE`                      | 147       | Audit record forwarded to subscribers     |
+
+The `authorization` parameter is a **promise that may resolve asynchronously**; on rejection the socket closes without a DISCONNECT frame (relevant to JWT failure flows).
+
+### `Replicator extends Resource` (`replicator.ts`)
+
+A `Resource` class installed as a `source` of a table. Declared inside `setReplicator()` and passed to `table.sourcedFrom(...)`. Its `static async load(entry)` method picks the lowest-latency available node for cache-miss fetches.
+
+### Shared status buffers (`getReplicationSharedStatus` in `knownNodes.ts`)
+
+Per (database, remote_node) pair: an mmap-backed `Float64Array` shared across threads, used to avoid IPC for hot-path status updates. Position constants live in `replicationConnection.ts`:
+
+| Position | Constant                       |
+| -------- | ------------------------------ |
+| 0        | `CONFIRMATION_STATUS_POSITION` |
+| 1        | `RECEIVED_VERSION_POSITION`    |
+| 2        | `RECEIVED_TIME_POSITION`       |
+| 3        | `SENDING_TIME_POSITION`        |
+| 4        | `LATENCY_POSITION`             |
+| 5        | `RECEIVING_STATUS_POSITION`    |
+| 6        | `BACK_PRESSURE_RATIO_POSITION` |
+
+These are written concurrently by `replicationConnection.ts` without explicit synchronization. Don't introduce read-modify-write patterns on this buffer.
+
+### `hdb_nodes` system table (`getHDBNodeTable` in `knownNodes.ts`)
+
+Schema (defined in that function): `name` (PK), `subscriptions[]`, `system_info`, `url`, `routes`, `ca`, `ca_info`, `replicates`, `revoked_certificates`, plus `__createdtime__` / `__updatedtime__`. Subscription updates flow through `subscribeToNodeUpdates`, which fans out to `monitorNodeCAs` → refresh `replicationCertificateAuthorities` (exported from `replicator.ts`).
+
+---
+
+## Subsystems
+
+**Connection management** — `NodeReplicationConnection.connect()` (`replicationConnection.ts`), `subscriptionManager.startOnMainThread()`. Dial/retry, thread-pool delegation, recovery on disconnect.
+
+**Binary protocol** — `replicateOverWS` (`replicationConnection.ts`); command constants are the `*_REQUEST` / `*_UPDATE` / `*_RESPONSE` consts at module top; msgpack body; back-pressure ratio recomputed on `BACK_PRESSURE_INTERVAL` (30 s).
+
+**Data propagation** — Audit-record iteration → forwarding; blob streaming with concurrency cap `MAX_OUTSTANDING_BLOBS_BEING_SENT` (declared inside `replicateOverWS`); commit confirmation batched on `COMMITTED_UPDATE_DELAY` (2 ms).
+
+**Latency awareness** — Ping every `PING_INTERVAL` (default 30 s, `replication.pingInterval`); a connection with no socket activity for `PING_TIMEOUT` (default 2× interval, `replication.pingTimeout`) is terminated; latency captured on pong; `Replicator.load()` routes cache-miss fetches to the lowest-latency node.
+
+**Node discovery & TLS** — `hdb_nodes` subscriptions, `setNode.ts` for member ops, `buildReplicationMtlsConfig()` (`replicator.ts`), `monitorNodeCAs()` (`replicator.ts`).
+
+---
+
+## Non-obvious behaviors
+
+1. **Auth failures don't send DISCONNECT.** When the `authorization` promise rejects in `replicateOverWS`, the connection closes with "Unauthorized" but no DISCONNECT frame is sent — the client is expected to retry. This is the lineage of JWT/cluster auth bugs (issue #135).
+
+2. **Origin loop prevention via delayed sequence updates.** A node receiving its own message (checked against `remoteToLocalNodeId`) skips local processing but still forwards. To avoid feedback loops, the sequence-update emit is delayed by `SKIPPED_MESSAGE_SEQUENCE_UPDATE_DELAY` (300 ms; in `replicationConnection.ts`).
+
+3. **Blob back-pressure & timeout.** Blobs time out after `blobTimeout` (default 120s); concurrent sends are capped at `MAX_OUTSTANDING_BLOBS_BEING_SENT = 5`; back-pressure ratio (computed every `BACK_PRESSURE_INTERVAL`) tells senders to pause. If you're seeing large-data replication hangs, look here first.
+
+4. **Shared-buffer concurrency.** The Float64Array status buffers are touched from multiple threads with no lock. Treat them as eventually consistent; use the callback param of `subscribeToNodeUpdates` if you need notification.
+
+5. **Per-route table exclusion (`excludeTables`).** Route entries in `sendsTo`/`receivesFrom` can specify `excludeTables: ['hdb_nodes']` to prevent specific tables from crossing the wire. Three layers enforce this: (a) subscriber omits them from SUBSCRIPTION_REQUEST (in `sendSubscriptionRequestUpdate`); (b) sender skips their audit records before streaming (in `sendAuditRecord`); (c) receiver drops any that arrive (in the incoming message loop). Route config `routeReplicates` is threaded from `subscriptionManager.ts` onto `nodeSubscriptions` objects so static-route exclusions are available inside HTTP worker threads where `replicateOverWS` runs. Primary use case: v4→v5 migration bridges that share `system` database users/roles but must keep per-cluster `hdb_nodes` topology tables isolated.
+
+6. **Keep-alive is measured from byte activity, not a single ping interval.** `shouldTerminateIdlePing` terminates a connection only after no socket bytes have moved in either direction for the full `PING_TIMEOUT`. A bulk transfer — notably the initial clone copy of a large table — makes slow but real progress (the sender's buffer drains in bursts as the peer consumes), so bytes keep moving within the window and it is not killed mid-copy (the old "no bytes since last ping → terminate" heuristic restarted the copy from zero — issue #241). A genuinely dead peer moves no bytes and still trips the timeout — including the case where the sender filled its socket buffer and the `drain` event never fires. The sole exemption is `pauseReasons > 0` (the receiver intentionally stopped reading to drain its own queue): that stall is local and self-clearing, so the caller keeps liveness fresh while paused. Relatedly, the receive decode loop in `replicateOverWS` yields the event loop on a time budget (`RECEIVE_YIELD_INTERVAL`) — not only when the consumer queue exceeds `RECEIVE_EVENT_HIGH_WATER_MARK` — so a single large message can't decode in one synchronous turn and starve ping responses (core's `MAX_EVENT_DELAY_TIME` monitor).
+
+6. **The initial bulk clone copy is resumable (PK cursor).** When a follower requests a full copy (`startTime: 0`), the leader sends `COPY_START{copyStartTime}`, walks each table's primary store in key order, flushes a checkpoint transaction every `COPY_CHECKPOINT_RECORDS` (timed at `copyStartTime` so the persisted `seqId` stays pinned there, never a record's `localTime`), and sends `COPY_COMPLETE` at the end. The follower persists a cursor `{copyStartTime, currentTable, afterKey}` under `dbisDB` key `Symbol.for('copyCursor')` — but only in the `end_txn` `onCommit`, **after** the batch commits, so the cursor can never get ahead of committed data (a resume re-copies a few records idempotently but never skips). On reconnect, `sendSubscriptionRequestUpdate` reads the cursor and sends it as `copyResume` on the subscription request (overriding the persisted `seqId`, which alone would skip the un-copied tables); the leader skips tables before `currentTable` (stable iteration order ⇒ already committed) and resumes `currentTable` after `afterKey`. `COPY_COMPLETE` clears the cursor so subsequent connections resume normally from `seqId`. Before this, an interrupted copy restarted from zero and never converged for a large table (issue #241).
+
+---
+
+## Tests
+
+**Integration tests** live in `../integrationTests/cluster/`:
+
+| File                                   | Purpose                                                  |
+| -------------------------------------- | -------------------------------------------------------- |
+| `clusterShared.mjs`                    | Shared fixture/helper (cluster boot, node setup)         |
+| `fullyConnectedReplication.test.mjs`   | Full-mesh topology                                       |
+| `replicationTopology.test.mjs`         | Dynamic membership changes                               |
+| `replicationLoad.test.mjs`             | Concurrent-write load                                    |
+| `excludeTablesReplication.test.mjs`    | Per-route `excludeTables` bridge migration (issue #239)  |
+
+Most replication behavior is exercised via integration tests that spin up multi-node clusters. A few function-level invariants that don't need a cluster live in `../unitTests/replication/` (e.g. `listenerLifecycle.test.mjs`, `pingKeepalive.test.mjs`).
+
+---
+
+## "Where is X" cheat sheet
+
+| Question                                  | Where                                                                                          |
+| ----------------------------------------- | ---------------------------------------------------------------------------------------------- |
+| Where does a remote message get decoded?  | `replicationConnection.ts → replicateOverWS`                                                   |
+| Where do cache-miss fetches pick a peer?  | `replicator.ts → Replicator.load` (declared inside `setReplicator`)                            |
+| Where is the connection retry loop?       | `replicationConnection.ts → NodeReplicationConnection` (uses `INITIAL_RETRY_TIME`)             |
+| Where is mTLS configured?                 | `replicator.ts → buildReplicationMtlsConfig`                                                   |
+| Where is a new cluster member added?      | `setNode.ts` (the whole file is one operation)                                                 |
+| Where are protocol message types defined? | `replicationConnection.ts` — top-level consts (`SUBSCRIPTION_REQUEST` … `SUBSCRIPTION_UPDATE`) |
+| Where is `hdb_nodes` schema?              | `knownNodes.ts → getHDBNodeTable`                                                              |
+| What does `cluster_status` return?        | `clusterStatus.ts` (82 lines, whole file)                                                      |
+| Where is per-route table exclusion logic? | `knownNodes.ts → getExcludedTablesForRouteEntries`; threaded via `subscriptionManager.ts → routeReplicates` |

package/replication/knownNodes.ts

@@ -8,7 +8,7 @@ import { getThisNodeName } from '../core/server/nodeName.ts';
 import { replicationConfirmation } from '../core/resources/DatabaseTransaction.ts';
 import { isMainThread } from 'worker_threads';
 import { ClientError } from '../core/utility/errors/hdbError.js';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
 import { logger } from '../core/utility/logging/logger.ts';
 
@@ -74,43 +74,104 @@ export function getReplicationSharedStatus(
 		)
 	);
 }
-export function subscribeToNodeUpdates(listener: (node: any, id: string) => void) {
-	getHDBNodeTable()
-		.subscribe({})
-		.then(async (events) => {
+// If the async iterator for hdb_nodes throws or completes, the watcher used to die silently
+// and the node lost the ability to (re)establish outbound replication subscriptions for the
+// lifetime of the process. Run the watcher inside a restart loop so a single transient error
+// (e.g. WebSocket close, schema mismatch, downstream throw) does not permanently disable
+// node-update tracking. Per-event errors are caught individually so they cannot tear down
+// the loop.
+const NODE_WATCHER_RESTART_DELAY_MS = 1000;
+// Cap the exponential backoff so a persistent failure (subscribe throws every time)
+// doesn't run a tight 1s log+retry loop forever — back off up to 30s instead.
+const NODE_WATCHER_MAX_DELAY_MS = 30_000;
+type WatcherOptions = {
+	subscribe?: () => Promise<AsyncIterable<any>> | AsyncIterable<any>;
+	processEvent?: (event: any, listener: (node: any, id: string) => void) => Promise<void> | void;
+	restartDelayMs?: number;
+	maxDelayMs?: number;
+	maxRestarts?: number;
+};
+export async function runNodeUpdateWatcher(listener: (node: any, id: string) => void, options: WatcherOptions = {}) {
+	const subscribe = options.subscribe ?? (() => getHDBNodeTable().subscribe({}));
+	const processEvent = options.processEvent ?? processNodeUpdateEvent;
+	const restartDelayMs = options.restartDelayMs ?? NODE_WATCHER_RESTART_DELAY_MS;
+	const maxDelayMs = options.maxDelayMs ?? NODE_WATCHER_MAX_DELAY_MS;
+	const maxRestarts = options.maxRestarts ?? Infinity;
+	let restarts = 0;
+	let consecutiveFailures = 0;
+	while (restarts < maxRestarts) {
+		let iteratedSuccessfully = false;
+		try {
+			const events = await subscribe();
+			iteratedSuccessfully = true; // we got past subscribe — any later throw is a fresh failure
 			for await (const event of events) {
-				// remove any nodes that have been updated or deleted
-				const node_name = event?.value?.name;
-				logger.debug?.('adding node', node_name, 'on  node', getThisNodeName(), ' on process', process.pid);
-				server.nodes = server.nodes.filter((node) => node.name !== node_name);
-				if (event.type === 'put' && node_name !== getThisNodeName()) {
-					// add any new nodes
-					if (event.value) server.nodes.push(event.value);
-					else {
-						console.error('Invalid node update event', event);
-					}
-				}
-				const shards = new Map();
-				for await (const node of getHDBNodeTable().search({})) {
-					if (node.shard != undefined) {
-						let nodesForShard = shards.get(node.shard);
-						if (!nodesForShard) {
-							shards.set(node.shard, (nodesForShard = []));
-						}
-						nodesForShard.push(node);
-					}
-				}
-				server.shards = shards;
-				if (event.type === 'put' || event.type === 'delete') {
-					listener(event.value, event.id);
+				try {
+					await processEvent(event, listener);
+				} catch (error) {
+					// Don't let a single bad event tear down the watcher — log and continue.
+					// Optional chaining: this `logger` is the level-conditional one, where
+					// `.error` is undefined when the configured level filters it out, and an
+					// uncaught TypeError here would defeat the whole recovery loop.
+					logger.error?.('Error processing hdb_nodes update event', error);
 				}
 			}
-		});
+			logger.warn?.('hdb_nodes subscription ended unexpectedly; restarting watcher');
+		} catch (error) {
+			logger.error?.('hdb_nodes watcher failed; restarting', error);
+		}
+		// Successful subscribe → reset backoff so a fresh failure restarts quickly.
+		consecutiveFailures = iteratedSuccessfully ? 0 : consecutiveFailures + 1;
+		restarts++;
+		if (restarts >= maxRestarts) return;
+		const delay = Math.min(restartDelayMs * Math.pow(2, Math.min(consecutiveFailures, 5)), maxDelayMs);
+		await new Promise((resolve) => setTimeout(resolve, delay));
+	}
+}
+async function processNodeUpdateEvent(event: any, listener: (node: any, id: string) => void) {
+	// remove any nodes that have been updated or deleted
+	const node_name = event?.value?.name || event?.id;
+	logger.debug?.('adding node', node_name, 'on  node', getThisNodeName(), ' on process', process.pid);
+	server.nodes = server.nodes.filter((node) => node && node.name !== node_name);
+	if (event.type === 'put' && node_name !== getThisNodeName()) {
+		// add any new nodes
+		if (event.value) server.nodes.push(event.value);
+		else {
+			console.error('Invalid node update event', event);
+		}
+	} else if (event.type === 'patch' && node_name !== getThisNodeName() && event.value?.isLeader !== undefined) {
+		// add_node { isLeader: true } reaches us as a patch event; read the merged
+		// record from LMDB so server.nodes reflects the full record (including isLeader).
+		const fullRecord = getHDBNodeTable().primaryStore.get(node_name);
+		if (fullRecord) server.nodes.push(fullRecord);
+	}
+	const shards = new Map();
+	for await (const node of getHDBNodeTable().search({})) {
+		if (!node) continue;
+		if (node.shard != undefined) {
+			let nodesForShard = shards.get(node.shard);
+			if (!nodesForShard) {
+				shards.set(node.shard, (nodesForShard = []));
+			}
+			nodesForShard.push(node);
+		}
+	}
+	server.shards = shards;
+	if (event.type === 'put' || event.type === 'delete') {
+		listener(event.value, event.id);
+	} else if (event.type === 'patch' && event.value?.isLeader !== undefined) {
+		// isLeader patches need to drive subscription bootstrap; pass the merged record.
+		const fullRecord = getHDBNodeTable().primaryStore.get(event.id);
+		if (fullRecord) listener(fullRecord, event.id);
+	}
+}
+export function subscribeToNodeUpdates(listener: (node: any, id: string) => void) {
+	runNodeUpdateWatcher(listener);
 	server.nodes = [];
 	server.shards = new Map();
 
 	for (let entry of getHDBNodeTable().primaryStore.getRange({})) {
 		const { value: node, key } = entry;
+		if (!node) continue;
 		server.nodes.push(node);
 		if (node.shard != undefined) {
 			let nodesForShard = server.shards.get(node.shard);
@@ -131,6 +192,11 @@ export function shouldReplicateFromNode(node: Node, databaseName: string) {
 	const databaseReplications: string | Array<string | { name: string; sharded?: boolean }> = env.get(
 		CONFIG_PARAMS.REPLICATION_DATABASES
 	);
+	// When this peer is our leader, the database may not exist locally yet — that's the
+	// whole point of the full-table copy bootstrap. Skip the local-presence precondition
+	// so the subscription can be scheduled and the leader can push records (and schema)
+	// to create the database on this node.
+	const hasLocalDatabase = !!databases[databaseName] || !!node.isLeader;
 	return (
 		((typeof node.replicates === 'object'
 			? node.replicates?.sends ||
@@ -141,7 +207,7 @@ export function shouldReplicateFromNode(node: Node, databaseName: string) {
 						: sendsTo === getThisNodeName()
 				)
 			: node.replicates) &&
-			databases[databaseName] &&
+			hasLocalDatabase &&
 			(!databaseReplications ||
 				databaseReplications === '*' ||
 				(Array.isArray(databaseReplications) &&
@@ -238,16 +304,21 @@ function startSubscriptionToReplications() {
 				if (auditStore) break;
 			}
 			if (auditStore) {
-				const replicatedTime = getReplicationSharedStatus(auditStore, databaseName, nodeNameAtUpdate, () => {
-					const updatedTime = replicatedTime[0];
-					const lastTime = replicatedTime.lastTime;
-					for (const { txnTime, onConfirm } of commitsAwaitingReplication.get(databaseName) || []) {
-						if (txnTime > lastTime && txnTime <= updatedTime) {
-							onConfirm();
+				const replicatedTime: Float64Array & { lastTime?: number } = getReplicationSharedStatus(
+					auditStore,
+					databaseName,
+					nodeNameAtUpdate,
+					() => {
+						const updatedTime = replicatedTime[0];
+						const lastTime = replicatedTime.lastTime;
+						for (const { txnTime, onConfirm } of commitsAwaitingReplication.get(databaseName) || []) {
+							if (txnTime > lastTime && txnTime <= updatedTime) {
+								onConfirm();
+							}
 						}
+						replicatedTime.lastTime = updatedTime;
 					}
-					replicatedTime.lastTime = updatedTime;
-				});
+				);
 				replicatedTime.lastTime = 0;
 				confirmationsForNode.set(databaseName, replicatedTime);
 			}
@@ -255,6 +326,12 @@ function startSubscriptionToReplications() {
 		confirmationWatchersByNode.set(nodeNameAtUpdate, handle);
 	});
 }
+export type RouteEntry = {
+	target?: string;
+	source?: string;
+	database?: string;
+	excludeTables?: string[];
+};
 export type Route = {
 	url?: string;
 	subscriptions?: { database: string; schema: string; subscribe: boolean }[];
@@ -262,6 +339,18 @@ export type Route = {
 	host?: string;
 	port?: any;
 	routes?: any[];
+	sendsTo?: (RouteEntry | string)[];
+	receivesFrom?: (RouteEntry | string)[];
+	// yielded by iterateRoutes (may differ from raw config shape)
+	name?: string;
+	replicates?:
+		| boolean
+		| {
+				sends?: boolean;
+				sendsTo?: (RouteEntry | string)[];
+				receives?: boolean;
+				receivesFrom?: (RouteEntry | string)[];
+		  };
 };
 export type Node = {
 	name: string;
@@ -270,17 +359,42 @@ export type Node = {
 		| boolean
 		| {
 				sends?: boolean;
-				sendsTo?: ({ target: string; database: string } | string)[];
+				sendsTo?: (RouteEntry | string)[];
 				receives?: boolean;
-				receivesFrom?: ({ source: string; database: string } | string)[];
+				receivesFrom?: (RouteEntry | string)[];
 		  };
 	url?: string;
 	port?: number;
 	startTime?: number;
 	revoked_certificates?: string[];
 	shard?: number;
+	isLeader?: boolean;
 };
 
+/**
+ * Returns the set of tables to exclude for a given peer+database from a sendsTo or receivesFrom
+ * route-entry array. Returns null when there are no exclusions (hot path: avoids a Set allocation).
+ */
+export function getExcludedTablesForRouteEntries(
+	entries: (RouteEntry | string)[] | undefined,
+	peerName: string,
+	databaseName: string
+): Set<string> | null {
+	if (!entries) return null;
+	let excluded: Set<string> | null = null;
+	for (const entry of entries) {
+		if (typeof entry === 'string') continue;
+		const entryPeer = entry.target ?? entry.source;
+		if ((!entryPeer || entryPeer === peerName) && (!entry.database || entry.database === databaseName)) {
+			if (entry.excludeTables?.length) {
+				if (!excluded) excluded = new Set(entry.excludeTables);
+				else for (const t of entry.excludeTables) excluded.add(t);
+			}
+		}
+	}
+	return excluded;
+}
+
 export function* iterateRoutes(options: { routes: (Route | any)[] }) {
 	for (const route of options.routes || []) {
 		let url = route.url;
@@ -297,8 +411,17 @@ export function* iterateRoutes(options: { routes: (Route | any)[] }) {
 			continue;
 		}
 
+		// Support sendsTo/receivesFrom either nested under replicates: or as top-level route keys
+		let replicates = route.replicates;
+		if (replicates === undefined) {
+			if (route.sendsTo || route.receivesFrom) {
+				replicates = { sendsTo: route.sendsTo, receivesFrom: route.receivesFrom };
+			} else {
+				replicates = !route.subscriptions; // if there is not a list of subscriptions, then this node is authorized to fully replicate
+			}
+		}
 		yield {
-			replicates: route.replicates ?? !route.subscriptions, // if there is not a list of subscriptions, then this node is authorized to fully replicate
+			replicates,
 			name: host,
 			url,
 			port: route.port,
@@ -312,7 +435,7 @@ export function* iterateRoutes(options: { routes: (Route | any)[] }) {
 
 export function getNodeURL(node: Node): string {
 	if (node.url) return node.url;
-	const host = node.name;
+	let host = node.name;
 	const securePort = env.get(CONFIG_PARAMS.REPLICATION_SECUREPORT);
 	let port: any;
 	// if the host includes a port, use that port