@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/replication/replicator.ts

@@ -27,7 +27,7 @@ import {
 	LATENCY_POSITION,
 } from './replicationConnection.ts';
 import { server } from '../core/server/Server.ts';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import * as logger from '../core/utility/logging/harper_logger.js';
 import { verifyCertificate } from '../core/security/certificateVerification/index.ts';
 export { startOnMainThread } from './subscriptionManager.ts';
@@ -97,12 +97,12 @@ export function start(options) {
 	logger.notify('Starting replication server');
 	if (options.hostname && !env.get('node_hostname')) {
 		// for back-compat, carry this over
-		env.setProperty('node_hostname');
+		env.setProperty('node_hostname', options.hostname);
 		clearThisNodeName();
 	}
 	if (options.url && !env.get('node_url')) {
 		// for back-compat, carry this over
-		env.setProperty('node_url');
+		env.setProperty('node_url', options.url);
 		clearThisNodeName();
 	}
 	if (!options.port && !options.securePort) {
@@ -152,7 +152,7 @@ export function start(options) {
 	server.http(async (request, nextHandler) => {
 		if (request.isWebSocket && request.headers.get('Sec-WebSocket-Protocol') === 'harperdb-replication-v1') {
 			logger.debug('Incoming replication WS connection received, authorized: ' + request.authorized);
-			const authorizationError = request._nodeRequest.socket.authorizationError;
+			const authorizationError = (request._nodeRequest.socket as any).authorizationError;
 			if (authorizationError) {
 				if (authorizationError === 'CERT_HAS_EXPIRED') {
 					logger.error(
@@ -166,7 +166,7 @@ export function start(options) {
 						'certificate issuer',
 						request.peerCertificate.issuer,
 						'did not match any available CAs',
-						Array.from(Array.from(wsServers[0].secureContexts.values())[0].options.availableCAs.keys())
+						Array.from((Array.from(wsServers[0].secureContexts.values())[0] as any).options.availableCAs.keys())
 					);
 				}
 			}
@@ -249,7 +249,7 @@ export function start(options) {
 				// add a big performance penalty on connection setup
 				const contextsToUpdate = new Set(wsServer.secureContexts.values());
 				if (wsServer.defaultContext) contextsToUpdate.add(wsServer.defaultContext);
-				for (const context of contextsToUpdate) {
+				for (const context of contextsToUpdate as Set<any>) {
 					try {
 						const ca = Array.from(replicationCertificateAuthorities);
 						// add the replication CAs (and root CAs) to any existing CAs for the context
@@ -344,6 +344,7 @@ export function setReplicator(dbName: string, table: any, options: any) {
 	// We may try to consult this to get the other nodes for back-compat
 	// const { hub_routes } = getClusteringRoutes();
 	table.sourcedFrom(
+		// @ts-expect-error: static side intentionally differs from Resource
 		class Replicator extends Resource {
 			/**
 			 * This subscribes to the other nodes. Subscription events are notifications rather than
@@ -475,7 +476,9 @@ function getSubscriptionConnection(
 			(connection = new NodeReplicationConnection(connectingUrl, subscription, dbName, nodeName, authorization))
 		);
 		connection.connect();
-		connection.once('finished', () => dbConnections.delete(dbName));
+		connection.once('finished', () => {
+			if (dbConnections.get(dbName) === connection) dbConnections.delete(dbName);
+		});
 		return connection;
 	}
 }
@@ -502,7 +505,7 @@ function getRetrievalConnectionByName(nodeName, subscription, dbName): NodeRepli
 	return connection;
 }
 
-export async function sendOperationToNode(node, operation, options) {
+export async function sendOperationToNode(node, operation, options?) {
 	if (!options) options = {};
 	options.serverName = node.name;
 	const socket = await createWebSocket(getNodeURL(node), options);
@@ -590,7 +593,7 @@ export async function unsubscribeFromNode({ url, nodes, database }) {
 	}
 }
 
-server.replication = {
+(server as any).replication = {
 	exportIdMapping,
 	getIdOfRemoteNode,
 };
@@ -659,8 +662,8 @@ function hasExplicitlyReplicatedTable(databaseName) {
 	}
 }
 
-export async function replicateOperation(req) {
-	const response = { message: '' };
+export async function replicateOperation(req, options?: { onPeerResult?: (result: any) => void }) {
+	const response: { message: string; replicated?: any[] } = { message: '' };
 	if (req.replicated !== false) {
 		req.replicated = false; // don't send a replicated flag to the nodes we are sending to
 		logger.trace?.(
@@ -669,21 +672,35 @@ export async function replicateOperation(req) {
 			'to nodes',
 			server.nodes.map((node) => node.name)
 		);
-		const replicatedResults = await Promise.allSettled(
-			server.nodes.map((node) => {
-				// do all the nodes in parallel
-				return sendOperationToNode(node, req);
-			})
+		// Dispatch all peers in parallel. Each promise resolves to a normalized result
+		// (with `.node` attached), and the optional `onPeerResult` callback fires as
+		// each peer settles — letting callers surface per-peer progress in real time
+		// rather than waiting for the aggregate at the end.
+		const onPeerResult = options?.onPeerResult;
+		const perPeer = server.nodes.map((node) =>
+			sendOperationToNode(node, req)
+				.then((value: any) => {
+					const result: any = value && typeof value === 'object' ? value : { value };
+					result.node = node.name;
+					return result;
+				})
+				.catch((reason) => ({
+					status: 'failed',
+					reason: reason?.toString?.() ?? String(reason),
+					node: node.name,
+				}))
+				.then((result) => {
+					if (onPeerResult) {
+						try {
+							onPeerResult(result);
+						} catch (err) {
+							logger.warn?.('onPeerResult callback threw; ignoring', err);
+						}
+					}
+					return result;
+				})
 		);
-		// map the settled results to the response
-		response.replicated = replicatedResults.map((settledResult, index) => {
-			const result =
-				settledResult.status === 'rejected'
-					? { status: 'failed', reason: settledResult.reason.toString() }
-					: settledResult.value;
-			result.node = server.nodes[index]?.name; // add the node to the result so we know which node succeeded/failed
-			return result;
-		});
+		response.replicated = await Promise.all(perPeer);
 	}
 	return response;
 }

package/replication/setNode.ts

@@ -17,10 +17,13 @@ const { HTTP_STATUS_CODES } = hdbErrors;
 const validationSchema = Joi.object({
 	hostname: Joi.string(),
 	verify_tls: Joi.boolean(),
-	replicates: Joi.boolean(),
+	replicates: Joi.alternatives().try(Joi.boolean(), Joi.object()),
+	sendsTo: Joi.array(),
+	receivesFrom: Joi.array(),
 	subscriptions: Joi.array(),
 	revoked_certificates: Joi.array(),
 	shard: Joi.number(),
+	isLeader: Joi.boolean(),
 });
 
 /**
@@ -97,7 +100,7 @@ export async function setNode(req: any) {
 	}
 
 	// This is the record that will be added to the other nodes hdbNodes table
-	const targetAddNodeObj = {
+	const targetAddNodeObj: any = {
 		operation: 'add_node_back',
 		hostname: getThisNodeName(),
 		target_hostname: hostname,
@@ -112,6 +115,13 @@ export async function setNode(req: any) {
 		targetAddNodeObj.subscriptions = req.subscriptions.map(reverseSubscription);
 	} else targetAddNodeObj.subscriptions = null;
 
+	// Pass route exclusions to the peer with directions swapped: what LOCAL sends to PEER
+	// becomes what PEER receives from LOCAL, and vice versa.
+	if (req.sendsTo || req.receivesFrom) {
+		targetAddNodeObj.sendsTo = req.receivesFrom;
+		targetAddNodeObj.receivesFrom = req.sendsTo;
+	}
+
 	if (req.hasOwnProperty('subscribe') || req.hasOwnProperty('publish')) {
 		const rev = reverseSubscription(req);
 		targetAddNodeObj.subscribe = rev.subscribe;
@@ -125,7 +135,7 @@ export async function setNode(req: any) {
 
 	let targetNodeResponse: any;
 	let targetNodeResponseError: Error;
-	hdbLogger.debug('sending add_node_back operation', targetAddNodeObj);
+	hdbLogger.notify('sending add_node_back operation', targetAddNodeObj);
 	try {
 		targetNodeResponse = await sendOperationToNode({ url }, targetAddNodeObj, req);
 	} catch (err) {
@@ -167,9 +177,11 @@ export async function setNode(req: any) {
 		cert_auth = targetNodeResponse.signingCA;
 	}
 
-	const nodeRecord = { url, ca: targetNodeResponse?.usingCA };
+	const nodeRecord: any = { url, ca: targetNodeResponse?.usingCA };
 	if (req.hostname) nodeRecord.name = req.hostname;
 	if (req.subscriptions) nodeRecord.subscriptions = req.subscriptions;
+	else if (req.sendsTo || req.receivesFrom)
+		nodeRecord.replicates = { sends: true, sendsTo: req.sendsTo, receivesFrom: req.receivesFrom };
 	else nodeRecord.replicates = true;
 	if (req.start_time) {
 		nodeRecord.start_time = typeof req.start_time === 'string' ? new Date(req.start_time).getTime() : req.start_time;
@@ -178,9 +190,13 @@ export async function setNode(req: any) {
 	if (req.revoked_certificates) nodeRecord.revoked_certificates = req.revoked_certificates;
 	if (targetNodeResponse?.shard !== undefined) nodeRecord.shard = targetNodeResponse.shard;
 	else if (req.shard !== undefined) nodeRecord.shard = req.shard;
+	// isLeader is LOCAL-ONLY: it means "this peer is my leader; request a full-table copy from it".
+	// It must NOT be forwarded to the peer via targetAddNodeObj — doing so would cause the peer
+	// to treat this node as its leader and attempt to full-copy in the wrong direction.
+	if (req.isLeader !== undefined) nodeRecord.isLeader = req.isLeader;
 
 	if (nodeRecord.replicates) {
-		const thisNode = {
+		const thisNode: any = {
 			url: thisUrl,
 			ca: cert_auth,
 			replicates: true,
@@ -212,7 +228,7 @@ export async function setNode(req: any) {
 export async function addNodeBack(req) {
 	hdbLogger.trace('addNodeBack received request:', req);
 
-	let certs = {};
+	let certs: any = {};
 	// If the add_node req has a CSR attached, return the CA that was used to issue the CSR,
 	// else return whatever CA this node is using for replication
 	let originCa: string;
@@ -230,9 +246,12 @@ export async function addNodeBack(req) {
 		);
 	}
 
-	const nodeRecord = { url: req.url, ca: originCa };
+	const nodeRecord: any = { url: req.url, ca: originCa };
 	if (req.subscriptions) nodeRecord.subscriptions = req.subscriptions;
-	else {
+	else if (req.sendsTo || req.receivesFrom) {
+		nodeRecord.replicates = { sends: true, sendsTo: req.sendsTo, receivesFrom: req.receivesFrom };
+		nodeRecord.subscriptions = null;
+	} else {
 		nodeRecord.replicates = true;
 		nodeRecord.subscriptions = null;
 	}
@@ -243,7 +262,7 @@ export async function addNodeBack(req) {
 
 	const repCa = await getReplicationCertAuth();
 	if (nodeRecord.replicates) {
-		const thisNode = {
+		const thisNode: any = {
 			url: getThisNodeUrl(),
 			ca: repCa?.certificate,
 			replicates: true,

package/replication/subscriptionManager.ts

@@ -14,25 +14,25 @@ import {
 	getHDBNodeTable,
 	iterateRoutes,
 	shouldReplicateFromNode,
-	type Node,
 	type Route,
 	getNodeURL,
 } from './knownNodes.ts';
 import * as logger from '../core/utility/logging/harper_logger.js';
 import lodash from 'lodash';
 const { cloneDeep } = lodash;
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
 import { X509Certificate } from 'crypto';
 import minimist from 'minimist';
 const cliArgs = minimist(process.argv);
 
 type ConnectedWorkerStatus = {
-	worker: Worker | null;
+	worker: any;
 	connected?: boolean;
 	latency?: number;
 };
 type ReplicationConnectionStatus = {
+	url?: string;
 	nodes: ({
 		name: string;
 		url: string;
@@ -41,9 +41,12 @@ type ReplicationConnectionStatus = {
 		startTime?: number;
 		endTime?: number;
 		shard?: string;
+		subscriptions?: any;
+		worker?: any;
+		isLeader?: boolean;
 	} & ConnectedWorkerStatus)[];
 } & ConnectedWorkerStatus;
-type DBReplicationStatusMap = Map<string, ReplicationConnectionStatus> & { iterator: any };
+type DBReplicationStatusMap = Map<string, ReplicationConnectionStatus> & { iterator?: any };
 
 const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to other nodes, so operations can complete first
 // When a worker dies it may have been holding subscriptions for many (database, node) pairs.
@@ -52,8 +55,36 @@ const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to othe
 // caused the OOM in the first place. We stagger the re-subscriptions in time so the new
 // worker(s) absorb them gradually.
 const WORKER_EXIT_REASSIGN_STAGGER_MS = 100;
+// Cadence of the per-process safety-net reconcile that rebinds subscriptions whose
+// worker no longer exists. Pure read-side filter against `workers` and
+// `connectionReplicationMap` on each tick when nothing is wrong, so a short interval
+// is cheap. Sized for the deploy-time rapid-restart-storm pattern (stacked
+// `restart_http_workers` at ~1.5s spacing under live write traffic), where the per-
+// worker exit chain races against shutdown and silently drops half the subscription
+// assignments — this is the user-visible recovery latency for the resulting drift.
+const RECONCILE_INTERVAL_MS = 5_000;
 let nextWorkerExitReassignAt = 0;
 const connectionReplicationMap = new Map<string, DBReplicationStatusMap>();
+// Returns the set of node URLs whose replication entries either point at a worker no longer
+// in the supplied http pool, OR have no worker assigned at all while live workers exist.
+// The second case covers "all workers were down at registration time" — onDatabase stores
+// `worker: undefined` when httpWorkers is empty, and without this the entry would never
+// get reassigned once workers came back. Pure helper so the reconcile pass below — and its
+// unit tests — can verify the broken-chain detection without spinning up real workers.
+export function findStaleNodeUrls(connectionMap: Map<string, DBReplicationStatusMap>, httpWorkers: any[]): Set<string> {
+	const staleNodeUrls = new Set<string>();
+	// No live workers to reassign to — flagging here would cause endless no-op reassignments.
+	if (httpWorkers.length === 0) return staleNodeUrls;
+	for (const [url, dbReplicationWorkers] of connectionMap) {
+		for (const entry of dbReplicationWorkers.values()) {
+			if (!entry.worker || !httpWorkers.includes(entry.worker)) {
+				staleNodeUrls.add(url);
+				break;
+			}
+		}
+	}
+	return staleNodeUrls;
+}
 export let disconnectedFromNode; // this is set by thread to handle when a node is disconnected (or notify main thread so it can handle)
 export let connectedToNode; // this is set by thread to handle when a node is connected (or notify main thread so it can handle)
 const nodeMap = new Map(); // this is a map of all nodes that are available to connect to
@@ -71,7 +102,7 @@ export async function startOnMainThread(options) {
 		for (const tableName in database) {
 			const table = database[tableName];
 			if (table.auditStore) {
-				selfCatchupOfDatabase.set(dbName, lastTimeInAuditStore(table.auditStore));
+				selfCatchupOfDatabase.set(dbName, lastTimeInAuditStore(table.auditStore) as number);
 				break;
 			}
 		}
@@ -102,7 +133,7 @@ export async function startOnMainThread(options) {
 			}
 		}
 		if (getHDBNodeTable().primaryStore.get(thisName)) ensureThisNode(); // if this node record already exists, check for config changes
-		for (const route of iterateRoutes(options)) {
+		for (const route of iterateRoutes(options) as any) {
 			try {
 				const replicateAll = !route.subscriptions;
 				if (replicateAll) {
@@ -143,6 +174,7 @@ export async function startOnMainThread(options) {
 		logger.info('Setting up node replication for', node);
 		if (!node) {
 			// deleted node
+			nodeMap.delete(hostname);
 			for (const [url, dbReplicationWorkers] of connectionReplicationMap) {
 				let foundNode;
 				for (const [_database, { nodes }] of dbReplicationWorkers) {
@@ -170,7 +202,16 @@ export async function startOnMainThread(options) {
 		if (isSelf) return;
 		let dbReplicationWorkers = connectionReplicationMap.get(getNodeURL(node));
 		if (dbReplicationWorkers) dbReplicationWorkers.iterator.remove(); // we need to remove the old iterator so we can create a new one
-		if (!(node.replicates === true || node.replicates?.sends) && !node.subscriptions?.length && !dbReplicationWorkers)
+		if (
+			!(
+				node.replicates === true ||
+				node.replicates?.sends ||
+				node.replicates?.sendsTo?.length ||
+				node.replicates?.receivesFrom?.length
+			) &&
+			!node.subscriptions?.length &&
+			!dbReplicationWorkers
+		)
 			return; // we don't have any subscriptions and we haven't connected yet, so just return
 		logger.info(`Added node ${node.name} at ${getNodeURL(node)} for process ${getThisNodeName()}`);
 		if (node.replicates && node.subscriptions) {
@@ -211,12 +252,37 @@ export async function startOnMainThread(options) {
 				}
 			}
 		}
+		// When this peer is our leader, bootstrap subscriptions for configured databases
+		// that don't exist locally yet — they need a full-table copy from the leader.
+		// forEachReplicatedDatabase above only iterates local databases, so an empty node
+		// joining a populated leader would never schedule the catchup without this.
+		if (node.isLeader && Array.isArray(options?.databases)) {
+			for (const dbConfig of options.databases) {
+				const databaseName = typeof dbConfig === 'string' ? dbConfig : dbConfig?.name;
+				if (databaseName && !databases[databaseName]) {
+					logger.warn(
+						`isLeader: bootstrapping full-copy subscription for non-existent database ${databaseName} from ${node.name}`
+					);
+					onDatabase(databaseName, true);
+				}
+			}
+		}
 
 		function onDatabase(databaseName, tablesReplicateByDefault) {
 			logger.trace('Setting up replication for database', databaseName, 'on node', node.name);
-			const existingEntry = dbReplicationWorkers.get(databaseName);
+			let existingEntry = dbReplicationWorkers.get(databaseName);
 			let worker;
-			const nodes = [{ replicateByDefault: tablesReplicateByDefault, ...node }];
+			// Find the matching route config for this peer so we can pass its receivesFrom/sendsTo
+			// exclusions to the worker thread (via the node subscription payload). For dynamic
+			// routes (add_node), fall back to the node's own hdb_nodes replicates object.
+			const matchingRoute = routes.find((r) => r.name === node.name);
+			const routeReplicates =
+				typeof matchingRoute?.replicates === 'object'
+					? matchingRoute.replicates
+					: node.replicates && typeof node.replicates === 'object'
+						? node.replicates
+						: null;
+			const nodes = [{ replicateByDefault: tablesReplicateByDefault, ...node, routeReplicates }];
 			// Self catchup is done in case we have replicated any records that weren't actually written to our storage
 			// before a crash.
 			if (selfCatchupOfDatabase.has(databaseName) && env.get(CONFIG_PARAMS.REPLICATION_FAILOVER)) {
@@ -233,6 +299,18 @@ export async function startOnMainThread(options) {
 			}
 			const shouldSubscribe = shouldReplicateFromNode(node, databaseName);
 			const httpWorkers = workers.filter((worker) => worker.name === 'http');
+			// Defensively detect entries that point at a worker no longer in the http pool.
+			// This happens when the worker.on('exit') handler below never fired (hung WebSocket
+			// refs blocking exit), the identity check rejected the reassignment, or its
+			// setTimeout retry was lost. We also catch the case where the entry has no worker
+			// assigned at all (all workers were down at registration time) so it gets rebound
+			// once workers come back. Without these checks, the early-return branch keeps the
+			// entry stuck and the subscription never recovers.
+			if (existingEntry && httpWorkers.length > 0 && !httpWorkers.includes(existingEntry.worker as any)) {
+				logger.warn(`Subscription for ${databaseName} on node ${node.name} has no live worker; reassigning`);
+				dbReplicationWorkers.delete(databaseName);
+				existingEntry = undefined;
+			}
 			if (existingEntry) {
 				worker = existingEntry.worker;
 				existingEntry.nodes = nodes;
@@ -268,6 +346,12 @@ export async function startOnMainThread(options) {
 					cliArgs.HDB_LEADER_URL ?? // first see if there was a leader explicitly specified
 					process.env.HDB_LEADER_URL ??
 					routes[0]?.url; // if we have routes, use the first one
+				// Track whether the leader is explicitly configured (env/cli/routes). The
+				// fallback "first other node in hdb_nodes" is only a guess and must NOT be
+				// treated as authoritative — otherwise a bidirectional add_node handshake
+				// where the responder has no leader config will incorrectly mark the
+				// requester as its leader and trigger a reverse full-table copy.
+				const hasExplicitLeader = !!leaderUrl;
 
 				let leaderName = leaderUrl
 					? new URL(leaderUrl).hostname
@@ -278,7 +362,13 @@ export async function startOnMainThread(options) {
 						)[0]; // try to find the first node
 				const nodeName = nodes[0].name ?? (nodes[0].url && new URL(nodes[0].url).hostname);
 				logger.warn(`Setting up subscription with leader ${leaderName} for node ${nodeName}`);
-				nodes[0].isLeader = !leaderName || nodeName === leaderName;
+				// isLeader is true only if:
+				//   1. it was explicitly persisted (e.g. by add_node { isLeader: true }), OR
+				//   2. there is no leader candidate at all, OR
+				//   3. an explicitly configured leader (env/cli/routes) matches this node.
+				// We deliberately do NOT honour nodeName === leaderName when leaderName came
+				// from the "first other node in hdb_nodes" fallback — that's just a guess.
+				nodes[0].isLeader = nodes[0].isLeader || !leaderName || (hasExplicitLeader && nodeName === leaderName);
 				nodes[0].url ??= getNodeURL(nodes[0]);
 				setTimeout(() => {
 					const request = {
@@ -353,8 +443,16 @@ export async function startOnMainThread(options) {
 				// if failover is disabled, immediately return
 				return;
 			}
-			const mainNode = existingWorkerEntry.nodes[0];
-			if (!(mainNode.replicates === true || mainNode.replicates?.sends || mainNode.subscriptions?.length)) {
+			const mainNode: any = existingWorkerEntry.nodes[0];
+			if (
+				!(
+					mainNode.replicates === true ||
+					mainNode.replicates?.sends ||
+					mainNode.replicates?.sendsTo?.length ||
+					mainNode.replicates?.receivesFrom?.length ||
+					mainNode.subscriptions?.length
+				)
+			) {
 				// no replication, so just return
 				return;
 			}
@@ -462,7 +560,7 @@ export async function startOnMainThread(options) {
 			}
 		}
 	};
-	function connectToNextWorker(node: Node, database: string, connectingNode = node) {
+	function connectToNextWorker(node: any, database: string, connectingNode = node) {
 		const httpWorkers = workers.filter((worker: any) => worker.name === 'http');
 		nextWorkerIndex = nextWorkerIndex % httpWorkers.length; // wrap around as necessary
 		const worker = httpWorkers[nextWorkerIndex++];
@@ -478,6 +576,32 @@ export async function startOnMainThread(options) {
 			});
 		} else subscribeToNode({ url: getNodeURL(connectingNode), name: connectingNode.name, database, nodes: [node] });
 	}
+	// Periodic safety net for stale subscription entries. The existing per-database
+	// worker.on('exit') chain reassigns to a healthy worker after a worker dies, but a
+	// single broken link in that chain (identity check failing, setTimeout retry being
+	// lost under load, shouldSubscribe early-return pinning to a dead worker before
+	// the defensive check was added) used to leave the entry permanently pointing at
+	// an exited worker, silently breaking outbound replication for the lifetime of the
+	// process. This reconciles independently of the chain so the broken-state node
+	// can never get stuck.
+	function reconcileWorkers() {
+		const httpWorkers = workers.filter((worker) => worker.name === 'http');
+		const staleNodeUrls = findStaleNodeUrls(connectionReplicationMap, httpWorkers);
+		if (staleNodeUrls.size === 0) return;
+		logger.warn(
+			'Reconciling replication subscriptions for nodes pointing at exited workers:',
+			Array.from(staleNodeUrls)
+		);
+		for (const node of nodeMap.values()) {
+			if (!staleNodeUrls.has(getNodeURL(node))) continue;
+			try {
+				onNodeUpdate(node);
+			} catch (error) {
+				logger.error('Error reconciling node', node?.name, error);
+			}
+		}
+	}
+	setInterval(reconcileWorkers, RECONCILE_INTERVAL_MS).unref();
 	onMessageByType('disconnected-from-node', disconnectedFromNode);
 	onMessageByType('connected-to-node', connectedToNode);
 	onMessageByType('request-cluster-status', requestClusterStatus);
@@ -489,7 +613,7 @@ export async function startOnMainThread(options) {
  * @param message
  * @param port
  */
-export function requestClusterStatus(message, port) {
+export function requestClusterStatus(message?, port?) {
 	const connections = [];
 	for (const [node_name, node] of nodeMap) {
 		try {

package/security/certificate.ts

@@ -18,7 +18,7 @@ import {
 	getPrimaryHostName,
 	setCertTable,
 } from '../core/security/keys.js';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { LICENSE_KEY_DIR_NAME } from '../core/utility/hdbTerms.ts';
 import harperLogger from '../core/utility/logging/harper_logger.js';
 import { getThisNodeName } from '../core/server/nodeName.ts';
@@ -36,7 +36,7 @@ const fileExists = async (path: string): Promise<boolean> =>
 		.catch(() => false);
 
 export async function signCertificate(req) {
-	const response = {};
+	const response: any = {};
 	const hdbKeysDir = join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME);
 
 	if (req.csr) {
@@ -117,7 +117,11 @@ export async function createCsr() {
 	const hdbKeysDir = join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME);
 	let opsCert, opsPrivateKey, certName, privateKeyName;
 	for await (const cert of certificateTable.search([])) {
-		if (cert.is_self_signed && (cert.details?.issuer?.includes('Harper-Certificate-Authority') || cert.details?.issuer?.includes('HarperDB-Certificate-Authority'))) {
+		if (
+			cert.is_self_signed &&
+			(cert.details?.issuer?.includes('Harper-Certificate-Authority') ||
+				cert.details?.issuer?.includes('HarperDB-Certificate-Authority'))
+		) {
 			// privateKeys Map is populated from config-referenced paths only, so a Harper CA key
 			// on disk but not referenced in config won't appear there — fall back to reading it.
 			let key: string | Buffer | undefined = privateKeys.get(cert.private_key_name);
@@ -361,7 +365,7 @@ async function removeCertificate(req: { name: string }): Promise<{ message: stri
 		);
 
 		// Only delete the key file if this is the only cert referencing it.
-		if (matchingKeys.length === 1 && matchingKeys[0].name === name) {
+		if (matchingKeys.length === 1 && (matchingKeys[0] as any).name === name) {
 			try {
 				logger.info?.('Removing private key named', private_key_name);
 				await unlink(join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME, private_key_name));

package/security/sshKeyOperations.ts

@@ -6,7 +6,7 @@ import { validateBySchema } from '../core/validation/validationWrapper.js';
 import harperLogger from '../core/utility/logging/harper_logger.js';
 import { ClientError } from '../core/utility/errors/hdbError.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { replicateOperation } from '../replication/replicator.ts';
 
 // SSH key name can only be alphanumeric, dash and underscores

package/static/defaultConfig.yaml

@@ -22,6 +22,7 @@ authentication:
   refreshTokenTimeout: 30d
 analytics:
   aggregatePeriod: 60
+  profiling: true
   replicate: false
 applications:
   lockdown: freeze-after-load