@harperfast/harper-pro 5.0.30 → 5.1.0-beta.2

package/replication/subscriptionManager.ts

@@ -14,25 +14,28 @@ import {
 	getHDBNodeTable,
 	iterateRoutes,
 	shouldReplicateFromNode,
-	type Node,
 	type Route,
 	getNodeURL,
 } from './knownNodes.ts';
 import * as logger from '../core/utility/logging/harper_logger.js';
 import lodash from 'lodash';
 const { cloneDeep } = lodash;
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
 import { X509Certificate } from 'crypto';
 import minimist from 'minimist';
 const cliArgs = minimist(process.argv);
 
 type ConnectedWorkerStatus = {
-	worker: Worker | null;
+	worker: any;
 	connected?: boolean;
 	latency?: number;
+	// Timestamp (ms) of the most recent transition to connected:false, used by the reconcile to
+	// distinguish a connection that is briefly retrying from one that is wedged. Cleared on connect.
+	disconnectedAt?: number;
 };
 type ReplicationConnectionStatus = {
+	url?: string;
 	nodes: ({
 		name: string;
 		url: string;
@@ -41,9 +44,12 @@ type ReplicationConnectionStatus = {
 		startTime?: number;
 		endTime?: number;
 		shard?: string;
+		subscriptions?: any;
+		worker?: any;
+		isLeader?: boolean;
 	} & ConnectedWorkerStatus)[];
 } & ConnectedWorkerStatus;
-type DBReplicationStatusMap = Map<string, ReplicationConnectionStatus> & { iterator: any };
+type DBReplicationStatusMap = Map<string, ReplicationConnectionStatus> & { iterator?: any };
 
 const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to other nodes, so operations can complete first
 // When a worker dies it may have been holding subscriptions for many (database, node) pairs.
@@ -52,8 +58,79 @@ const NODE_SUBSCRIBE_DELAY = 200; // delay before sending node subscribe to othe
 // caused the OOM in the first place. We stagger the re-subscriptions in time so the new
 // worker(s) absorb them gradually.
 const WORKER_EXIT_REASSIGN_STAGGER_MS = 100;
+// Cadence of the per-process safety-net reconcile that rebinds subscriptions whose
+// worker no longer exists. Pure read-side filter against `workers` and
+// `connectionReplicationMap` on each tick when nothing is wrong, so a short interval
+// is cheap. Sized for the deploy-time rapid-restart-storm pattern (stacked
+// `restart_http_workers` at ~1.5s spacing under live write traffic), where the per-
+// worker exit chain races against shutdown and silently drops half the subscription
+// assignments — this is the user-visible recovery latency for the resulting drift.
+const RECONCILE_INTERVAL_MS = 5_000;
+// A connection that is connected:false but still actively retrying reconnects within seconds (the
+// retry backoff starts at 500ms). Only re-drive a connection that has stayed disconnected well
+// beyond that, so the reconcile targets genuinely wedged connections (e.g. an intentionally-closed
+// connection with no pending retry) rather than churning connections that are mid-reconnect.
+const WEDGE_RECONCILE_THRESHOLD_MS = 30_000;
 let nextWorkerExitReassignAt = 0;
 const connectionReplicationMap = new Map<string, DBReplicationStatusMap>();
+// Returns the set of node URLs whose replication entries either point at a worker no longer
+// in the supplied http pool, OR have no worker assigned at all while live workers exist.
+// The second case covers "all workers were down at registration time" — onDatabase stores
+// `worker: undefined` when httpWorkers is empty, and without this the entry would never
+// get reassigned once workers came back. Pure helper so the reconcile pass below — and its
+// unit tests — can verify the broken-chain detection without spinning up real workers.
+export function findStaleNodeUrls(connectionMap: Map<string, DBReplicationStatusMap>, httpWorkers: any[]): Set<string> {
+	const staleNodeUrls = new Set<string>();
+	// No live workers to reassign to — flagging here would cause endless no-op reassignments.
+	if (httpWorkers.length === 0) return staleNodeUrls;
+	for (const [url, dbReplicationWorkers] of connectionMap) {
+		for (const entry of dbReplicationWorkers.values()) {
+			if (!entry.worker || !httpWorkers.includes(entry.worker)) {
+				staleNodeUrls.add(url);
+				break;
+			}
+		}
+	}
+	return staleNodeUrls;
+}
+// Returns the set of node URLs that have a desired replication subscription but have been
+// connected:false on a *live* worker for longer than `thresholdMs`. This is the recovery path for a
+// connection that wedged without a pending retry — most notably the empty-subscription delayed close
+// (intentionallyUnsubscribed) firing during a peer restart and then never re-establishing even though
+// the peer is reachable and still subscribed. findStaleNodeUrls does not catch this because the
+// worker is alive. Re-driving these through onNodeUpdate creates a fresh connection (the prior one is
+// no longer reusable — see replicator.isReusableConnection). A threshold well above the normal
+// reconnect backoff keeps this from firing on connections that are merely mid-retry.
+// `isDesired` must be the same predicate onDatabase uses to decide shouldSubscribe
+// (shouldReplicateFromNode), so a connection intentionally unsubscribed because this node should NOT
+// subscribe (replication off, or a sendsTo/subscription targeting another database) is not flagged
+// and re-driven forever. See harper-pro#233 / #289.
+export function findWedgedNodeUrls(
+	connectionMap: Map<string, DBReplicationStatusMap>,
+	httpWorkers: any[],
+	now: number,
+	thresholdMs: number,
+	isDesired: (node: any, database: string) => boolean
+): Set<string> {
+	const wedgedNodeUrls = new Set<string>();
+	if (httpWorkers.length === 0) return wedgedNodeUrls;
+	for (const [url, dbReplicationWorkers] of connectionMap) {
+		for (const [database, entry] of dbReplicationWorkers) {
+			if (
+				entry.connected === false &&
+				entry.worker &&
+				httpWorkers.includes(entry.worker) &&
+				entry.disconnectedAt != null &&
+				now - entry.disconnectedAt >= thresholdMs &&
+				isDesired(entry.nodes?.[0], database)
+			) {
+				wedgedNodeUrls.add(url);
+				break;
+			}
+		}
+	}
+	return wedgedNodeUrls;
+}
 export let disconnectedFromNode; // this is set by thread to handle when a node is disconnected (or notify main thread so it can handle)
 export let connectedToNode; // this is set by thread to handle when a node is connected (or notify main thread so it can handle)
 const nodeMap = new Map(); // this is a map of all nodes that are available to connect to
@@ -71,7 +148,7 @@ export async function startOnMainThread(options) {
 		for (const tableName in database) {
 			const table = database[tableName];
 			if (table.auditStore) {
-				selfCatchupOfDatabase.set(dbName, lastTimeInAuditStore(table.auditStore));
+				selfCatchupOfDatabase.set(dbName, lastTimeInAuditStore(table.auditStore) as number);
 				break;
 			}
 		}
@@ -102,7 +179,7 @@ export async function startOnMainThread(options) {
 			}
 		}
 		if (getHDBNodeTable().primaryStore.get(thisName)) ensureThisNode(); // if this node record already exists, check for config changes
-		for (const route of iterateRoutes(options)) {
+		for (const route of iterateRoutes(options) as any) {
 			try {
 				const replicateAll = !route.subscriptions;
 				if (replicateAll) {
@@ -126,7 +203,7 @@ export async function startOnMainThread(options) {
 	 * This is called when a new node is added to the hdbNodes table
 	 * @param node
 	 */
-	function onNodeUpdate(node, hostname = node?.name) {
+	function onNodeUpdate(node, hostname = node?.name, forceResubscribe = false) {
 		const isSelf =
 			(getThisNodeName() && hostname === getThisNodeName()) || (getThisNodeUrl() && node?.url === getThisNodeUrl());
 		if (isSelf) {
@@ -143,6 +220,7 @@ export async function startOnMainThread(options) {
 		logger.info('Setting up node replication for', node);
 		if (!node) {
 			// deleted node
+			nodeMap.delete(hostname);
 			for (const [url, dbReplicationWorkers] of connectionReplicationMap) {
 				let foundNode;
 				for (const [_database, { nodes }] of dbReplicationWorkers) {
@@ -170,7 +248,16 @@ export async function startOnMainThread(options) {
 		if (isSelf) return;
 		let dbReplicationWorkers = connectionReplicationMap.get(getNodeURL(node));
 		if (dbReplicationWorkers) dbReplicationWorkers.iterator.remove(); // we need to remove the old iterator so we can create a new one
-		if (!(node.replicates === true || node.replicates?.sends) && !node.subscriptions?.length && !dbReplicationWorkers)
+		if (
+			!(
+				node.replicates === true ||
+				node.replicates?.sends ||
+				node.replicates?.sendsTo?.length ||
+				node.replicates?.receivesFrom?.length
+			) &&
+			!node.subscriptions?.length &&
+			!dbReplicationWorkers
+		)
 			return; // we don't have any subscriptions and we haven't connected yet, so just return
 		logger.info(`Added node ${node.name} at ${getNodeURL(node)} for process ${getThisNodeName()}`);
 		if (node.replicates && node.subscriptions) {
@@ -194,9 +281,9 @@ export async function startOnMainThread(options) {
 		}
 		dbReplicationWorkers.iterator = forEachReplicatedDatabase(options, (database, databaseName, replicateByDefault) => {
 			if (replicateByDefault) {
-				onDatabase(databaseName, true);
+				onDatabase(databaseName, true, forceResubscribe);
 			} else {
-				onDatabase(databaseName, false);
+				onDatabase(databaseName, false, forceResubscribe);
 			}
 		});
 		// check to see if there are any explicit subscriptions to databases that don't exist yet
@@ -207,16 +294,41 @@ export async function startOnMainThread(options) {
 				const databaseName = sub.database || sub.schema;
 				if (!databases[databaseName]) {
 					logger.warn(`Database ${databaseName} not found for node ${node.name}, making a subscription anyway`);
-					onDatabase(databaseName, false);
+					onDatabase(databaseName, false, forceResubscribe);
+				}
+			}
+		}
+		// When this peer is our leader, bootstrap subscriptions for configured databases
+		// that don't exist locally yet — they need a full-table copy from the leader.
+		// forEachReplicatedDatabase above only iterates local databases, so an empty node
+		// joining a populated leader would never schedule the catchup without this.
+		if (node.isLeader && Array.isArray(options?.databases)) {
+			for (const dbConfig of options.databases) {
+				const databaseName = typeof dbConfig === 'string' ? dbConfig : dbConfig?.name;
+				if (databaseName && !databases[databaseName]) {
+					logger.warn(
+						`isLeader: bootstrapping full-copy subscription for non-existent database ${databaseName} from ${node.name}`
+					);
+					onDatabase(databaseName, true, forceResubscribe);
 				}
 			}
 		}
 
-		function onDatabase(databaseName, tablesReplicateByDefault) {
+		function onDatabase(databaseName, tablesReplicateByDefault, forceResubscribe = false) {
 			logger.trace('Setting up replication for database', databaseName, 'on node', node.name);
-			const existingEntry = dbReplicationWorkers.get(databaseName);
+			let existingEntry = dbReplicationWorkers.get(databaseName);
 			let worker;
-			const nodes = [{ replicateByDefault: tablesReplicateByDefault, ...node }];
+			// Find the matching route config for this peer so we can pass its receivesFrom/sendsTo
+			// exclusions to the worker thread (via the node subscription payload). For dynamic
+			// routes (add_node), fall back to the node's own hdb_nodes replicates object.
+			const matchingRoute = routes.find((r) => r.name === node.name);
+			const routeReplicates =
+				typeof matchingRoute?.replicates === 'object'
+					? matchingRoute.replicates
+					: node.replicates && typeof node.replicates === 'object'
+						? node.replicates
+						: null;
+			const nodes = [{ replicateByDefault: tablesReplicateByDefault, ...node, routeReplicates }];
 			// Self catchup is done in case we have replicated any records that weren't actually written to our storage
 			// before a crash.
 			if (selfCatchupOfDatabase.has(databaseName) && env.get(CONFIG_PARAMS.REPLICATION_FAILOVER)) {
@@ -233,11 +345,28 @@ export async function startOnMainThread(options) {
 			}
 			const shouldSubscribe = shouldReplicateFromNode(node, databaseName);
 			const httpWorkers = workers.filter((worker) => worker.name === 'http');
+			// Defensively detect entries that point at a worker no longer in the http pool.
+			// This happens when the worker.on('exit') handler below never fired (hung WebSocket
+			// refs blocking exit), the identity check rejected the reassignment, or its
+			// setTimeout retry was lost. We also catch the case where the entry has no worker
+			// assigned at all (all workers were down at registration time) so it gets rebound
+			// once workers come back. Without these checks, the early-return branch keeps the
+			// entry stuck and the subscription never recovers.
+			if (existingEntry && httpWorkers.length > 0 && !httpWorkers.includes(existingEntry.worker as any)) {
+				logger.warn(`Subscription for ${databaseName} on node ${node.name} has no live worker; reassigning`);
+				dbReplicationWorkers.delete(databaseName);
+				existingEntry = undefined;
+			}
 			if (existingEntry) {
 				worker = existingEntry.worker;
 				existingEntry.nodes = nodes;
-				if (shouldSubscribe) {
-					// already subscribed, don't need to do anything
+				// Normally an existing subscribed entry is left alone. Only the wedge reconcile passes
+				// forceResubscribe for a connection that has been connected:false past the threshold: that
+				// falls through to re-post subscribe-to-node on the same worker (the worker then reuses a
+				// still-retrying connection or builds a fresh one — replicator.isReusableConnection). We
+				// deliberately do NOT re-subscribe every connected:false entry on an ordinary onNodeUpdate —
+				// doing so disrupts in-flight replication (e.g. an active legacy-node base copy).
+				if (shouldSubscribe && !(forceResubscribe && existingEntry.connected === false)) {
 					return;
 				}
 			} else if (shouldSubscribe) {
@@ -268,6 +397,12 @@ export async function startOnMainThread(options) {
 					cliArgs.HDB_LEADER_URL ?? // first see if there was a leader explicitly specified
 					process.env.HDB_LEADER_URL ??
 					routes[0]?.url; // if we have routes, use the first one
+				// Track whether the leader is explicitly configured (env/cli/routes). The
+				// fallback "first other node in hdb_nodes" is only a guess and must NOT be
+				// treated as authoritative — otherwise a bidirectional add_node handshake
+				// where the responder has no leader config will incorrectly mark the
+				// requester as its leader and trigger a reverse full-table copy.
+				const hasExplicitLeader = !!leaderUrl;
 
 				let leaderName = leaderUrl
 					? new URL(leaderUrl).hostname
@@ -278,7 +413,13 @@ export async function startOnMainThread(options) {
 						)[0]; // try to find the first node
 				const nodeName = nodes[0].name ?? (nodes[0].url && new URL(nodes[0].url).hostname);
 				logger.warn(`Setting up subscription with leader ${leaderName} for node ${nodeName}`);
-				nodes[0].isLeader = !leaderName || nodeName === leaderName;
+				// isLeader is true only if:
+				//   1. it was explicitly persisted (e.g. by add_node { isLeader: true }), OR
+				//   2. there is no leader candidate at all, OR
+				//   3. an explicitly configured leader (env/cli/routes) matches this node.
+				// We deliberately do NOT honour nodeName === leaderName when leaderName came
+				// from the "first other node in hdb_nodes" fallback — that's just a guess.
+				nodes[0].isLeader = nodes[0].isLeader || !leaderName || (hasExplicitLeader && nodeName === leaderName);
 				nodes[0].url ??= getNodeURL(nodes[0]);
 				setTimeout(() => {
 					const request = {
@@ -345,6 +486,9 @@ export async function startOnMainThread(options) {
 				logger.warn('Disconnected node not found in replication map', connection.database, dbReplicationWorkers);
 				return;
 			}
+			// Record the first transition to disconnected so the reconcile can tell a wedged connection
+			// from one that is briefly mid-retry; don't reset it on repeated disconnect notifications.
+			if (existingWorkerEntry.connected !== false) existingWorkerEntry.disconnectedAt = Date.now();
 			existingWorkerEntry.connected = false;
 			if (connection.finished) {
 				return;
@@ -353,8 +497,16 @@ export async function startOnMainThread(options) {
 				// if failover is disabled, immediately return
 				return;
 			}
-			const mainNode = existingWorkerEntry.nodes[0];
-			if (!(mainNode.replicates === true || mainNode.replicates?.sends || mainNode.subscriptions?.length)) {
+			const mainNode: any = existingWorkerEntry.nodes[0];
+			if (
+				!(
+					mainNode.replicates === true ||
+					mainNode.replicates?.sends ||
+					mainNode.replicates?.sendsTo?.length ||
+					mainNode.replicates?.receivesFrom?.length ||
+					mainNode.subscriptions?.length
+				)
+			) {
 				// no replication, so just return
 				return;
 			}
@@ -413,6 +565,7 @@ export async function startOnMainThread(options) {
 			return;
 		}
 		mainWorkerEntry.connected = true;
+		mainWorkerEntry.disconnectedAt = undefined;
 		mainWorkerEntry.latency = connection.latency;
 		const restoredNode = mainWorkerEntry.nodes[0];
 		if (!restoredNode) {
@@ -462,7 +615,7 @@ export async function startOnMainThread(options) {
 			}
 		}
 	};
-	function connectToNextWorker(node: Node, database: string, connectingNode = node) {
+	function connectToNextWorker(node: any, database: string, connectingNode = node) {
 		const httpWorkers = workers.filter((worker: any) => worker.name === 'http');
 		nextWorkerIndex = nextWorkerIndex % httpWorkers.length; // wrap around as necessary
 		const worker = httpWorkers[nextWorkerIndex++];
@@ -478,6 +631,59 @@ export async function startOnMainThread(options) {
 			});
 		} else subscribeToNode({ url: getNodeURL(connectingNode), name: connectingNode.name, database, nodes: [node] });
 	}
+	// Periodic safety net for stale subscription entries. The existing per-database
+	// worker.on('exit') chain reassigns to a healthy worker after a worker dies, but a
+	// single broken link in that chain (identity check failing, setTimeout retry being
+	// lost under load, shouldSubscribe early-return pinning to a dead worker before
+	// the defensive check was added) used to leave the entry permanently pointing at
+	// an exited worker, silently breaking outbound replication for the lifetime of the
+	// process. This reconciles independently of the chain so the broken-state node
+	// can never get stuck.
+	function reconcileWorkers() {
+		const httpWorkers = workers.filter((worker) => worker.name === 'http');
+		const staleNodeUrls = findStaleNodeUrls(connectionReplicationMap, httpWorkers);
+		const wedgedNodeUrls = findWedgedNodeUrls(
+			connectionReplicationMap,
+			httpWorkers,
+			Date.now(),
+			WEDGE_RECONCILE_THRESHOLD_MS,
+			shouldReplicateFromNode
+		);
+		if (staleNodeUrls.size === 0 && wedgedNodeUrls.size === 0) return;
+		if (staleNodeUrls.size > 0)
+			logger.warn(
+				'Reconciling replication subscriptions for nodes pointing at exited workers:',
+				Array.from(staleNodeUrls)
+			);
+		if (wedgedNodeUrls.size > 0)
+			logger.warn(
+				'Reconciling replication subscriptions for nodes wedged disconnected on a live worker:',
+				Array.from(wedgedNodeUrls)
+			);
+		for (const node of nodeMap.values()) {
+			const url = getNodeURL(node);
+			const isWedged = wedgedNodeUrls.has(url);
+			if (!staleNodeUrls.has(url) && !isWedged) continue;
+			if (isWedged) {
+				// Restart the disconnect clock before re-driving so a peer that is still unreachable after
+				// the re-subscribe (it stays connected:false, which does not re-stamp disconnectedAt) is
+				// retried at most once per threshold window rather than on every reconcile tick.
+				// onNodeUpdate -> onDatabase then re-posts subscribe-to-node for the connected:false entry
+				// (it no longer early-returns), reusing the existing worker — so no entry/listener churn.
+				const entries = connectionReplicationMap.get(url);
+				if (entries)
+					for (const entry of entries.values()) if (entry.connected === false) entry.disconnectedAt = Date.now();
+			}
+			try {
+				// forceResubscribe only for wedged entries, so a normal stale-worker reconcile keeps its
+				// original behavior and ordinary onNodeUpdate calls never re-subscribe live subscriptions.
+				onNodeUpdate(node, undefined, isWedged);
+			} catch (error) {
+				logger.error('Error reconciling node', node?.name, error);
+			}
+		}
+	}
+	setInterval(reconcileWorkers, RECONCILE_INTERVAL_MS).unref();
 	onMessageByType('disconnected-from-node', disconnectedFromNode);
 	onMessageByType('connected-to-node', connectedToNode);
 	onMessageByType('request-cluster-status', requestClusterStatus);
@@ -489,7 +695,7 @@ export async function startOnMainThread(options) {
  * @param message
  * @param port
  */
-export function requestClusterStatus(message, port) {
+export function requestClusterStatus(message?, port?) {
 	const connections = [];
 	for (const [node_name, node] of nodeMap) {
 		try {

package/security/certificate.ts

@@ -18,7 +18,7 @@ import {
 	getPrimaryHostName,
 	setCertTable,
 } from '../core/security/keys.js';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { LICENSE_KEY_DIR_NAME } from '../core/utility/hdbTerms.ts';
 import harperLogger from '../core/utility/logging/harper_logger.js';
 import { getThisNodeName } from '../core/server/nodeName.ts';
@@ -36,7 +36,7 @@ const fileExists = async (path: string): Promise<boolean> =>
 		.catch(() => false);
 
 export async function signCertificate(req) {
-	const response = {};
+	const response: any = {};
 	const hdbKeysDir = join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME);
 
 	if (req.csr) {
@@ -117,7 +117,11 @@ export async function createCsr() {
 	const hdbKeysDir = join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME);
 	let opsCert, opsPrivateKey, certName, privateKeyName;
 	for await (const cert of certificateTable.search([])) {
-		if (cert.is_self_signed && (cert.details?.issuer?.includes('Harper-Certificate-Authority') || cert.details?.issuer?.includes('HarperDB-Certificate-Authority'))) {
+		if (
+			cert.is_self_signed &&
+			(cert.details?.issuer?.includes('Harper-Certificate-Authority') ||
+				cert.details?.issuer?.includes('HarperDB-Certificate-Authority'))
+		) {
 			// privateKeys Map is populated from config-referenced paths only, so a Harper CA key
 			// on disk but not referenced in config won't appear there — fall back to reading it.
 			let key: string | Buffer | undefined = privateKeys.get(cert.private_key_name);
@@ -361,7 +365,7 @@ async function removeCertificate(req: { name: string }): Promise<{ message: stri
 		);
 
 		// Only delete the key file if this is the only cert referencing it.
-		if (matchingKeys.length === 1 && matchingKeys[0].name === name) {
+		if (matchingKeys.length === 1 && (matchingKeys[0] as any).name === name) {
 			try {
 				logger.info?.('Removing private key named', private_key_name);
 				await unlink(join(env.getHdbBasePath(), LICENSE_KEY_DIR_NAME, private_key_name));

package/security/sshKeyOperations.ts

@@ -6,7 +6,7 @@ import { validateBySchema } from '../core/validation/validationWrapper.js';
 import harperLogger from '../core/utility/logging/harper_logger.js';
 import { ClientError } from '../core/utility/errors/hdbError.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import { replicateOperation } from '../replication/replicator.ts';
 
 // SSH key name can only be alphanumeric, dash and underscores

package/static/defaultConfig.yaml

@@ -22,6 +22,7 @@ authentication:
   refreshTokenTimeout: 30d
 analytics:
   aggregatePeriod: 60
+  profiling: true
   replicate: false
 applications:
   lockdown: freeze-after-load