@harperfast/harper-pro 5.0.30 → 5.1.0-beta.2

package/replication/replicator.ts

@@ -27,7 +27,7 @@ import {
 	LATENCY_POSITION,
 } from './replicationConnection.ts';
 import { server } from '../core/server/Server.ts';
-import env from '../core/utility/environment/environmentManager.js';
+import * as env from '../core/utility/environment/environmentManager.js';
 import * as logger from '../core/utility/logging/harper_logger.js';
 import { verifyCertificate } from '../core/security/certificateVerification/index.ts';
 export { startOnMainThread } from './subscriptionManager.ts';
@@ -97,12 +97,12 @@ export function start(options) {
 	logger.notify('Starting replication server');
 	if (options.hostname && !env.get('node_hostname')) {
 		// for back-compat, carry this over
-		env.setProperty('node_hostname');
+		env.setProperty('node_hostname', options.hostname);
 		clearThisNodeName();
 	}
 	if (options.url && !env.get('node_url')) {
 		// for back-compat, carry this over
-		env.setProperty('node_url');
+		env.setProperty('node_url', options.url);
 		clearThisNodeName();
 	}
 	if (!options.port && !options.securePort) {
@@ -152,7 +152,7 @@ export function start(options) {
 	server.http(async (request, nextHandler) => {
 		if (request.isWebSocket && request.headers.get('Sec-WebSocket-Protocol') === 'harperdb-replication-v1') {
 			logger.debug('Incoming replication WS connection received, authorized: ' + request.authorized);
-			const authorizationError = request._nodeRequest.socket.authorizationError;
+			const authorizationError = (request._nodeRequest.socket as any).authorizationError;
 			if (authorizationError) {
 				if (authorizationError === 'CERT_HAS_EXPIRED') {
 					logger.error(
@@ -166,7 +166,7 @@ export function start(options) {
 						'certificate issuer',
 						request.peerCertificate.issuer,
 						'did not match any available CAs',
-						Array.from(Array.from(wsServers[0].secureContexts.values())[0].options.availableCAs.keys())
+						Array.from((Array.from(wsServers[0].secureContexts.values())[0] as any).options.availableCAs.keys())
 					);
 				}
 			}
@@ -249,7 +249,7 @@ export function start(options) {
 				// add a big performance penalty on connection setup
 				const contextsToUpdate = new Set(wsServer.secureContexts.values());
 				if (wsServer.defaultContext) contextsToUpdate.add(wsServer.defaultContext);
-				for (const context of contextsToUpdate) {
+				for (const context of contextsToUpdate as Set<any>) {
 					try {
 						const ca = Array.from(replicationCertificateAuthorities);
 						// add the replication CAs (and root CAs) to any existing CAs for the context
@@ -344,6 +344,7 @@ export function setReplicator(dbName: string, table: any, options: any) {
 	// We may try to consult this to get the other nodes for back-compat
 	// const { hub_routes } = getClusteringRoutes();
 	table.sourcedFrom(
+		// @ts-expect-error: static side intentionally differs from Resource
 		class Replicator extends Resource {
 			/**
 			 * This subscribes to the other nodes. Subscription events are notifications rather than
@@ -447,6 +448,16 @@ export function setReplicator(dbName: string, table: any, options: any) {
 }
 const connections = new Map<string, Map<string, NodeReplicationConnection>>();
 
+// A connection that was intentionally torn down — the empty-subscription delayed close marks it
+// intentionallyUnsubscribed/isFinished — must not be handed back out of the cache. connect()
+// early-returns forever on intentionallyUnsubscribed, so a peer whose subscription set transiently
+// emptied (e.g. during a peer restart) and was then restored would otherwise stay permanently dead
+// on a connected:false connection that never retries. Callers drop a non-reusable cached connection
+// and create a fresh one through the normal connect path. See harper-pro#233 / #289.
+export function isReusableConnection(connection?: NodeReplicationConnection): boolean {
+	return !!connection && !connection.isFinished && !connection.intentionallyUnsubscribed;
+}
+
 /**
  * Get or create a connection to the specified node
  * @param url
@@ -468,14 +479,16 @@ function getSubscriptionConnection(
 		connections.set(connectionKey, dbConnections);
 	}
 	let connection = dbConnections.get(dbName);
-	if (connection) return connection;
+	if (isReusableConnection(connection)) return connection;
 	if (subscription) {
 		dbConnections.set(
 			dbName,
 			(connection = new NodeReplicationConnection(connectingUrl, subscription, dbName, nodeName, authorization))
 		);
 		connection.connect();
-		connection.once('finished', () => dbConnections.delete(dbName));
+		connection.once('finished', () => {
+			if (dbConnections.get(dbName) === connection) dbConnections.delete(dbName);
+		});
 		return connection;
 	}
 }
@@ -490,7 +503,7 @@ function getRetrievalConnectionByName(nodeName, subscription, dbName): NodeRepli
 		nodeNameToRetrievalConnections.set(nodeName, dbConnections);
 	}
 	let connection = dbConnections.get(dbName);
-	if (connection) return connection;
+	if (isReusableConnection(connection)) return connection;
 	const node = getHDBNodeTable().primaryStore.get(nodeName);
 	if (node?.url) {
 		connection = new NodeReplicationConnection(getNodeURL(node), subscription, dbName, nodeName, node.authorization);
@@ -502,7 +515,7 @@ function getRetrievalConnectionByName(nodeName, subscription, dbName): NodeRepli
 	return connection;
 }
 
-export async function sendOperationToNode(node, operation, options) {
+export async function sendOperationToNode(node, operation, options?) {
 	if (!options) options = {};
 	options.serverName = node.name;
 	const socket = await createWebSocket(getNodeURL(node), options);
@@ -590,7 +603,7 @@ export async function unsubscribeFromNode({ url, nodes, database }) {
 	}
 }
 
-server.replication = {
+(server as any).replication = {
 	exportIdMapping,
 	getIdOfRemoteNode,
 };
@@ -659,8 +672,8 @@ function hasExplicitlyReplicatedTable(databaseName) {
 	}
 }
 
-export async function replicateOperation(req) {
-	const response = { message: '' };
+export async function replicateOperation(req, options?: { onPeerResult?: (result: any) => void }) {
+	const response: { message: string; replicated?: any[] } = { message: '' };
 	if (req.replicated !== false) {
 		req.replicated = false; // don't send a replicated flag to the nodes we are sending to
 		logger.trace?.(
@@ -669,21 +682,35 @@ export async function replicateOperation(req) {
 			'to nodes',
 			server.nodes.map((node) => node.name)
 		);
-		const replicatedResults = await Promise.allSettled(
-			server.nodes.map((node) => {
-				// do all the nodes in parallel
-				return sendOperationToNode(node, req);
-			})
+		// Dispatch all peers in parallel. Each promise resolves to a normalized result
+		// (with `.node` attached), and the optional `onPeerResult` callback fires as
+		// each peer settles — letting callers surface per-peer progress in real time
+		// rather than waiting for the aggregate at the end.
+		const onPeerResult = options?.onPeerResult;
+		const perPeer = server.nodes.map((node) =>
+			sendOperationToNode(node, req)
+				.then((value: any) => {
+					const result: any = value && typeof value === 'object' ? value : { value };
+					result.node = node.name;
+					return result;
+				})
+				.catch((reason) => ({
+					status: 'failed',
+					reason: reason?.toString?.() ?? String(reason),
+					node: node.name,
+				}))
+				.then((result) => {
+					if (onPeerResult) {
+						try {
+							onPeerResult(result);
+						} catch (err) {
+							logger.warn?.('onPeerResult callback threw; ignoring', err);
+						}
+					}
+					return result;
+				})
 		);
-		// map the settled results to the response
-		response.replicated = replicatedResults.map((settledResult, index) => {
-			const result =
-				settledResult.status === 'rejected'
-					? { status: 'failed', reason: settledResult.reason.toString() }
-					: settledResult.value;
-			result.node = server.nodes[index]?.name; // add the node to the result so we know which node succeeded/failed
-			return result;
-		});
+		response.replicated = await Promise.all(perPeer);
 	}
 	return response;
 }

package/replication/setNode.ts

@@ -17,10 +17,13 @@ const { HTTP_STATUS_CODES } = hdbErrors;
 const validationSchema = Joi.object({
 	hostname: Joi.string(),
 	verify_tls: Joi.boolean(),
-	replicates: Joi.boolean(),
+	replicates: Joi.alternatives().try(Joi.boolean(), Joi.object()),
+	sendsTo: Joi.array(),
+	receivesFrom: Joi.array(),
 	subscriptions: Joi.array(),
 	revoked_certificates: Joi.array(),
 	shard: Joi.number(),
+	isLeader: Joi.boolean(),
 });
 
 /**
@@ -28,7 +31,7 @@ const validationSchema = Joi.object({
  * @param req
  */
 export async function setNode(req: any) {
-	if (req.node_name && !req.hostname) req.hostname = req.node_name;
+	if (!req.hostname) req.hostname = req.node_name || req.name;
 	if (req.verify_tls !== undefined) req.rejectUnauthorized = req.verify_tls;
 	let { url, hostname } = req;
 	if (!url) url = hostnameToUrl(hostname);
@@ -97,7 +100,7 @@ export async function setNode(req: any) {
 	}
 
 	// This is the record that will be added to the other nodes hdbNodes table
-	const targetAddNodeObj = {
+	const targetAddNodeObj: any = {
 		operation: 'add_node_back',
 		hostname: getThisNodeName(),
 		target_hostname: hostname,
@@ -112,6 +115,13 @@ export async function setNode(req: any) {
 		targetAddNodeObj.subscriptions = req.subscriptions.map(reverseSubscription);
 	} else targetAddNodeObj.subscriptions = null;
 
+	// Pass route exclusions to the peer with directions swapped: what LOCAL sends to PEER
+	// becomes what PEER receives from LOCAL, and vice versa.
+	if (req.sendsTo || req.receivesFrom) {
+		targetAddNodeObj.sendsTo = req.receivesFrom;
+		targetAddNodeObj.receivesFrom = req.sendsTo;
+	}
+
 	if (req.hasOwnProperty('subscribe') || req.hasOwnProperty('publish')) {
 		const rev = reverseSubscription(req);
 		targetAddNodeObj.subscribe = rev.subscribe;
@@ -125,7 +135,7 @@ export async function setNode(req: any) {
 
 	let targetNodeResponse: any;
 	let targetNodeResponseError: Error;
-	hdbLogger.debug('sending add_node_back operation', targetAddNodeObj);
+	hdbLogger.notify('sending add_node_back operation', targetAddNodeObj);
 	try {
 		targetNodeResponse = await sendOperationToNode({ url }, targetAddNodeObj, req);
 	} catch (err) {
@@ -167,9 +177,11 @@ export async function setNode(req: any) {
 		cert_auth = targetNodeResponse.signingCA;
 	}
 
-	const nodeRecord = { url, ca: targetNodeResponse?.usingCA };
+	const nodeRecord: any = { url, ca: targetNodeResponse?.usingCA };
 	if (req.hostname) nodeRecord.name = req.hostname;
 	if (req.subscriptions) nodeRecord.subscriptions = req.subscriptions;
+	else if (req.sendsTo || req.receivesFrom)
+		nodeRecord.replicates = { sends: true, sendsTo: req.sendsTo, receivesFrom: req.receivesFrom };
 	else nodeRecord.replicates = true;
 	if (req.start_time) {
 		nodeRecord.start_time = typeof req.start_time === 'string' ? new Date(req.start_time).getTime() : req.start_time;
@@ -178,9 +190,13 @@ export async function setNode(req: any) {
 	if (req.revoked_certificates) nodeRecord.revoked_certificates = req.revoked_certificates;
 	if (targetNodeResponse?.shard !== undefined) nodeRecord.shard = targetNodeResponse.shard;
 	else if (req.shard !== undefined) nodeRecord.shard = req.shard;
+	// isLeader is LOCAL-ONLY: it means "this peer is my leader; request a full-table copy from it".
+	// It must NOT be forwarded to the peer via targetAddNodeObj — doing so would cause the peer
+	// to treat this node as its leader and attempt to full-copy in the wrong direction.
+	if (req.isLeader !== undefined) nodeRecord.isLeader = req.isLeader;
 
 	if (nodeRecord.replicates) {
-		const thisNode = {
+		const thisNode: any = {
 			url: thisUrl,
 			ca: cert_auth,
 			replicates: true,
@@ -212,7 +228,7 @@ export async function setNode(req: any) {
 export async function addNodeBack(req) {
 	hdbLogger.trace('addNodeBack received request:', req);
 
-	let certs = {};
+	let certs: any = {};
 	// If the add_node req has a CSR attached, return the CA that was used to issue the CSR,
 	// else return whatever CA this node is using for replication
 	let originCa: string;
@@ -230,9 +246,12 @@ export async function addNodeBack(req) {
 		);
 	}
 
-	const nodeRecord = { url: req.url, ca: originCa };
+	const nodeRecord: any = { url: req.url, ca: originCa };
 	if (req.subscriptions) nodeRecord.subscriptions = req.subscriptions;
-	else {
+	else if (req.sendsTo || req.receivesFrom) {
+		nodeRecord.replicates = { sends: true, sendsTo: req.sendsTo, receivesFrom: req.receivesFrom };
+		nodeRecord.subscriptions = null;
+	} else {
 		nodeRecord.replicates = true;
 		nodeRecord.subscriptions = null;
 	}
@@ -243,7 +262,7 @@ export async function addNodeBack(req) {
 
 	const repCa = await getReplicationCertAuth();
 	if (nodeRecord.replicates) {
-		const thisNode = {
+		const thisNode: any = {
 			url: getThisNodeUrl(),
 			ca: repCa?.certificate,
 			replicates: true,