@harperfast/harper-pro 5.0.0-beta.1 → 5.0.0-beta.3

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/harper-pro",
-	"version": "5.0.0-beta.1",
+	"version": "5.0.0-beta.3",
 	"description": "Harper is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
 	"keywords": [
 		"database",
@@ -71,16 +71,16 @@
 		"cluster:hostname:latest": "pushd utility/dev && docker compose -f docker-compose.hostname.yml --project-directory ../.. pull && docker compose -f docker-compose.hostname.yml --project-directory ../.. up; popd"
 	},
 	"dependencies": {
-		"@aws-sdk/client-s3": "3.964.0",
+		"@aws-sdk/client-s3": "^3.1012.0",
 		"@aws-sdk/lib-storage": "3.964.0",
 		"@datadog/pprof": "^5.11.1",
 		"@endo/static-module-record": "^1.1.2",
-		"@fastify/autoload": "5.10.0",
-		"@fastify/compress": "~6.5.0",
-		"@fastify/cors": "~9.0.1",
-		"@fastify/static": "~7.0.4",
+		"@fastify/autoload": "^6.3.1",
+		"@fastify/compress": "^8.3.1",
+		"@fastify/cors": "^11.2.0",
+		"@fastify/static": "^9.0.0",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^0.1.11",
+		"@harperfast/rocksdb-js": "^0.1.12",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
 		"@turf/boolean-disjoint": "6.5.0",
@@ -91,6 +91,7 @@
 		"@turf/helpers": "6.5.0",
 		"@turf/length": "6.5.0",
 		"alasql": "4.6.6",
+		"amaro": "^1.1.8",
 		"argon2": "0.44.0",
 		"asn1js": "3.0.7",
 		"cbor-x": "1.6.4",
@@ -101,8 +102,8 @@
 		"dotenv": "^16.4.7",
 		"easy-ocsp": "1.2.2",
 		"fast-glob": "3.3.3",
-		"fastify": "~4.29.0",
-		"fastify-plugin": "~4.5.1",
+		"fastify": "^5.8.2",
+		"fastify-plugin": "^5.1.0",
 		"fs-extra": "11.3.3",
 		"graphql": "^16.10.0",
 		"graphql-http": "^1.22.4",
@@ -116,7 +117,7 @@
 		"jsonata": "1.8.7",
 		"jsonwebtoken": "9.0.3",
 		"lmdb": "3.5.2",
-		"lodash": "4.17.21",
+		"lodash": "^4.17.23",
 		"mathjs": "11.12.0",
 		"micromatch": "^4.0.8",
 		"minimist": "1.2.8",
@@ -144,8 +145,8 @@
 		"ses": "^1.14.0",
 		"stream-chain": "2.2.5",
 		"stream-json": "1.9.1",
-		"systeminformation": "5.27.11",
-		"tar-fs": "3.0.9",
+		"systeminformation": "^5.31.4",
+		"tar-fs": "^3.1.2",
 		"ulidx": "0.5.0",
 		"uuid": "11.1.0",
 		"validate.js": "0.13.1",
@@ -171,7 +172,7 @@
 		"globals": "^16.5.0",
 		"intercept-stdout": "0.1.2",
 		"mkcert": "^3.2.0",
-		"mocha": "^11.7.4",
+		"mocha": "^11.7.5",
 		"mqtt": "~4.3.8",
 		"oxlint": "^1.31.0",
 		"prettier": "~3.7.3",
@@ -182,7 +183,7 @@
 		"tsx": "^4.20.6",
 		"typescript": "^5.8.2",
 		"typescript-eslint": "^8.45.0",
-		"undici": "^7.16.0",
+		"undici": "^7.24.4",
 		"why-is-node-still-running": "^1.0.0"
 	}
 }

package/replication/knownNodes.ts

@@ -10,7 +10,7 @@ import { isMainThread } from 'worker_threads';
 import { ClientError } from '../core/utility/errors/hdbError.js';
 import env from '../core/utility/environment/environmentManager.js';
 import { CONFIG_PARAMS } from '../core/utility/hdbTerms.ts';
-import * as logger from '../core/utility/logging/logger.ts';
+import { logger } from '../core/utility/logging/logger.ts';
 
 let hdbNodeTable;
 server.nodes = [];
@@ -110,7 +110,7 @@ export function subscribeToNodeUpdates(listener: (node: any, id: string) => void
 	server.shards = new Map();
 
 	for (let entry of getHDBNodeTable().primaryStore.getRange({})) {
-		const { value: node } = entry;
+		const { value: node, key } = entry;
 		server.nodes.push(node);
 		if (node.shard != undefined) {
 			let nodesForShard = server.shards.get(node.shard);
@@ -119,6 +119,7 @@ export function subscribeToNodeUpdates(listener: (node: any, id: string) => void
 			}
 			nodesForShard.push(node);
 		}
+		listener(node, key);
 	}
 	logger.debug?.(
 		'Known nodes at startup',

package/replication/nodeIdMapping.ts

@@ -1,7 +1,7 @@
 /**
  * This module is responsible for managing the mapping of node/host names to node ids.
  */
-import * as logger from '../core/utility/logging/logger.ts';
+import { logger } from '../core/utility/logging/logger.ts';
 import { lastTimeInAuditStore } from './replicator.ts';
 import { getThisNodeName } from '../core/server/nodeName.ts';
 import { pack, unpack } from 'msgpackr';

package/replication/replicationConnection.ts

@@ -1,3 +1,4 @@
+import type { Logger } from '../core/utility/logging/logger.ts';
 import {
 	getDatabases,
 	databases,
@@ -51,7 +52,7 @@ import {
 } from '../core/resources/blob.ts';
 import { PassThrough } from 'node:stream';
 import { getLastVersion } from 'lmdb';
-const logger = forComponent('replication').conditional;
+const logger = forComponent('replication').conditional as Logger;
 
 // these are the codes we use for the different commands
 const SUBSCRIPTION_REQUEST = 129;
@@ -121,10 +122,9 @@ export async function createWebSocket(
 	if (url == null) {
 		throw new TypeError(`Invalid URL: Expected a string URL for node "${node_name}" but received ${url}`);
 	}
-
 	if (url.includes('wss://')) {
 		if (!secureContexts) {
-			const SNICallback = createTLSSelector('operations-api');
+			const SNICallback = createTLSSelector('replication');
 			const secureTarget = {
 				secureContexts: null,
 			};
@@ -133,10 +133,22 @@ export async function createWebSocket(
 		}
 		secureContext = secureContexts.get(node_name);
 		if (secureContext) {
-			logger.debug?.('Creating web socket for URL', url, 'with certificate named:', secureContext.name);
+			logger.debug?.(
+				'Creating web socket for URL',
+				url,
+				'with certificate named:',
+				secureContext.name,
+				'is_self_signed',
+				secureContext.is_self_signed
+			);
 		}
 		if (!secureContext && rejectUnauthorized !== false) {
-			throw new Error('Unable to find a valid certificate to use for replication to connect to ' + url);
+			throw new Error(
+				'Unable to find a valid certificate to use for replication to connect to ' +
+					url +
+					' available:' +
+					Array.from(secureContexts.keys())
+			);
 		}
 	}
 	const headers = {};

package/replication/replicator.ts

@@ -150,7 +150,11 @@ export function start(options) {
 				} else {
 					logger.error(
 						`Incoming client connection from ${request.peerCertificate?.subjectaltname ?? request.ip} did not have valid certificate, you may need turn on enableRootCAs in the config if you are using a publicly signed certificate, or add the CA to the server's trusted CAs`,
-						authorizationError
+						authorizationError,
+						'certificate issuer',
+						request.peerCertificate.issuer,
+						'did not match any available CAs',
+						Array.from(Array.from(wsServers[0].secureContexts.values())[0].options.availableCAs.keys())
 					);
 				}
 			}
@@ -265,6 +269,7 @@ export function start(options) {
 export function monitorNodeCAs(listener: () => void) {
 	let lastCaCount = 0;
 	subscribeToNodeUpdates((node) => {
+		logger.debug('Adding node CA', node?.ca?.slice(0, 60));
 		if (node?.ca) {
 			// we only care about nodes that have a CA
 			replicationCertificateAuthorities.add(node.ca);
@@ -639,7 +644,7 @@ export function lastTimeInAuditStore(auditStore: Database) {
 
 export async function replicateOperation(req) {
 	const response = { message: '' };
-	if (req.replicated) {
+	if (req.replicated !== false) {
 		req.replicated = false; // don't send a replicated flag to the nodes we are sending to
 		logger.trace?.(
 			'Replicating operation',

package/replication/setNode.ts

@@ -153,7 +153,7 @@ export async function setNode(req: any) {
 		if (targetNodeResponse.certificate) {
 			await setCertTable({
 				name: getThisNodeName(),
-				uses: ['https', 'operations', 'wss'],
+				uses: ['https', 'operations', 'wss', 'replication'],
 				certificate: targetNodeResponse.certificate,
 				private_key_name: rep?.options?.key_file,
 				is_authority: false,

package/security/certificate.ts

@@ -1,3 +1,4 @@
+import type { Logger } from '../core/utility/logging/logger.ts';
 import Joi from 'joi';
 import forge from 'node-forge';
 import { access, constants, readFile, writeFile, unlink } from 'node:fs/promises';
@@ -25,7 +26,7 @@ import { server } from '../core/server/Server.ts';
 import { replicateOperation } from '../replication/replicator.ts';
 
 const { forComponent } = harperLogger;
-const logger = forComponent('certificate').conditional;
+const logger = forComponent('certificate').conditional as Logger;
 const pki = forge.pki;
 const CERT_VALIDITY_DAYS = 3650;