@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/utility/environment/environmentManager.js

@@ -0,0 +1,208 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const os = require('os');
+const PropertiesReader = require('properties-reader');
+const log = require('../logging/harper_logger.js');
+const commonUtils = require('../common_utils.js');
+const hdbTerms = require('../hdbTerms.ts');
+const configUtils = require('../../config/configUtils.js');
+const { mkdirSync } = require('node:fs');
+
+const INIT_ERR = 'Error initializing environment manager';
+const BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH';
+
+let propFileExists = false;
+
+const installPropsToSave = {
+	[hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER]: true,
+	[hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY]: true,
+	[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]: true,
+	BOOT_PROPS_FILE_PATH: true,
+};
+let installProps = {};
+Object.assign(
+	exports,
+	(module.exports = {
+		BOOT_PROPS_FILE_PATH,
+		getHdbBasePath,
+		setHdbBasePath,
+		get,
+		initSync,
+		setProperty,
+		initTestEnvironment,
+	})
+);
+
+/**
+ * The base path of the HDB install is often referenced, but is referenced as a const variable at the top of many
+ * modules.  This is a problem during install, as the path may not yet be defined.  We offer a function to get the
+ * currently known base path here to help with this case.
+ */
+function getHdbBasePath() {
+	return installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY];
+}
+
+/**
+ * Sets the HDB base path in the install props object that this module maintains.
+ * This is mainly used by install during a stage where the config file doesn't exist.
+ * @param hdbPath
+ */
+function setHdbBasePath(hdbPath) {
+	installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = hdbPath;
+}
+
+/**
+ * Gets a Harper configuration value.
+ * @param propName
+ * @returns {*}
+ */
+function get(propName) {
+	const value = configUtils.getConfigValue(propName);
+	if (value === undefined) {
+		return installProps[propName];
+	}
+
+	return value;
+}
+
+/**
+ * Will update install props if provided prop is part of that object.
+ * Will also update the config object configUtils maintains.
+ * Note - this function will NOT update the config file. If you want to update the file
+ * use the updateConfigValue method in configUtils.
+ *
+ * This function should only be used by the installer and unit tests.
+ * @param propName
+ * @param value
+ */
+function setProperty(propName, value) {
+	if (installPropsToSave[propName]) {
+		installProps[propName] = value;
+	}
+
+	configUtils.updateConfigObject(propName, value);
+}
+
+/**
+ * Checks to see if the Harper boot props file exists.
+ * If it does, it grabs the install user and settings path for future reference.
+ * @returns {boolean}
+ */
+function doesPropFileExist() {
+	let bootPropPath;
+	try {
+		bootPropPath = commonUtils.getPropsFilePath();
+		fs.accessSync(bootPropPath, fs.constants.F_OK | fs.constants.R_OK);
+		propFileExists = true;
+		const hdbPropsFile = PropertiesReader(bootPropPath);
+
+		installProps[hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER] = hdbPropsFile.get(hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER);
+		installProps[hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY] = hdbPropsFile.get(
+			hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY
+		);
+		installProps[BOOT_PROPS_FILE_PATH] = bootPropPath;
+
+		return true;
+	} catch {
+		log.trace(`Environment manager found no properties file at ${bootPropPath}`);
+		return false;
+	}
+}
+
+/**
+ * Synchronously initializes our config environment.
+ * @param force
+ */
+function initSync(force = false) {
+	try {
+		if (propFileExists || doesPropFileExist() || commonUtils.noBootFile() || force) {
+			configUtils.initConfig(force);
+			const configHdbRoot = configUtils.getConfigValue(hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY);
+			// Only overwrite if we actually got a value from config
+			if (configHdbRoot !== undefined) {
+				installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = configHdbRoot;
+			}
+		}
+	} catch (err) {
+		log.error(INIT_ERR);
+		log.error(err);
+		console.error(err);
+		process.exit(1);
+	}
+}
+
+/**
+ * Initializes a test environment.
+ * Most of this is legacy code from before the yaml config refactor.
+ * @param testConfigObj
+ */
+function initTestEnvironment(testConfigObj = {}) {
+	try {
+		const {
+			keep_alive_timeout,
+			headers_timeout,
+			server_timeout,
+			https_enabled,
+			cors_enabled,
+			cors_accesslist,
+			local_studio_on,
+		} = testConfigObj;
+		// __dirname is dist/utility/environment when running tests, so go up 3 levels to reach project root
+		const propsPath = path.join(__dirname, '../../../', 'unitTests');
+		installProps[BOOT_PROPS_FILE_PATH] = path.join(propsPath, 'hdb_boot_properties.file');
+		try {
+			mkdirSync(path.join(propsPath, 'envDir'));
+		} catch {}
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY, path.join(propsPath, 'settings.test'));
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER, os.userInfo() ? os.userInfo().username : undefined);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_LEVEL_KEY, `debug`);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY, path.join(propsPath, 'envDir', 'log'));
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.LOG_DAILY_ROTATE_KEY, false);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY, path.join(propsPath, 'envDir'));
+		setProperty(hdbTerms.CONFIG_PARAMS.STORAGE_PATH, path.join(propsPath, 'envDir'));
+		if (https_enabled) {
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_SECUREPORT, get(hdbTerms.CONFIG_PARAMS.HTTP_PORT));
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_PORT, null);
+		}
+		setProperty(hdbTerms.CONFIG_PARAMS.CUSTOMFUNCTIONS_NETWORK_HTTPS, Boolean(https_enabled));
+		setProperty(hdbTerms.CONFIG_PARAMS.HTTP_PORT, 9926);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.SERVER_PORT_KEY, 9925);
+		setProperty(hdbTerms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT, 9925);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.CORS_ENABLED_KEY, commonUtils.isEmpty(cors_enabled) ? false : cors_enabled);
+		setProperty(hdbTerms.CONFIG_PARAMS.HTTP_CORS, commonUtils.isEmpty(cors_enabled) ? false : cors_enabled);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.MAX_CUSTOM_FUNCTION_PROCESSES, 2);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.MAX_HDB_PROCESSES, 4);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.CUSTOM_FUNCTIONS_PORT_KEY, 9926);
+		setProperty(hdbTerms.HDB_SETTINGS_NAMES.CUSTOM_FUNCTIONS_ENABLED_KEY, true);
+		setProperty(
+			hdbTerms.HDB_SETTINGS_NAMES.CUSTOM_FUNCTIONS_DIRECTORY_KEY,
+			path.join(propsPath, 'server/fastifyRoutes/custom_functions')
+		);
+		setProperty(
+			hdbTerms.HDB_SETTINGS_NAMES.LOCAL_STUDIO_ON,
+			commonUtils.isEmpty(local_studio_on) ? false : local_studio_on
+		);
+		if (cors_accesslist) {
+			setProperty('CORS_ACCESSLIST', cors_accesslist);
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_CORSACCESSLIST, cors_accesslist);
+		}
+		if (server_timeout) {
+			setProperty(hdbTerms.HDB_SETTINGS_NAMES.SERVER_TIMEOUT_KEY, server_timeout);
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_TIMEOUT, server_timeout);
+		}
+		if (keep_alive_timeout) {
+			setProperty(hdbTerms.HDB_SETTINGS_NAMES.SERVER_KEEP_ALIVE_TIMEOUT_KEY, keep_alive_timeout);
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_KEEPALIVETIMEOUT, keep_alive_timeout);
+		}
+		if (headers_timeout) {
+			setProperty(hdbTerms.HDB_SETTINGS_NAMES.SERVER_HEADERS_TIMEOUT_KEY, headers_timeout);
+			setProperty(hdbTerms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT, headers_timeout);
+		}
+	} catch (err) {
+		let msg = `Error reading in HDB environment variables from path ${BOOT_PROPS_FILE_PATH}.  Please check your boot props and settings files`;
+		log.fatal(msg);
+		log.error(err);
+	}
+}

package/utility/environment/systemInformation.js

@@ -0,0 +1,355 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const si = require('systeminformation');
+const log = require('../logging/harper_logger.js');
+const terms = require('../hdbTerms.ts');
+const lmdbGetTableSize = require('../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js');
+const schemaDescribe = require('../../dataLayer/schemaDescribe.js');
+const { getThreadInfo } = require('../../server/threads/manageThreads.js');
+const env = require('./environmentManager.js');
+env.initSync();
+const { databases } = require('../../resources/databases.ts');
+
+//this will hold the system_information which is static to improve performance
+let systemInformationCache = undefined;
+
+class SystemInformationRequest {
+	constructor(attributes) {
+		this.operator = terms.OPERATIONS_ENUM.SYSTEM_INFORMATION;
+		this.attributes = attributes;
+	}
+}
+
+class SystemInformationResponse {
+	constructor(system, time, cpu, memory, disk, network, harperdbProcesses) {
+		this.system = system;
+		this.time = time;
+		this.cpu = cpu;
+		this.memory = memory;
+		this.disk = disk;
+		this.network = network;
+		this.harperdb_processes = harperdbProcesses;
+	}
+}
+
+module.exports = {
+	getHDBProcessInfo,
+	getNetworkInfo,
+	getDiskInfo,
+	getMemoryInfo,
+	getCPUInfo,
+	getTimeInfo,
+	getSystemInformation,
+	systemInformation,
+	getTableSize,
+	getMetrics,
+	SystemInformationRequest,
+};
+
+/**
+ * executes the time function to return the time info for the system
+ * @returns {si.Systeminformation.TimeData}
+ */
+function getTimeInfo() {
+	return si.time();
+}
+
+/**
+ * executes cpu related functions
+ * @returns {Promise<{}|Pick<si.Systeminformation.CpuData, "manufacturer" | "brand" | "vendor" | "speed" | "cores" | "physicalCores" | "processors">>}
+ */
+async function getCPUInfo() {
+	try {
+		// eslint-disable-next-line no-unused-vars
+		let { family, model, stepping, revision, voltage, speedmin, speedmax, governor, socket, cache, ...cpuInfo } =
+			await si.cpu();
+		cpuInfo.cpu_speed = await si.cpuCurrentSpeed();
+
+		let {
+			// eslint-disable-next-line no-unused-vars
+			rawCurrentload,
+			cpus,
+			...cpuCurrentLoad
+		} = await si.currentLoad();
+		cpuCurrentLoad.cpus = [];
+		cpus.forEach((cpuData) => {
+			// eslint-disable-next-line no-unused-vars
+			let { rawLoad, rawLoadIdle, rawLoadIrq, rawLoadNice, rawLoadSystem, rawLoadUser, ...cpuLoad } = cpuData;
+			cpuCurrentLoad.cpus.push(cpuLoad);
+		});
+		cpuInfo.current_load = cpuCurrentLoad;
+		return cpuInfo;
+	} catch (e) {
+		log.error(`error in getCPUInfo: ${e}`);
+		return {};
+	}
+}
+
+/**
+ * fetches information related memory
+ * @returns {Promise<{}|Pick<si.Systeminformation.MemData, "total" | "free" | "used" | "active" | "available" | "swaptotal" | "swapused" | "swapfree">>}
+ */
+async function getMemoryInfo() {
+	try {
+		// eslint-disable-next-line no-unused-vars
+		let { buffers, cached, slab, buffcache, ...memInfo } = await si.mem();
+		return Object.assign(memInfo, process.memoryUsage());
+	} catch (e) {
+		log.error(`error in getMemoryInfo: ${e}`);
+		return {};
+	}
+}
+
+/**
+ * searches for & returns the processes for hdb core
+ * @returns {Promise<{core: []}>}
+ */
+async function getHDBProcessInfo() {
+	let harperdbProcesses = {
+		core: [],
+	};
+	try {
+		let processes = await si.processes();
+
+		let hdbPid;
+		try {
+			hdbPid = Number.parseInt(
+				await fs.readFile(path.join(env.get(terms.CONFIG_PARAMS.ROOTPATH), terms.HDB_PID_FILE), 'utf8')
+			);
+		} catch (err) {
+			if (err.code === terms.NODE_ERROR_CODES.ENOENT) {
+				log.warn(
+					`Unable to locate 'hdb.pid' file, try stopping and starting Harper. This could be because Harper is not running.`
+				);
+			} else {
+				throw err;
+			}
+		}
+
+		processes.list.forEach((p) => {
+			if (p.pid === hdbPid) {
+				harperdbProcesses.core.push(p);
+			}
+		});
+
+		for (const hdbP of harperdbProcesses.core) {
+			for (const p of processes.list) {
+				if (p.pid === hdbP.parentPid && (p.name === 'PM2' || p.command === 'PM2')) {
+					hdbP.parent = 'PM2';
+				}
+			}
+		}
+
+		return harperdbProcesses;
+	} catch (e) {
+		log.error(`error in getHDBProcessInfo: ${e}`);
+		return harperdbProcesses;
+	}
+}
+
+/**
+ * gets disk related info & stats
+ * @returns {Promise<{}>}
+ */
+async function getDiskInfo() {
+	let disk = {};
+	try {
+		if (!env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_DISK)) return disk;
+		// eslint-disable-next-line no-unused-vars
+		let { rIO_sec, wIO_sec, tIO_sec, ms, ...diskIo } = await si.disksIO();
+		disk.io = diskIo;
+
+		// eslint-disable-next-line no-unused-vars
+		let { rxSec, txSec, wxSec, ...fsStats } = await si.fsStats();
+		disk.read_write = fsStats;
+
+		disk.size = await si.fsSize();
+
+		return disk;
+	} catch (e) {
+		log.error(`error in getDiskInfo: ${e}`);
+		return disk;
+	}
+}
+
+/**
+ * gets networking & connection information & stats
+ * @returns {Promise<{interfaces: [], default_interface: null, stats: [], latency: {}, connections: []}>}
+ */
+async function getNetworkInfo() {
+	let network = {
+		default_interface: null,
+		latency: {},
+		interfaces: [],
+		stats: [],
+		connections: [],
+	};
+	try {
+		if (!env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_NETWORK)) return network;
+		network.default_interface = await si.networkInterfaceDefault();
+
+		network.latency = await si.inetChecksite('google.com');
+
+		let nInterfaces = await si.networkInterfaces();
+		nInterfaces.forEach((_interface) => {
+			// eslint-disable-next-line no-unused-vars
+			let { internal, virtual, mtu, dhcp, dnsSuffix, ieee8021xAuth, ieee8021xState, carrierChanges, ...networkInt } =
+				_interface;
+			network.interfaces.push(networkInt);
+		});
+
+		let stats = await si.networkStats();
+		stats.forEach((nStat) => {
+			// eslint-disable-next-line no-unused-vars
+			let { rxSec, txSec, ms, ...stat } = nStat;
+			network.stats.push(stat);
+		});
+
+		return network;
+	} catch (e) {
+		log.error(`error in getNetworkInfo: ${e}`);
+		return network;
+	}
+}
+
+/**
+ * gets system information
+ * @returns {Promise<Pick<si.Systeminformation.OsData, "platform" | "distro" | "release" | "codename" | "kernel" | "arch" | "hostname">|{}>}
+ */
+async function getSystemInformation() {
+	if (systemInformationCache !== undefined) {
+		return systemInformationCache;
+	}
+
+	let system_info = {};
+	try {
+		// eslint-disable-next-line no-unused-vars
+		let { codepage, logofile, serial, build, servicepack, uefi, ...sysInfo } = await si.osInfo();
+		system_info = sysInfo;
+		let versions = await si.versions('node, npm');
+		system_info.node_version = versions.node;
+		system_info.npm_version = versions.npm;
+
+		systemInformationCache = system_info;
+		return systemInformationCache;
+	} catch (e) {
+		log.error(`error in getSystemInformation: ${e}`);
+		return system_info;
+	}
+}
+
+async function getTableSize() {
+	//get details for all tables
+	let tableSizes = [];
+	let allSchemas = await schemaDescribe.describeAll();
+	for (const tables of Object.values(allSchemas)) {
+		for (const tableData of Object.values(tables)) {
+			tableSizes.push(await lmdbGetTableSize(tableData));
+		}
+	}
+
+	return tableSizes;
+}
+async function getMetrics() {
+	let schemaStats = {};
+	for (let schemaName in databases) {
+		let dbStats = (schemaStats[schemaName] = {});
+		let tableStats = (dbStats.tables = {});
+		for (let tableName in databases[schemaName]) {
+			try {
+				let table = databases[schemaName][tableName];
+				if (!dbStats.readers) {
+					Object.assign(dbStats, table.primaryStore.rootStore.getStats());
+					delete dbStats.root;
+					dbStats.readers = table.primaryStore.rootStore
+						.readerList()
+						.split(/\n\s+/)
+						.slice(1)
+						.map((line) => {
+							const [pid, thread, txnid] = line.trim().split(' ');
+							return { pid, thread, txnid };
+						});
+					if (table.auditStore) {
+						const { treeDepth, treeBranchPageCount, treeLeafPageCount, entryCount, overflowPages } =
+							table.auditStore.getStats();
+						dbStats.audit = { treeDepth, treeBranchPageCount, treeLeafPageCount, entryCount, overflowPages };
+					}
+				}
+				let tableFullStats = table.primaryStore.getStats();
+				let tablePrunedStats = {};
+				for (let storeKey of ['treeDepth', 'treeBranchPageCount', 'treeLeafPageCount', 'entryCount', 'overflowPages']) {
+					tablePrunedStats[storeKey] = tableFullStats[storeKey];
+				}
+				tableStats[tableName] = tablePrunedStats;
+			} catch (error) {
+				// if a schema no longer exists, don't want to throw an error
+				log.notify(`Error getting stats for table ${tableName}: ${error}`);
+			}
+		}
+	}
+	return schemaStats;
+}
+
+/**
+ *
+ * @param {SystemInformationRequest} systemInfoReq
+ * @returns {Promise<SystemInformationResponse>}
+ */
+async function systemInformation(systemInfoReq) {
+	let response = new SystemInformationResponse();
+	if (!Array.isArray(systemInfoReq.attributes) || systemInfoReq.attributes.length === 0) {
+		response.system = await getSystemInformation();
+		response.time = getTimeInfo();
+		response.cpu = await getCPUInfo();
+		response.memory = await getMemoryInfo();
+		response.disk = await getDiskInfo();
+		response.network = await getNetworkInfo();
+		response.harperdb_processes = await getHDBProcessInfo();
+		response.table_size = await getTableSize();
+		response.metrics = await getMetrics();
+		response.threads = await getThreadInfo();
+		return response;
+	}
+
+	for (let attr of systemInfoReq.attributes) {
+		switch (attr) {
+			case 'system':
+				response.system = await getSystemInformation();
+				break;
+			case 'time':
+				response.time = getTimeInfo();
+				break;
+			case 'cpu':
+				response.cpu = await getCPUInfo();
+				break;
+			case 'memory':
+				response.memory = await getMemoryInfo();
+				break;
+			case 'disk':
+				response.disk = await getDiskInfo();
+				break;
+			case 'network':
+				response.network = await getNetworkInfo();
+				break;
+			case 'harperdb_processes':
+				response.harperdb_processes = await getHDBProcessInfo();
+				break;
+			case 'table_size':
+				response.table_size = await getTableSize();
+				break;
+			case 'database_metrics':
+			case 'metrics':
+				response.metrics = await getMetrics();
+				break;
+			case 'threads':
+				response.threads = await getThreadInfo();
+				break;
+			default:
+				break;
+		}
+	}
+
+	return response;
+}