@harperfast/harper 5.0.7 → 5.0.8

package/utility/operation_authorization.js

@@ -26,7 +26,7 @@ const restart = require('../bin/restart.js');
 const terms = require('./hdbTerms.ts');
 const { expandOperationsPerms } = require('./operationPermissions.ts');
 const permsTranslator = require('../security/permissionsTranslator.js');
-const systemInformation = require('../utility/environment/systemInformation.js');
+const { systemInformation } = require('../utility/environment/systemInformation.ts');
 const tokenAuthentication = require('../security/tokenAuthentication.ts');
 const auth = require('../security/auth.ts');
 const configUtils = require('../config/configUtils.js');
@@ -157,10 +157,7 @@ requiredPermissions.set(restart.restartService.name, new permission(true, []));
 requiredPermissions.set(readAuditLog.name, new permission(true, [], terms.OPERATIONS_ENUM.READ_AUDIT_LOG));
 requiredPermissions.set(getBackup.name, new permission(true, [READ_PERM]));
 requiredPermissions.set(schema.cleanupOrphanBlobs.name, new permission(true, []));
-requiredPermissions.set(
-	systemInformation.systemInformation.name,
-	new permission(true, [], terms.OPERATIONS_ENUM.SYSTEM_INFORMATION)
-);
+requiredPermissions.set(systemInformation.name, new permission(true, [], terms.OPERATIONS_ENUM.SYSTEM_INFORMATION));
 requiredPermissions.set(
 	configUtils.getConfiguration.name,
 	new permission(true, [], terms.OPERATIONS_ENUM.GET_CONFIGURATION)

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js

@@ -1,25 +0,0 @@
-'use strict';
-
-/**
- * represents the table size entry for a table
- */
-class TableSizeObject {
-	/**
-	 * @param {String} schema
-	 * @param {String} table
-	 * @param {Number} tableSize - data size of the table in bytes
-	 * @param {Number} recordCount - number of entries in the table
-	 * @param {Number} transactionLogSize - number of entries in the transaction log
-	 * @param {Number} transactionLogRecordCount - data size of the transaction log in bytes
-	 */
-	constructor(schema, table, tableSize = 0, recordCount = 0, transactionLogSize = 0, transactionLogRecordCount = 0) {
-		this.schema = schema;
-		this.table = table;
-		this.table_size = tableSize;
-		this.record_count = recordCount;
-		this.transaction_log_size = transactionLogSize;
-		this.transaction_log_record_count = transactionLogRecordCount;
-	}
-}
-
-module.exports = TableSizeObject;

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js

@@ -1,34 +0,0 @@
-'use strict';
-
-const TableSizeObject = require('./TableSizeObject.js');
-const log = require('../../../../utility/logging/harper_logger.js');
-const { getDatabases } = require('../../../../resources/databases.ts');
-
-module.exports = lmdbGetTableSize;
-
-/**
- * calculates the number of entries & data size in bytes for a table & its transaction log
- * @param tableObject
- * @returns {Promise<TableSizeObject>}
- */
-async function lmdbGetTableSize(tableObject) {
-	let tableStats = new TableSizeObject();
-	try {
-		//get the table record count
-		let table = getDatabases()[tableObject.schema]?.[tableObject.name];
-
-		let dbiStat = table.primaryStore.getStats();
-
-		//get the txn log record count
-		let txnDbiStat = table.auditStore?.getStats();
-
-		tableStats.schema = tableObject.schema;
-		tableStats.table = tableObject.name;
-		tableStats.record_count = dbiStat.entryCount;
-		tableStats.transaction_log_record_count = txnDbiStat.entryCount;
-	} catch (e) {
-		log.warn(`unable to stat table dbi due to ${e}`);
-	}
-
-	return tableStats;
-}

package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.d.ts

@@ -1,21 +0,0 @@
-export = TableSizeObject;
-/**
- * represents the table size entry for a table
- */
-declare class TableSizeObject {
-    /**
-     * @param {String} schema
-     * @param {String} table
-     * @param {Number} tableSize - data size of the table in bytes
-     * @param {Number} recordCount - number of entries in the table
-     * @param {Number} transactionLogSize - number of entries in the transaction log
-     * @param {Number} transactionLogRecordCount - data size of the transaction log in bytes
-     */
-    constructor(schema: string, table: string, tableSize?: number, recordCount?: number, transactionLogSize?: number, transactionLogRecordCount?: number);
-    schema: string;
-    table: string;
-    table_size: number;
-    record_count: number;
-    transaction_log_size: number;
-    transaction_log_record_count: number;
-}

package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js

@@ -1,24 +0,0 @@
-'use strict';
-/**
- * represents the table size entry for a table
- */
-class TableSizeObject {
-    /**
-     * @param {String} schema
-     * @param {String} table
-     * @param {Number} tableSize - data size of the table in bytes
-     * @param {Number} recordCount - number of entries in the table
-     * @param {Number} transactionLogSize - number of entries in the transaction log
-     * @param {Number} transactionLogRecordCount - data size of the transaction log in bytes
-     */
-    constructor(schema, table, tableSize = 0, recordCount = 0, transactionLogSize = 0, transactionLogRecordCount = 0) {
-        this.schema = schema;
-        this.table = table;
-        this.table_size = tableSize;
-        this.record_count = recordCount;
-        this.transaction_log_size = transactionLogSize;
-        this.transaction_log_record_count = transactionLogRecordCount;
-    }
-}
-module.exports = TableSizeObject;
-//# sourceMappingURL=TableSizeObject.js.map

package/dist/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TableSizeObject.js","sourceRoot":"","sources":["../../../../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb;;GAEG;AACH,MAAM,eAAe;IACpB;;;;;;;OAOG;IACH,YAAY,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,kBAAkB,GAAG,CAAC,EAAE,yBAAyB,GAAG,CAAC;QAC/G,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;QAC/C,IAAI,CAAC,4BAA4B,GAAG,yBAAyB,CAAC;IAC/D,CAAC;CACD;AAED,MAAM,CAAC,OAAO,GAAG,eAAe,CAAC"}

package/utility/environment/systemInformation.js

@@ -1,355 +0,0 @@
-'use strict';
-
-const fs = require('fs-extra');
-const path = require('path');
-const si = require('systeminformation');
-const log = require('../logging/harper_logger.js');
-const terms = require('../hdbTerms.ts');
-const lmdbGetTableSize = require('../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js');
-const schemaDescribe = require('../../dataLayer/schemaDescribe.js');
-const { getThreadInfo } = require('../../server/threads/manageThreads.js');
-const env = require('./environmentManager.js');
-env.initSync();
-const { databases } = require('../../resources/databases.ts');
-
-//this will hold the system_information which is static to improve performance
-let systemInformationCache = undefined;
-
-class SystemInformationRequest {
-	constructor(attributes) {
-		this.operator = terms.OPERATIONS_ENUM.SYSTEM_INFORMATION;
-		this.attributes = attributes;
-	}
-}
-
-class SystemInformationResponse {
-	constructor(system, time, cpu, memory, disk, network, harperdbProcesses) {
-		this.system = system;
-		this.time = time;
-		this.cpu = cpu;
-		this.memory = memory;
-		this.disk = disk;
-		this.network = network;
-		this.harperdb_processes = harperdbProcesses;
-	}
-}
-
-module.exports = {
-	getHDBProcessInfo,
-	getNetworkInfo,
-	getDiskInfo,
-	getMemoryInfo,
-	getCPUInfo,
-	getTimeInfo,
-	getSystemInformation,
-	systemInformation,
-	getTableSize,
-	getMetrics,
-	SystemInformationRequest,
-};
-
-/**
- * executes the time function to return the time info for the system
- * @returns {si.Systeminformation.TimeData}
- */
-function getTimeInfo() {
-	return si.time();
-}
-
-/**
- * executes cpu related functions
- * @returns {Promise<{}|Pick<si.Systeminformation.CpuData, "manufacturer" | "brand" | "vendor" | "speed" | "cores" | "physicalCores" | "processors">>}
- */
-async function getCPUInfo() {
-	try {
-		// eslint-disable-next-line no-unused-vars
-		let { family, model, stepping, revision, voltage, speedmin, speedmax, governor, socket, cache, ...cpuInfo } =
-			await si.cpu();
-		cpuInfo.cpu_speed = await si.cpuCurrentSpeed();
-
-		let {
-			// eslint-disable-next-line no-unused-vars
-			rawCurrentload,
-			cpus,
-			...cpuCurrentLoad
-		} = await si.currentLoad();
-		cpuCurrentLoad.cpus = [];
-		cpus.forEach((cpuData) => {
-			// eslint-disable-next-line no-unused-vars
-			let { rawLoad, rawLoadIdle, rawLoadIrq, rawLoadNice, rawLoadSystem, rawLoadUser, ...cpuLoad } = cpuData;
-			cpuCurrentLoad.cpus.push(cpuLoad);
-		});
-		cpuInfo.current_load = cpuCurrentLoad;
-		return cpuInfo;
-	} catch (e) {
-		log.error(`error in getCPUInfo: ${e}`);
-		return {};
-	}
-}
-
-/**
- * fetches information related memory
- * @returns {Promise<{}|Pick<si.Systeminformation.MemData, "total" | "free" | "used" | "active" | "available" | "swaptotal" | "swapused" | "swapfree">>}
- */
-async function getMemoryInfo() {
-	try {
-		// eslint-disable-next-line no-unused-vars
-		let { buffers, cached, slab, buffcache, ...memInfo } = await si.mem();
-		return Object.assign(memInfo, process.memoryUsage());
-	} catch (e) {
-		log.error(`error in getMemoryInfo: ${e}`);
-		return {};
-	}
-}
-
-/**
- * searches for & returns the processes for hdb core
- * @returns {Promise<{core: []}>}
- */
-async function getHDBProcessInfo() {
-	let harperdbProcesses = {
-		core: [],
-	};
-	try {
-		let processes = await si.processes();
-
-		let hdbPid;
-		try {
-			hdbPid = Number.parseInt(
-				await fs.readFile(path.join(env.get(terms.CONFIG_PARAMS.ROOTPATH), terms.HDB_PID_FILE), 'utf8')
-			);
-		} catch (err) {
-			if (err.code === terms.NODE_ERROR_CODES.ENOENT) {
-				log.warn(
-					`Unable to locate 'hdb.pid' file, try stopping and starting Harper. This could be because Harper is not running.`
-				);
-			} else {
-				throw err;
-			}
-		}
-
-		processes.list.forEach((p) => {
-			if (p.pid === hdbPid) {
-				harperdbProcesses.core.push(p);
-			}
-		});
-
-		for (const hdbP of harperdbProcesses.core) {
-			for (const p of processes.list) {
-				if (p.pid === hdbP.parentPid && (p.name === 'PM2' || p.command === 'PM2')) {
-					hdbP.parent = 'PM2';
-				}
-			}
-		}
-
-		return harperdbProcesses;
-	} catch (e) {
-		log.error(`error in getHDBProcessInfo: ${e}`);
-		return harperdbProcesses;
-	}
-}
-
-/**
- * gets disk related info & stats
- * @returns {Promise<{}>}
- */
-async function getDiskInfo() {
-	let disk = {};
-	try {
-		if (!env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_DISK)) return disk;
-		// eslint-disable-next-line no-unused-vars
-		let { rIO_sec, wIO_sec, tIO_sec, ms, ...diskIo } = await si.disksIO();
-		disk.io = diskIo;
-
-		// eslint-disable-next-line no-unused-vars
-		let { rxSec, txSec, wxSec, ...fsStats } = await si.fsStats();
-		disk.read_write = fsStats;
-
-		disk.size = await si.fsSize();
-
-		return disk;
-	} catch (e) {
-		log.error(`error in getDiskInfo: ${e}`);
-		return disk;
-	}
-}
-
-/**
- * gets networking & connection information & stats
- * @returns {Promise<{interfaces: [], default_interface: null, stats: [], latency: {}, connections: []}>}
- */
-async function getNetworkInfo() {
-	let network = {
-		default_interface: null,
-		latency: {},
-		interfaces: [],
-		stats: [],
-		connections: [],
-	};
-	try {
-		if (!env.get(terms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_NETWORK)) return network;
-		network.default_interface = await si.networkInterfaceDefault();
-
-		network.latency = await si.inetChecksite('google.com');
-
-		let nInterfaces = await si.networkInterfaces();
-		nInterfaces.forEach((_interface) => {
-			// eslint-disable-next-line no-unused-vars
-			let { internal, virtual, mtu, dhcp, dnsSuffix, ieee8021xAuth, ieee8021xState, carrierChanges, ...networkInt } =
-				_interface;
-			network.interfaces.push(networkInt);
-		});
-
-		let stats = await si.networkStats();
-		stats.forEach((nStat) => {
-			// eslint-disable-next-line no-unused-vars
-			let { rxSec, txSec, ms, ...stat } = nStat;
-			network.stats.push(stat);
-		});
-
-		return network;
-	} catch (e) {
-		log.error(`error in getNetworkInfo: ${e}`);
-		return network;
-	}
-}
-
-/**
- * gets system information
- * @returns {Promise<Pick<si.Systeminformation.OsData, "platform" | "distro" | "release" | "codename" | "kernel" | "arch" | "hostname">|{}>}
- */
-async function getSystemInformation() {
-	if (systemInformationCache !== undefined) {
-		return systemInformationCache;
-	}
-
-	let system_info = {};
-	try {
-		// eslint-disable-next-line no-unused-vars
-		let { codepage, logofile, serial, build, servicepack, uefi, ...sysInfo } = await si.osInfo();
-		system_info = sysInfo;
-		let versions = await si.versions('node, npm');
-		system_info.node_version = versions.node;
-		system_info.npm_version = versions.npm;
-
-		systemInformationCache = system_info;
-		return systemInformationCache;
-	} catch (e) {
-		log.error(`error in getSystemInformation: ${e}`);
-		return system_info;
-	}
-}
-
-async function getTableSize() {
-	//get details for all tables
-	let tableSizes = [];
-	let allSchemas = await schemaDescribe.describeAll();
-	for (const tables of Object.values(allSchemas)) {
-		for (const tableData of Object.values(tables)) {
-			tableSizes.push(await lmdbGetTableSize(tableData));
-		}
-	}
-
-	return tableSizes;
-}
-async function getMetrics() {
-	let schemaStats = {};
-	for (let schemaName in databases) {
-		let dbStats = (schemaStats[schemaName] = {});
-		let tableStats = (dbStats.tables = {});
-		for (let tableName in databases[schemaName]) {
-			try {
-				let table = databases[schemaName][tableName];
-				if (!dbStats.readers) {
-					Object.assign(dbStats, table.primaryStore.rootStore.getStats());
-					delete dbStats.root;
-					dbStats.readers = table.primaryStore.rootStore
-						.readerList()
-						.split(/\n\s+/)
-						.slice(1)
-						.map((line) => {
-							const [pid, thread, txnid] = line.trim().split(' ');
-							return { pid, thread, txnid };
-						});
-					if (table.auditStore) {
-						const { treeDepth, treeBranchPageCount, treeLeafPageCount, entryCount, overflowPages } =
-							table.auditStore.getStats();
-						dbStats.audit = { treeDepth, treeBranchPageCount, treeLeafPageCount, entryCount, overflowPages };
-					}
-				}
-				let tableFullStats = table.primaryStore.getStats();
-				let tablePrunedStats = {};
-				for (let storeKey of ['treeDepth', 'treeBranchPageCount', 'treeLeafPageCount', 'entryCount', 'overflowPages']) {
-					tablePrunedStats[storeKey] = tableFullStats[storeKey];
-				}
-				tableStats[tableName] = tablePrunedStats;
-			} catch (error) {
-				// if a schema no longer exists, don't want to throw an error
-				log.notify(`Error getting stats for table ${tableName}: ${error}`);
-			}
-		}
-	}
-	return schemaStats;
-}
-
-/**
- *
- * @param {SystemInformationRequest} systemInfoReq
- * @returns {Promise<SystemInformationResponse>}
- */
-async function systemInformation(systemInfoReq) {
-	let response = new SystemInformationResponse();
-	if (!Array.isArray(systemInfoReq.attributes) || systemInfoReq.attributes.length === 0) {
-		response.system = await getSystemInformation();
-		response.time = getTimeInfo();
-		response.cpu = await getCPUInfo();
-		response.memory = await getMemoryInfo();
-		response.disk = await getDiskInfo();
-		response.network = await getNetworkInfo();
-		response.harperdb_processes = await getHDBProcessInfo();
-		response.table_size = await getTableSize();
-		response.metrics = await getMetrics();
-		response.threads = await getThreadInfo();
-		return response;
-	}
-
-	for (let attr of systemInfoReq.attributes) {
-		switch (attr) {
-			case 'system':
-				response.system = await getSystemInformation();
-				break;
-			case 'time':
-				response.time = getTimeInfo();
-				break;
-			case 'cpu':
-				response.cpu = await getCPUInfo();
-				break;
-			case 'memory':
-				response.memory = await getMemoryInfo();
-				break;
-			case 'disk':
-				response.disk = await getDiskInfo();
-				break;
-			case 'network':
-				response.network = await getNetworkInfo();
-				break;
-			case 'harperdb_processes':
-				response.harperdb_processes = await getHDBProcessInfo();
-				break;
-			case 'table_size':
-				response.table_size = await getTableSize();
-				break;
-			case 'database_metrics':
-			case 'metrics':
-				response.metrics = await getMetrics();
-				break;
-			case 'threads':
-				response.threads = await getThreadInfo();
-				break;
-			default:
-				break;
-		}
-	}
-
-	return response;
-}