@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/utility/logging/logRotator.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const { promises: fsProm, createReadStream, createWriteStream } = require('fs');
+const { createGzip } = require('zlib');
+const { promisify } = require('util');
+const { pipeline } = require('stream');
+const pipe = promisify(pipeline);
+const path = require('path');
+const envMgr = require('../environment/environmentManager.js');
+envMgr.initSync();
+const hdbLogger = require('./harper_logger.js');
+const { CONFIG_PARAMS } = require('../hdbTerms.ts');
+const { convertToMS } = require('../common_utils.js');
+const { onStorageReclamation } = require('../../server/storageReclamation.ts');
+
+// Interval in ms to check log file and decide if it should be rotated.
+const LOG_AUDIT_INTERVAL = 60000;
+const INT_SIZE_UNDEFINED_MSG =
+	"'interval' and 'maxSize' are both undefined, to enable logging rotation at least one of these values must be defined in harperdb-config.yaml";
+const PATH_UNDEFINED_MSG =
+	"'logging.rotation.path' is undefined, to enable logging rotation set this value in harperdb-config.yaml";
+
+let lastRotationTime;
+let setIntervalId;
+
+module.exports = logRotator;
+
+/**
+ * Rotates hdb.log using an interval and/or maxSize param to determine if log should be rotated.
+ * Uses an unref setInterval to periodically check time passed since rotation and size of log file.
+ * If log file is within the values set in config, log file will be renamed/moved and a new empty hdb.log created.
+ * @returns LogRotator
+ */
+function logRotator({ logger, maxSize, interval, retention, enabled, path: rotatedLogDir, auditInterval }) {
+	if (enabled === false) return;
+	let reclamationPriority = 0;
+	onStorageReclamation(
+		logger.path,
+		(priority) => {
+			reclamationPriority = priority;
+		},
+		true
+	);
+
+	if (!maxSize && !interval) {
+		throw new Error(INT_SIZE_UNDEFINED_MSG);
+	}
+
+	if (!rotatedLogDir) {
+		throw new Error(PATH_UNDEFINED_MSG);
+	}
+
+	// Convert maxSize param to bytes.
+	let maxBytes;
+	if (maxSize) {
+		const unit = maxSize.slice(-1);
+		const size = maxSize.slice(0, -1);
+		if (unit === 'G') maxBytes = size * 1000000000;
+		else if (unit === 'M') maxBytes = size * 1000000;
+		else maxBytes = size * 1000;
+	}
+
+	// Convert interval param to ms.
+	let maxInterval;
+	if (interval) {
+		maxInterval = convertToMS(interval);
+	}
+
+	let lastRotatedLogPath;
+	// convert date.now to minutes
+	lastRotationTime = Date.now();
+	hdbLogger.trace('Log rotate enabled, maxSize:', maxSize, 'interval:', interval);
+	setIntervalId = setInterval(async () => {
+		if (maxBytes) {
+			let fileStats;
+			try {
+				fileStats = await fsProm.stat(logger.path);
+			} catch (err) {
+				// If the log file doesn't exist, skip rotation check
+				if (err.code === 'ENOENT') return;
+				throw err;
+			}
+
+			if (fileStats.size >= maxBytes) {
+				try {
+					lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir);
+				} catch (err) {
+					// If the log file doesn't exist, skip rotation
+					if (err.code === 'ENOENT') return;
+					throw err;
+				}
+			}
+		}
+
+		if (maxInterval) {
+			const minSinceLastRotate = Date.now() - lastRotationTime;
+			if (minSinceLastRotate >= maxInterval) {
+				try {
+					lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir);
+					lastRotationTime = Date.now();
+				} catch (err) {
+					// If the log file doesn't exist, skip rotation
+					if (err.code === 'ENOENT') return;
+					throw err;
+				}
+			}
+		}
+		if (retention || reclamationPriority) {
+			// remove old logs after retention time
+			// adjust retention time if there is a reclamation priority in place
+			const retentionMs = convertToMS(retention ?? '1M') / (1 + reclamationPriority);
+			reclamationPriority = 0; // reset it after use
+			const files = await fsProm.readdir(rotatedLogDir);
+			for (const file of files) {
+				try {
+					const fileStats = await fsProm.stat(path.join(rotatedLogDir, file));
+					if (Date.now() - fileStats.mtimeMs > retentionMs) {
+						await fsProm.unlink(path.join(rotatedLogDir, file));
+					}
+				} catch (err) {
+					hdbLogger.error('Error trying to remove log', file, err);
+				}
+			}
+		}
+	}, auditInterval ?? LOG_AUDIT_INTERVAL).unref();
+	return {
+		end() {
+			clearInterval(setIntervalId);
+		},
+		getLastRotatedLogPath() {
+			return lastRotatedLogPath;
+		},
+	};
+}
+
+async function moveLogFile(logPath, rotatedLogPath) {
+	const compress = envMgr.get(CONFIG_PARAMS.LOGGING_ROTATION_COMPRESS);
+	let fullRotateLogPath = path.join(
+		rotatedLogPath,
+		`HDB-${new Date(Date.now()).toISOString().replaceAll(':', '-')}.log`
+	);
+	// Move log file to rotated log path first (if we crash
+	// during compression, we don't want to restart the compression with a new file)
+	await fsProm.rename(logPath, fullRotateLogPath);
+	if (compress) {
+		logPath = fullRotateLogPath;
+		fullRotateLogPath += '.gz';
+		await pipe(createReadStream(logPath), createGzip(), createWriteStream(fullRotateLogPath));
+		await fsProm.unlink(logPath);
+	}
+
+	// Close old log file.
+	hdbLogger.closeLogFile();
+	// This notify log will create a new log file after the previous one has been rotated. It's important to keep this log as notify
+	hdbLogger.notify(`hdb.log rotated, old log moved to ${fullRotateLogPath}`);
+	return fullRotateLogPath;
+}

package/utility/logging/logger.ts

@@ -0,0 +1,24 @@
+/** Like harperLogger, but conditionally exports functions based on the log level. */
+import harperLogger from './harper_logger.js';
+
+export const logger: Logger = {};
+
+for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'notify']) {
+	if (harperLogger.logsAtLevel(level)) {
+		logger[level] = harperLogger[level];
+	}
+}
+
+export function loggerWithTag(tag: string): Logger {
+	return harperLogger.loggerWithTag(tag, true) as Logger;
+}
+
+export interface Logger {
+	notify?: (...args: any[]) => void;
+	fatal?: (...args: any[]) => void;
+	error?: (...args: any[]) => void;
+	warn?: (...args: any[]) => void;
+	info?: (...args: any[]) => void;
+	debug?: (...args: any[]) => void;
+	trace?: (...args: any[]) => void;
+}

package/utility/logging/readLog.js

@@ -0,0 +1,355 @@
+'use strict';
+
+const hdbTerms = require('../hdbTerms.ts');
+const hdbLogger = require('./harper_logger.js');
+const envMangr = require('../environment/environmentManager.js');
+const validator = require('../../validation/readLogValidator.js');
+const path = require('path');
+const fs = require('fs-extra');
+const { once } = require('events');
+const { handleHDBError, hdbErrors } = require('../errors/hdbError.js');
+const { PACKAGE_ROOT } = require('../../utility/packageUtils.js');
+const { server } = require('../../server/Server.ts');
+
+// Install log is created in harperdb/logs because the hdb folder doesn't exist initially during the install process.
+const INSTALL_LOG_LOCATION = path.join(PACKAGE_ROOT, `logs`);
+const DEFAULT_READ_LOG_LIMIT = 1000;
+const ESTIMATED_AVERAGE_ENTRY_SIZE = 200;
+
+module.exports = readLog;
+
+/**
+ * Reads a log via a read stream and filters lines if filter params are passed.
+ * Returns an object array where each object is a line from the log.
+ * @param request
+ * @returns {Promise<*[]>}
+ */
+async function readLog(request) {
+	const validation = validator(request);
+	if (validation) {
+		throw handleHDBError(
+			validation,
+			validation.message,
+			hdbErrors.HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+	// start pulling logs from the other nodes now so it can be done in parallel
+	let whenReplicatedResponse = server.replication.replicateOperation(request);
+
+	const logPath = envMangr.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY);
+	const logName = request.log_name === undefined ? hdbTerms.LOG_NAMES.HDB : request.log_name;
+	const readLogPath =
+		logName === hdbTerms.LOG_NAMES.INSTALL
+			? path.join(INSTALL_LOG_LOCATION, hdbTerms.LOG_NAMES.INSTALL)
+			: path.join(logPath, logName);
+
+	// support 'until' attribute for backwards compatibility
+	if (request.to === undefined && request.until !== undefined) {
+		request.to = request.until;
+	}
+
+	const levelDefined = request.level !== undefined;
+	const level = levelDefined ? request.level : undefined;
+	const fromDefined = request.from !== undefined;
+	const from = fromDefined ? new Date(request.from) : undefined;
+	const toDefined = request.to !== undefined;
+	const to = toDefined ? new Date(request.to) : undefined;
+	const limit = request.limit === undefined ? DEFAULT_READ_LOG_LIMIT : request.limit;
+	const order = request.order === undefined ? undefined : request.order;
+	const start = request.start === undefined ? 0 : request.start;
+	const max = start + limit;
+	const filter = request.filter;
+	let fileStart = 0;
+	if (order === 'desc' && !from && !to) {
+		fileStart = Math.max(fs.statSync(readLogPath).size - (max + 5) * ESTIMATED_AVERAGE_ENTRY_SIZE, 0);
+	}
+	const readLogInputStream = fs.createReadStream(readLogPath, { start: fileStart });
+	readLogInputStream.on('error', (err) => {
+		hdbLogger.error(err);
+	});
+
+	let count = 0;
+	let result = [];
+	let remaining = '';
+	let pendingLogEntry;
+	let processedCount = 0;
+	readLogInputStream.on('data', (logData) => {
+		let reader = /(?:^|\n)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:[\d.]+Z) \[(.+?)]: /g;
+		logData = remaining + logData;
+		let lastPosition = 0;
+		let parsed;
+		while ((parsed = reader.exec(logData))) {
+			if (readLogInputStream.destroyed) break;
+			if (pendingLogEntry) {
+				pendingLogEntry.message = logData.slice(lastPosition, parsed.index);
+				onLogMessage(pendingLogEntry);
+			}
+			let [intro, timestamp, tagsString] = parsed;
+			let tags = tagsString.split('] [');
+			let thread = tags[0];
+			let level = tags[1];
+			tags.splice(0, 2);
+			pendingLogEntry = {
+				timestamp,
+				thread,
+				level,
+				tags,
+				message: '',
+			};
+			lastPosition = parsed.index + intro.length;
+		}
+		remaining = logData.slice(lastPosition);
+	});
+	readLogInputStream.on('end', () => {
+		if (readLogInputStream.destroyed) return;
+		if (pendingLogEntry) {
+			pendingLogEntry.message = remaining.trim();
+			onLogMessage(pendingLogEntry);
+		}
+	});
+	readLogInputStream.resume();
+	function onLogMessage(line) {
+		if (filter !== undefined) {
+			let found = false;
+			if (
+				['timestamp', 'thread', 'level', 'tags', 'message'].some((attr) => {
+					if (Array.isArray(line[attr])) {
+						return line[attr].some((val) => val.includes(filter));
+					}
+					return line[attr].includes(filter);
+				})
+			) {
+				found = true;
+			}
+			if (!found) return;
+		}
+
+		// Yield to event loop every 10 lines to heavily deprioritize this filtering relative to other operations
+		processedCount++;
+		if (processedCount % 10 === 0) {
+			readLogInputStream.pause();
+			setImmediate(() => readLogInputStream.resume());
+		}
+
+		let logDate;
+		let fromDate;
+		let toDate;
+		switch (true) {
+			case levelDefined && fromDefined && toDefined:
+				logDate = new Date(line.timestamp);
+				fromDate = new Date(from);
+				toDate = new Date(to);
+
+				// If the line matches the log level and timestamp falls between the from & to dates but the result count is less that the start,
+				// increment count and go to next line.
+				if (line.level === level && logDate >= fromDate && logDate <= toDate && count < start) count++;
+				// Else if all the criteria match and the count is equal/above the start, push line to result array.
+				else if (line.level === level && logDate >= fromDate && logDate <= toDate) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If all the criteria do not match, ignore the line and go to the next.
+				break;
+			case levelDefined && fromDefined:
+				logDate = new Date(line.timestamp);
+				fromDate = new Date(from);
+
+				// If the line matches the log level and timestamp is equal/above the fromDate but the result count is less that the start,
+				// increment count and go to next line.
+				if (line.level === level && logDate >= fromDate && count < start) count++;
+				// Else if the level and from date criteria match and the count is equal/above the start, push line to result array.
+				else if (line.level === level && logDate >= fromDate) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If criteria do not match, ignore the line and go to the next.
+				break;
+			case levelDefined && toDefined:
+				logDate = new Date(line.timestamp);
+				toDate = new Date(to);
+
+				// If the line matches the log level and timestamp is equal/below the toDate but the result count is less that the start,
+				// increment count and go to next line.
+				if (line.level === level && logDate <= toDate && count < start) count++;
+				// Else if the level and to date criteria match and the count is equal/above the start, push line to result array.
+				else if (line.level === level && logDate <= toDate) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If criteria do not match, ignore the line and go to the next.
+				break;
+			case fromDefined && toDefined:
+				logDate = new Date(line.timestamp);
+				fromDate = new Date(from);
+				toDate = new Date(to);
+
+				// If timestamp falls between the from & to dates but the result count is less that the start,
+				// increment count and go to next line.
+				if (logDate >= fromDate && logDate <= toDate && count < start) count++;
+				// Else if all the criteria match and the count is equal/above the start, push line to result array.
+				else if (logDate >= fromDate && logDate <= toDate) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If all the criteria do not match, ignore the line and go to the next.
+				break;
+			case levelDefined:
+				// If line level matches but count is below start, just increment count
+				if (line.level === level && count < start) count++;
+				// If level matches and count is equal/above start, add line to result in increment count.
+				else if (line.level === level) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If level criteria do not match, ignore the line and go to the next.
+				break;
+			case fromDefined:
+				logDate = new Date(line.timestamp);
+				fromDate = new Date(from);
+
+				// If timestamp is equal/above the fromDate but the result count is less that the start,
+				// increment count and go to next line.
+				if (logDate >= fromDate && count < start) count++;
+				// Else if from date criteria match and the count is equal/above the start, push line to result array.
+				else if (logDate >= fromDate && count >= start) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If criteria do not match, ignore the line and go to the next.
+				break;
+			case toDefined:
+				logDate = new Date(line.timestamp);
+				toDate = new Date(to);
+
+				// If timestamp is equal/below the toDate but the result count is less that the start,
+				// increment count and go to next line.
+				if (logDate <= toDate && count < start) count++;
+				// Else if to date criteria match and the count is equal/above the start, push line to result array.
+				else if (logDate <= toDate && count >= start) {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+
+				// If criteria do not match, ignore the line and go to the next.
+				break;
+			default:
+				// If count is under the start, increment count and go to next line
+				if (count < start) count++;
+				// Else push line to result and increment count
+				else {
+					pushLineToResult(line, order, result);
+					count++;
+					// If the count of matching lines is the max number of results, end the readline.
+					if (count === max) readLogInputStream.destroy();
+				}
+		}
+	}
+
+	await once(readLogInputStream, 'close');
+	let replicatedResponse = await whenReplicatedResponse;
+	if (replicatedResponse.replicated) {
+		// if this was a replicated request, add our node name to each of our own lines
+		for (let line of result) {
+			line.node = server.hostname;
+		}
+		// and then add the lines from the other nodes
+		for (let nodeResult of replicatedResponse.replicated) {
+			let node = nodeResult.node;
+			if (nodeResult.status === 'failed') {
+				// if the node failed to replicate, add an error line
+				pushLineToResult(
+					{
+						timestamp: new Date().toISOString(),
+						level: 'error',
+						node,
+						message: `Error retrieving logs: ${nodeResult.reason}`,
+					},
+					order,
+					result
+				);
+			} else {
+				for (let line of nodeResult.results) {
+					line.node = node;
+					pushLineToResult(line, order, result);
+				}
+			}
+		}
+	}
+	return result;
+}
+
+/**
+ * Pushes a line from the readline stream to the result array.
+ * If an order was passed in request, insert the line in the correct order.
+ * @param line
+ * @param order
+ * @param result
+ */
+function pushLineToResult(line, order, result) {
+	if (order === 'desc') {
+		insertDescending(line, result);
+	} else if (order === 'asc') {
+		insertAscending(line, result);
+	} else {
+		result.push(line);
+	}
+}
+
+/**
+ * Insert a line from log into result array in descending order by date.
+ * @param value
+ * @param result
+ */
+function insertDescending(value, result) {
+	const dateVal = new Date(value.timestamp);
+	let low = 0;
+	let high = result.length;
+	while (low < high) {
+		let mid = (low + high) >>> 1;
+		if (new Date(result[mid].timestamp) > dateVal) low = mid + 1;
+		else high = mid;
+	}
+
+	result.splice(low, 0, value);
+}
+
+/**
+ * Insert a line from log into result array in descending order by date.
+ * @param value
+ * @param result
+ */
+function insertAscending(value, result) {
+	const dateVal = new Date(value.timestamp);
+	let low = 0;
+	let high = result.length;
+	while (low < high) {
+		let mid = (low + high) >>> 1;
+		if (new Date(result[mid].timestamp) < dateVal) low = mid + 1;
+		else high = mid;
+	}
+
+	result.splice(low, 0, value);
+}

package/utility/logging/transactionLog.js

@@ -0,0 +1,57 @@
+'use strict';
+
+const hdbUtils = require('../common_utils.js');
+const log = require('./harper_logger.js');
+const { handleHDBError, hdbErrors } = require('../errors/hdbError.js');
+const { HTTP_STATUS_CODES } = hdbErrors;
+const {
+	readTransactionLogValidator,
+	deleteTransactionLogsBeforeValidator,
+} = require('../../validation/transactionLogValidator.js');
+const harperBridge = require('../../dataLayer/harperBridge/harperBridge.js');
+
+module.exports = {
+	readTransactionLog,
+	deleteTransactionLogsBefore,
+};
+
+async function readTransactionLog(req) {
+	const validation = readTransactionLogValidator(req);
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
+	}
+
+	req.database = req.database ?? req.schema ?? 'data';
+	const invalidSchemaTableMsg = hdbUtils.checkSchemaTableExist(req.database, req.table);
+	if (invalidSchemaTableMsg) {
+		throw handleHDBError(new Error(), invalidSchemaTableMsg, HTTP_STATUS_CODES.NOT_FOUND, undefined, undefined, true);
+	}
+
+	log.info('Reading Harper logs used by Plexus');
+
+	if (req.from || req.to) {
+		req.search_type = 'timestamp';
+		req.search_values = [req.from ?? 0];
+		if (req.to) req.search_values[1] = req.to;
+	}
+
+	return harperBridge.readAuditLog(req);
+}
+
+/**
+ * Deletes messages from a tables local stream (persistence layer),
+ * where all transactions against that table are stored.
+ * @param req - {schema, table, timestamp}
+ * @returns {Promise<string>}
+ */
+async function deleteTransactionLogsBefore(req) {
+	const validation = deleteTransactionLogsBeforeValidator(req);
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
+	}
+
+	req.database = req.database ?? req.schema ?? 'data';
+
+	log.info('Delete transaction logs called for Plexus');
+	return harperBridge.deleteAuditLogsBefore(req);
+}

package/utility/mount_hdb.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const { mkdirpSync, copySync } = require('fs-extra');
+const path = require('path');
+const terms = require('../utility/hdbTerms.ts');
+const hdbLogger = require('../utility/logging/harper_logger.js');
+const bridge = require('../dataLayer/harperBridge/harperBridge.js');
+const systemSchema = require('../json/systemSchema.json');
+const initPaths = require('../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js');
+const { PACKAGE_ROOT } = require('../utility/packageUtils');
+
+module.exports = mountHdb;
+
+async function mountHdb(hdbPath) {
+	hdbLogger.trace('Mounting Harper');
+
+	makeDirectory(hdbPath);
+	makeDirectory(path.join(hdbPath, 'backup'));
+	makeDirectory(path.join(hdbPath, 'keys'));
+	makeDirectory(path.join(hdbPath, 'log'));
+	makeDirectory(path.join(hdbPath, 'database'));
+	makeDirectory(path.join(hdbPath, 'components'));
+	copySync(path.join(PACKAGE_ROOT, 'static/README.md'), path.join(hdbPath, 'README.md'));
+
+	await createTables();
+}
+
+/**
+ * creates the environments & dbis needed for lmdb  based on the systemSchema
+ * @returns {Promise<void>}
+ */
+async function createTables() {
+	const CreateTableObject = require('../dataLayer/CreateTableObject.js');
+
+	let tables = Object.keys(systemSchema);
+
+	for (const tableName of tables) {
+		let hash_attribute = systemSchema[tableName].hash_attribute;
+		try {
+			initPaths.initSystemSchemaPaths(terms.SYSTEM_SCHEMA_NAME, tableName);
+			let createTable = new CreateTableObject(terms.SYSTEM_SCHEMA_NAME, tableName, hash_attribute);
+			createTable.attributes = systemSchema[tableName].attributes;
+			let primaryKeyAttribute = createTable.attributes.find(({ attribute }) => attribute === hash_attribute);
+			primaryKeyAttribute.isPrimaryKey = true;
+
+			// with RocksDB at least, we need to audit everything or there will be lost data
+			createTable.audit = true;
+			await bridge.createTable(tableName, createTable);
+		} catch (e) {
+			hdbLogger.error(`issue creating environment for ${terms.SYSTEM_SCHEMA_NAME}.${tableName}: ${e}`);
+			throw e;
+		}
+	}
+}
+
+function makeDirectory(targetDir) {
+	mkdirpSync(targetDir, { mode: terms.HDB_FILE_PERMISSIONS });
+	hdbLogger.info(`Directory ${targetDir} created`);
+}