@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/server/itc/serverHandlers.js

@@ -0,0 +1,161 @@
+'use strict';
+
+/* global threads */
+const hdbLogger = require('../../utility/logging/harper_logger.js');
+const hdbTerms = require('../../utility/hdbTerms.ts');
+const cleanLmdbMap = require('../../utility/lmdb/cleanLMDBMap.js');
+const userSchema = require('../../security/user.ts');
+const { validateEvent } = require('../threads/itc.js');
+const harperBridge = require('../../dataLayer/harperBridge/harperBridge.js');
+const process = require('process');
+const { resetDatabases } = require('../../resources/databases.ts');
+
+/**
+ * This object/functions are passed to the ITC client instance and dynamically added as event handlers.
+ * @type {{schema: ((function(*): Promise<void>)|*), job: ((function(*): Promise<void>)|*), user: ((function(): Promise<void>)|*)}}
+ */
+const serverItcHandlers = {
+	[hdbTerms.ITC_EVENT_TYPES.SCHEMA]: schemaHandler,
+	[hdbTerms.ITC_EVENT_TYPES.USER]: userHandler,
+	[hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST]: componentStatusRequestHandler,
+};
+
+/**
+ * Updates the global hdbSchema object.
+ * @param event
+ * @returns {Promise<void>}
+ */
+async function schemaHandler(event) {
+	const validate = validateEvent(event);
+	if (validate) {
+		hdbLogger.error(validate);
+		return;
+	}
+
+	hdbLogger.trace(`ITC schemaHandler received schema event:`, event);
+	await cleanLmdbMap(event.message);
+	await syncSchemaMetadata(event.message);
+}
+
+/**
+ * Switch statement to handle schema-related messages from other forked processes - i.e. if another process completes an
+ * operation that updates schema and, therefore, requires that we update the global schema value for the process
+ *
+ * @param msg
+ * @returns {Promise<void>}
+ */
+async function syncSchemaMetadata(msg) {
+	try {
+		// reset current read transactions to ensure that we are getting the very latest data
+		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.TABLE_TABLE_NAME);
+		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME);
+		harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.SCHEMA_TABLE_NAME);
+		// TODO: Eventually should indicate which database/table changed so we don't have to scan everything
+		let databases = resetDatabases();
+		if (msg.table && msg.database)
+			// wait for a write to finish to ensure all writes have been written
+			await databases[msg.database][msg.table].put(Symbol.for('write-verify'), null);
+	} catch (e) {
+		hdbLogger.error(e);
+	}
+}
+
+const userListeners = [];
+/**
+ * Updates the global hdbUsers object by querying the hdbRole table.
+ * @param event
+ * @returns {Promise<void>}
+ */
+async function userHandler(event) {
+	try {
+		try {
+			harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.USER_TABLE_NAME);
+			harperBridge.resetReadTxn(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME);
+		} catch (error) {
+			// this can happen during tests, best to ignore
+			hdbLogger.warn(error);
+		}
+		const validate = validateEvent(event);
+		if (validate) {
+			hdbLogger.error(validate);
+			return;
+		}
+
+		hdbLogger.trace(`ITC userHandler ${hdbTerms.HDB_ITC_CLIENT_PREFIX}${process.pid} received user event:`, event);
+		await userSchema.setUsersWithRolesCache();
+		for (let listener of userListeners) listener();
+	} catch (err) {
+		hdbLogger.error(err);
+	}
+}
+
+userHandler.addListener = function (listener) {
+	userListeners.push(listener);
+};
+
+/**
+ * Handles incoming requests for component status from inter-thread communication (ITC).
+ * Validates the event, retrieves the current thread's component statuses, and sends a response
+ * back to the originator thread with the requested information.
+ *
+ * @async
+ * @function componentStatusRequestHandler
+ * @param {Object} event - The event object containing the request details.
+ * @param {Object} event.message - The message object within the event.
+ * @param {string} event.message.originator - The identifier of the thread that originated the request.
+ * @param {string} event.message.requestId - The unique identifier for the request.
+ * @returns {Promise<void>} Sends a response back to the originator thread or logs an error if validation fails.
+ */
+async function componentStatusRequestHandler(event) {
+	try {
+		const validate = validateEvent(event);
+		if (validate) {
+			hdbLogger.error(validate);
+			return;
+		}
+
+		hdbLogger.trace(`ITC componentStatusRequestHandler received request:`, event);
+
+		// Get current thread's component status
+		const { internal } = require('../../components/status/index.ts');
+		const { getWorkerIndex } = require('../threads/manageThreads.js');
+		const { sendItcEvent } = require('../threads/itc.js');
+		const componentStatuses = internal.componentStatusRegistry.getAllStatuses();
+
+		// Convert Map to array for serialization
+		const statusArray = Array.from(componentStatuses.entries());
+
+		// Get worker index and determine if this is the main thread
+		const workerIndex = getWorkerIndex();
+		const isMainThread = workerIndex === undefined;
+
+		// Send response directly back to the originating thread
+		const originatorThreadId = event.message.originator;
+		const responseMessage = {
+			type: hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_RESPONSE,
+			message: {
+				requestId: event.message.requestId,
+				statuses: statusArray,
+				workerIndex: workerIndex,
+				isMainThread: isMainThread,
+			},
+		};
+
+		// Use global threads (connectedPorts) to send directly to originator
+		if (originatorThreadId !== undefined && threads.sendToThread(originatorThreadId, responseMessage)) {
+			hdbLogger.trace(`Sent component status response directly to thread ${originatorThreadId}`);
+		} else {
+			// Fallback to broadcast if direct send fails or originator is missing
+			if (originatorThreadId === undefined) {
+				hdbLogger.debug('No originator threadId, falling back to broadcast');
+			} else {
+				hdbLogger.warn(`Failed to send direct response to thread ${originatorThreadId}, falling back to broadcast`);
+			}
+			await sendItcEvent(responseMessage);
+		}
+	} catch (error) {
+		hdbLogger.error('Error handling component status request:', error);
+	}
+}
+
+module.exports = serverItcHandlers;

package/server/itc/utility/ITCEventObject.js

@@ -0,0 +1,10 @@
+'use strict';
+
+class ITCEventObject {
+	constructor(type, message) {
+		this.type = type;
+		this.message = message;
+	}
+}
+
+module.exports = ITCEventObject;

package/server/jobs/JobObject.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const hdbTerm = require('../../utility/hdbTerms.ts');
+const moment = require('moment');
+const uuidV4 = require('uuid').v4;
+
+/**
+ * This class represents a Job as it resides in the jobs table.
+ */
+class JobObject {
+	constructor() {
+		this.id = uuidV4();
+		this.type = undefined;
+		this.start_datetime = moment().valueOf();
+		this.created_datetime = moment().valueOf();
+		this.end_datetime = undefined;
+		this.status = hdbTerm.JOB_STATUS_ENUM.CREATED;
+		this.message = undefined;
+		this.user = undefined;
+		this.request = undefined;
+	}
+}
+
+module.exports = JobObject;

package/server/jobs/jobProcess.js

@@ -0,0 +1,69 @@
+'use strict';
+
+const hdbTerms = require('../../utility/hdbTerms.ts');
+const hdbUtils = require('../../utility/common_utils.js');
+const harperLogger = require('../../utility/logging/harper_logger.js');
+const globalSchema = require('../../utility/globalSchema.js');
+const user = require('../../security/user.ts');
+const serverUtils = require('../serverHelpers/serverUtilities.ts');
+const moment = require('moment');
+const jobs = require('./jobs.js');
+const { cloneDeep } = require('lodash');
+const JOB_NAME = process.env[hdbTerms.PROCESS_NAME_ENV_PROP];
+const JOB_ID = JOB_NAME.substring(4);
+
+/**
+ * Finds the appropriate function for the request and runs it.
+ * Then updates the job table accordingly.
+ * @returns {Promise<void>}
+ */
+(async function job() {
+	// The request value could potentially be quite large so it's set to undefined to clear it out after being processed.
+	let jobObj = { id: JOB_ID, request: undefined };
+	let exitCode = 0;
+	try {
+		harperLogger.notify('Starting job:', JOB_ID);
+		globalSchema.setSchemaDataToGlobal();
+		await user.setUsersWithRolesCache();
+
+		// When the job record is first inserted in hdbJob table by HDB, the incoming API request is included, this is
+		// how we pass the request to the job process. IPC was initially used but messages were getting lost under heavy load.
+		const jobRecord = await jobs.getJobById(JOB_ID);
+		if (hdbUtils.isEmptyOrZeroLength(jobRecord)) {
+			throw new Error(`Unable to find a record in hdbJob for job: ${JOB_ID}`);
+		}
+
+		let { request } = jobRecord[0];
+		if (hdbUtils.isEmptyOrZeroLength(request)) {
+			throw new Error('Did not find job request in hdb_job table, unable to proceed');
+		}
+		request = cloneDeep(request);
+
+		const operation = serverUtils.getOperationFunction(request);
+		harperLogger.trace('Running operation:', request.operation, 'for job', JOB_ID);
+
+		// Run the job operation.
+		const results = await operation.job_operation_function(request);
+		harperLogger.trace('Result from job:', JOB_ID, results);
+
+		jobObj.status = hdbTerms.JOB_STATUS_ENUM.COMPLETE;
+		if (typeof results === 'string') jobObj.message = results;
+		else {
+			jobObj.result = results;
+			jobObj.message = 'Successfully completed job: ' + JOB_ID;
+		}
+		jobObj.end_datetime = moment().valueOf();
+		harperLogger.notify('Successfully completed job:', JOB_ID);
+	} catch (err) {
+		exitCode = 1;
+		harperLogger.error(err);
+		jobObj.status = hdbTerms.JOB_STATUS_ENUM.ERROR;
+		jobObj.message = err.message ? err.message : err;
+		jobObj.end_datetime = moment().valueOf();
+	} finally {
+		await jobs.updateJob(jobObj);
+		setTimeout(() => {
+			process.exit(exitCode);
+		}, 3000).unref();
+	}
+})();

package/server/jobs/jobRunner.js

@@ -0,0 +1,162 @@
+'use strict';
+
+const { join } = require('node:path');
+
+const hdbUtil = require('../../utility/common_utils.js');
+const hdbTerms = require('../../utility/hdbTerms.ts');
+const moment = require('moment');
+const bulkLoad = require('../../dataLayer/bulkLoad.js');
+const log = require('../../utility/logging/harper_logger.js');
+const jobs = require('./jobs.js');
+const hdbExport = require('../../dataLayer/export.js');
+const hdbDelete = require('../../dataLayer/delete.js');
+const threadsStart = require('../threads/manageThreads.js');
+const transactionLog = require('../../utility/logging/transactionLog.js');
+const restart = require('../../bin/restart.js');
+const { parentPort, isMainThread } = require('worker_threads');
+const { onMessageByType } = require('../threads/manageThreads.js');
+
+class RunnerMessage {
+	constructor(jobObject, messageJson) {
+		this.job = jobObject;
+		this.json = messageJson;
+	}
+}
+
+/**
+ * Parses a RunnerMessage and runs the specified job.
+ * @param runnerMessage
+ * @throws Error
+ */
+async function parseMessage(runnerMessage) {
+	if (!runnerMessage || Object.keys(runnerMessage).length === 0) {
+		throw new Error('Empty runner passed to parseMessage');
+	}
+	if (!runnerMessage.json || Object.keys(runnerMessage.json).length === 0) {
+		throw new Error('Empty JSON passed to parseMessage');
+	}
+	if (!runnerMessage.job || Object.keys(runnerMessage.job).length === 0) {
+		throw new Error('Empty job passed to parseMessage');
+	}
+	if (hdbUtil.isEmptyOrZeroLength(runnerMessage.json.operation)) {
+		throw new Error('Invalid operation');
+	}
+	if (hdbUtil.isEmptyOrZeroLength(runnerMessage.job.id)) {
+		throw new Error('Empty job id specified');
+	}
+
+	switch (runnerMessage.json.operation) {
+		case hdbTerms.JOB_TYPE_ENUM.csv_file_load:
+			await runJob(runnerMessage, bulkLoad.csvFileLoad);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.csv_url_load:
+			await runJob(runnerMessage, bulkLoad.csvURLLoad);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.csv_data_load:
+			await runJob(runnerMessage, bulkLoad.csvDataLoad);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.import_from_s3:
+			await runJob(runnerMessage, bulkLoad.importFromS3);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.empty_trash:
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.export_local:
+			await runJob(runnerMessage, hdbExport.export_local);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.export_to_s3:
+			await runJob(runnerMessage, hdbExport.export_to_s3);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.delete_files_before:
+		case hdbTerms.JOB_TYPE_ENUM.delete_records_before:
+			await runJob(runnerMessage, hdbDelete.deleteFilesBefore);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.delete_audit_logs_before:
+			await runJob(runnerMessage, hdbDelete.deleteAuditLogsBefore);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.delete_transaction_logs_before:
+			await runJob(runnerMessage, transactionLog.deleteTransactionLogsBefore);
+			break;
+		case hdbTerms.JOB_TYPE_ENUM.restart_service:
+			await runJob(runnerMessage, restart.restartService);
+			return `Restarting ${runnerMessage.json.service}`;
+			break;
+		default:
+			return `Invalid operation ${runnerMessage.json.operation} specified`;
+	}
+}
+
+/**
+ * Helper function to run the specified operation using the job update 'workflow'.
+ * @param runnerMessage - The RunnerMessage created by the signal flow
+ * @param operation - The operation to run.
+ */
+async function runJob(runnerMessage, operation) {
+	try {
+		runnerMessage.job.status = hdbTerms.JOB_STATUS_ENUM.IN_PROGRESS;
+		runnerMessage.job.start_datetime = moment().valueOf();
+		// Update with "IN PROGRESS"
+		await jobs.updateJob(runnerMessage.job);
+		// Run the operation.
+		await launchJobThread(runnerMessage.job.id);
+	} catch (e) {
+		let errMessage = e.message !== undefined ? e.message : e;
+		if (typeof errMessage === 'string') {
+			errMessage = `There was an error running ${operation.name} job with id ${runnerMessage.job.id} - ${errMessage}`;
+			e.message = errMessage;
+		} else {
+			//This ensures that the op/job id error is logged if the error message is passed as a non-string which will
+			// be logged right after this below.  If the message is a string, everything will be logged below as the errMessage
+			log.error(`There was an error running ${operation.name} job with id ${runnerMessage.job.id}`);
+		}
+		log.error(errMessage);
+		runnerMessage.job.message = errMessage;
+		runnerMessage.job.status = hdbTerms.JOB_STATUS_ENUM.ERROR;
+		try {
+			// Update with "Error"
+			await jobs.updateJob(runnerMessage.job);
+		} catch (ex) {
+			log.error(`Unable to update job with id ${runnerMessage.job.id}`);
+			throw ex;
+		}
+		throw e;
+	}
+}
+
+/**
+ * Launches job in a separate process using processManagement
+ * @param job_id
+ * @returns {Promise<void>}
+ */
+async function launchJobThread(job_id) {
+	log.trace('launching job thread:', job_id);
+	if (isMainThread) {
+		threadsStart.startWorker(join(__dirname, './jobProcess.js'), {
+			autoRestart: false,
+			name: 'job',
+			env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${job_id}` },
+		});
+	} else {
+		parentPort.postMessage({
+			type: hdbTerms.ITC_EVENT_TYPES.START_JOB,
+			jobId: job_id,
+		});
+	}
+}
+if (isMainThread) {
+	onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => {
+		try {
+			threadsStart.startWorker(join(__dirname, './jobProcess.js'), {
+				autoRestart: false,
+				name: 'job',
+				env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${message.jobId}` },
+			});
+		} catch (e) {
+			log.error(e);
+		}
+	});
+}
+
+module.exports = {
+	parseMessage,
+	RunnerMessage,
+};