@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/server/itc/serverHandlers.js

@@ -1,12 +1,15 @@
 'use strict';
 
 /* global threads */
-const hdbLogger = require('../../utility/logging/harper_logger.js');
+const hdbLogger = require('../../utility/logging/harper_logger.ts');
 const hdbTerms = require('../../utility/hdbTerms.ts');
-const cleanLmdbMap = require('../../utility/lmdb/cleanLMDBMap.js');
+const cleanLmdbMap =
+	require('../../utility/lmdb/cleanLMDBMap.ts').default || require('../../utility/lmdb/cleanLMDBMap.ts');
 const userSchema = require('../../security/user.ts');
 const { validateEvent } = require('../threads/itc.js');
-const harperBridge = require('../../dataLayer/harperBridge/harperBridge.js');
+const harperBridge =
+	require('../../dataLayer/harperBridge/harperBridge.ts').default ||
+	require('../../dataLayer/harperBridge/harperBridge.ts');
 const process = require('process');
 const { resetDatabases } = require('../../resources/databases.ts');
 
@@ -18,6 +21,7 @@ const serverItcHandlers = {
 	[hdbTerms.ITC_EVENT_TYPES.SCHEMA]: schemaHandler,
 	[hdbTerms.ITC_EVENT_TYPES.USER]: userHandler,
 	[hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST]: componentStatusRequestHandler,
+	[hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_REQUEST]: resourceOpenApiRequestHandler,
 };
 
 /**
@@ -25,6 +29,7 @@ const serverItcHandlers = {
  * @param event
  * @returns {Promise<void>}
  */
+const schemaListeners = [];
 async function schemaHandler(event) {
 	const validate = validateEvent(event);
 	if (validate) {
@@ -35,8 +40,19 @@ async function schemaHandler(event) {
 	hdbLogger.trace(`ITC schemaHandler received schema event:`, event);
 	await cleanLmdbMap(event.message);
 	await syncSchemaMetadata(event.message);
+	for (let listener of schemaListeners) {
+		try {
+			listener(event?.message);
+		} catch (err) {
+			hdbLogger.error(err);
+		}
+	}
 }
 
+schemaHandler.addListener = function (listener) {
+	schemaListeners.push(listener);
+};
+
 /**
  * Switch statement to handle schema-related messages from other forked processes - i.e. if another process completes an
  * operation that updates schema and, therefore, requires that we update the global schema value for the process
@@ -115,7 +131,6 @@ async function componentStatusRequestHandler(event) {
 		// Get current thread's component status
 		const { internal } = require('../../components/status/index.ts');
 		const { getWorkerIndex } = require('../threads/manageThreads.js');
-		const { sendItcEvent } = require('../threads/itc.js');
 		const componentStatuses = internal.componentStatusRegistry.getAllStatuses();
 
 		// Convert Map to array for serialization
@@ -125,7 +140,8 @@ async function componentStatusRequestHandler(event) {
 		const workerIndex = getWorkerIndex();
 		const isMainThread = workerIndex === undefined;
 
-		// Send response directly back to the originating thread
+		// Send response directly back to the originating thread. validateEvent already
+		// ensures originator is present.
 		const originatorThreadId = event.message.originator;
 		const responseMessage = {
 			type: hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_RESPONSE,
@@ -137,21 +153,66 @@ async function componentStatusRequestHandler(event) {
 			},
 		};
 
-		// Use global threads (connectedPorts) to send directly to originator
-		if (originatorThreadId !== undefined && threads.sendToThread(originatorThreadId, responseMessage)) {
+		if (threads.sendToThread(originatorThreadId, responseMessage)) {
 			hdbLogger.trace(`Sent component status response directly to thread ${originatorThreadId}`);
 		} else {
-			// Fallback to broadcast if direct send fails or originator is missing
-			if (originatorThreadId === undefined) {
-				hdbLogger.debug('No originator threadId, falling back to broadcast');
-			} else {
-				hdbLogger.debug(`Failed to send direct response to thread ${originatorThreadId}, falling back to broadcast`);
-			}
-			await sendItcEvent(responseMessage);
+			// Originator's port is no longer in connectedPorts (thread exited / disconnected
+			// during the request). Dropping the response is correct — the originator is
+			// unreachable, and the collector's own timeout will handle the missing reply.
+			hdbLogger.trace(
+				`Dropping component status response for request ${event.message.requestId}: originator thread ${originatorThreadId} is unreachable`
+			);
 		}
 	} catch (error) {
 		hdbLogger.error('Error handling component status request:', error);
 	}
 }
 
+/**
+ * Handles incoming requests for the REST OpenAPI spec from the main thread.
+ * Generates the spec from the local resources (which are only registered on worker threads)
+ * and sends it back to the requesting thread.
+ */
+async function resourceOpenApiRequestHandler(event) {
+	try {
+		const validate = validateEvent(event);
+		if (validate) {
+			hdbLogger.error(validate);
+			return;
+		}
+
+		hdbLogger.trace(`ITC resourceOpenApiRequestHandler received request:`, event);
+
+		const { resources } = require('../../resources/Resources.ts');
+		// Only respond if this thread has registered resources. Job-type workers with an empty
+		// resources map must stay silent so that an app worker with real resources replies first.
+		// If no worker has resources the main thread gets a 503 after the timeout, which is a
+		// more honest response than silently returning an empty spec.
+		if (!resources || resources.size === 0) return;
+		const { generateJsonApi } = require('../../resources/openApi.ts');
+		const openapi = generateJsonApi(resources, event.message.serverHttpURL);
+
+		const originatorThreadId = event.message.originator;
+		const responseMessage = {
+			type: hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_RESPONSE,
+			message: {
+				requestId: event.message.requestId,
+				openapi,
+			},
+		};
+
+		if (!threads.sendToThread(originatorThreadId, responseMessage)) {
+			hdbLogger.trace(
+				`Dropping resource OpenAPI response for request ${event.message.requestId}: originator thread ${originatorThreadId} is unreachable`
+			);
+		}
+	} catch (error) {
+		hdbLogger.error('Error handling resource OpenAPI request:', error);
+	}
+}
+
 module.exports = serverItcHandlers;
+// Named exports so consumers (e.g., MCP listChanged) can subscribe via
+// `userHandler.addListener(fn)` / `schemaHandler.addListener(fn)`.
+module.exports.userHandler = userHandler;
+module.exports.schemaHandler = schemaHandler;

package/server/jobs/{JobObject.js → JobObject.ts}

@@ -1,13 +1,22 @@
 'use strict';
 
-const hdbTerm = require('../../utility/hdbTerms.ts');
-const moment = require('moment');
-const uuidV4 = require('uuid').v4;
+import * as hdbTerm from '../../utility/hdbTerms.ts';
+import moment from 'moment';
+import { v4 as uuidV4 } from 'uuid';
 
 /**
  * This class represents a Job as it resides in the jobs table.
  */
-class JobObject {
+export default class JobObject {
+	id: string;
+	type: any;
+	start_datetime: number;
+	created_datetime: number;
+	end_datetime: any;
+	status: any;
+	message: any;
+	user: any;
+	request: any;
 	constructor() {
 		this.id = uuidV4();
 		this.type = undefined;
@@ -20,5 +29,3 @@ class JobObject {
 		this.request = undefined;
 	}
 }
-
-module.exports = JobObject;

package/server/jobs/{jobProcess.js → jobProcess.ts}

@@ -1,19 +1,23 @@
 'use strict';
 
-const hdbTerms = require('../../utility/hdbTerms.ts');
-const hdbUtils = require('../../utility/common_utils.js');
-const harperLogger = require('../../utility/logging/harper_logger.js');
-const globalSchema = require('../../utility/globalSchema.js');
-const user = require('../../security/user.ts');
-const serverUtils = require('../serverHelpers/serverUtilities.ts');
-const moment = require('moment');
-const jobs = require('./jobs.js');
-const { cloneDeep } = require('lodash');
-const { getEnvBuiltInComponents } = require('../../components/Application.ts');
-const { pathToFileURL } = require('node:url');
-const { join } = require('node:path');
-const { PACKAGE_ROOT } = require('../../utility/packageUtils.js');
-const JOB_NAME = process.env[hdbTerms.PROCESS_NAME_ENV_PROP];
+// Install the worker process guard first so user job code cannot terminate
+// the worker via process.exit() or an unhandled rejection.
+import { realExit } from '../threads/workerProcessGuard.ts';
+import * as hdbTerms from '../../utility/hdbTerms.ts';
+import * as hdbUtils from '../../utility/common_utils.ts';
+import harperLogger from '../../utility/logging/harper_logger.ts';
+import * as globalSchema from '../../utility/globalSchema.ts';
+import * as user from '../../security/user.ts';
+import * as serverUtils from '../serverHelpers/serverUtilities.ts';
+import moment from 'moment';
+import * as jobs from './jobs.ts';
+import { cloneDeep } from 'lodash';
+
+import { pathToFileURL } from 'node:url';
+import { join } from 'node:path';
+import { getEnvBuiltInComponents } from './../../components/Application.ts';
+import { PACKAGE_ROOT } from '../../utility/packageUtils.js';
+const JOB_NAME = process.env[(hdbTerms as any).PROCESS_NAME_ENV_PROP] as string;
 const JOB_ID = JOB_NAME.substring(4);
 
 /**
@@ -23,7 +27,7 @@ const JOB_ID = JOB_NAME.substring(4);
  */
 (async function job() {
 	// The request value could potentially be quite large so it's set to undefined to clear it out after being processed.
-	let jobObj = { id: JOB_ID, request: undefined };
+	let jobObj: any = { id: JOB_ID, request: undefined };
 	let exitCode = 0;
 	try {
 		harperLogger.notify('Starting job:', JOB_ID);
@@ -74,7 +78,7 @@ const JOB_ID = JOB_NAME.substring(4);
 	} finally {
 		await jobs.updateJob(jobObj);
 		setTimeout(() => {
-			process.exit(exitCode);
+			realExit(exitCode);
 		}, 3000).unref();
 	}
 })();

package/server/jobs/{jobRunner.js → jobRunner.ts}

@@ -1,22 +1,24 @@
 'use strict';
 
-const { join } = require('node:path');
+import { join } from 'node:path';
 
-const hdbUtil = require('../../utility/common_utils.js');
-const hdbTerms = require('../../utility/hdbTerms.ts');
-const moment = require('moment');
-const bulkLoad = require('../../dataLayer/bulkLoad.js');
-const log = require('../../utility/logging/harper_logger.js');
-const jobs = require('./jobs.js');
-const hdbExport = require('../../dataLayer/export.js');
-const hdbDelete = require('../../dataLayer/delete.js');
-const threadsStart = require('../threads/manageThreads.js');
-const transactionLog = require('../../utility/logging/transactionLog.js');
-const restart = require('../../bin/restart.js');
-const { parentPort, isMainThread } = require('worker_threads');
-const { onMessageByType } = require('../threads/manageThreads.js');
+import * as hdbUtil from '../../utility/common_utils.ts';
+import * as hdbTerms from '../../utility/hdbTerms.ts';
+import moment from 'moment';
+import * as bulkLoad from '../../dataLayer/bulkLoad.ts';
+import log from '../../utility/logging/harper_logger.ts';
+import * as jobs from './jobs.ts';
+import * as hdbExport from '../../dataLayer/export.ts';
+import * as hdbDelete from '../../dataLayer/delete.ts';
+import * as threadsStart from '../threads/manageThreads.js';
+import * as transactionLog from '../../utility/logging/transactionLog.ts';
+import * as restart from '../../bin/restart.ts';
+import { parentPort, isMainThread } from 'worker_threads';
+import { onMessageByType } from '../threads/manageThreads.js';
 
 class RunnerMessage {
+	job: any;
+	json: any;
 	constructor(jobObject, messageJson) {
 		this.job = jobObject;
 		this.json = messageJson;
@@ -28,7 +30,7 @@ class RunnerMessage {
  * @param runnerMessage
  * @throws Error
  */
-async function parseMessage(runnerMessage) {
+async function parseMessage(runnerMessage: any) {
 	if (!runnerMessage || Object.keys(runnerMessage).length === 0) {
 		throw new Error('Empty runner passed to parseMessage');
 	}
@@ -91,7 +93,7 @@ async function parseMessage(runnerMessage) {
  * @param runnerMessage - The RunnerMessage created by the signal flow
  * @param operation - The operation to run.
  */
-async function runJob(runnerMessage, operation) {
+async function runJob(runnerMessage: any, operation: any) {
 	try {
 		runnerMessage.job.status = hdbTerms.JOB_STATUS_ENUM.IN_PROGRESS;
 		runnerMessage.job.start_datetime = moment().valueOf();
@@ -128,7 +130,7 @@ async function runJob(runnerMessage, operation) {
  * @param job_id
  * @returns {Promise<void>}
  */
-async function launchJobThread(job_id) {
+async function launchJobThread(job_id: any) {
 	log.trace('launching job thread:', job_id);
 	if (isMainThread) {
 		threadsStart.startWorker(join(__dirname, './jobProcess.js'), {
@@ -157,7 +159,4 @@ if (isMainThread) {
 	});
 }
 
-module.exports = {
-	parseMessage,
-	RunnerMessage,
-};
+export { parseMessage, RunnerMessage };

package/server/jobs/{jobs.js → jobs.ts}

@@ -5,25 +5,25 @@
  * exposed method to simplify the interaction.
  */
 
-const uuidV4 = require('uuid').v4;
-const insert = require('../../dataLayer/insert.js');
-const search = require('../../dataLayer/search.js');
-const Search_Object = require('../../dataLayer/SearchObject.js');
-const searchByHashObj = require('../../dataLayer/SearchByHashObject.js');
-const SQL_Search_Object = require('../../dataLayer/SqlSearchObject.js');
-const hdbTerms = require('../../utility/hdbTerms.ts');
-const JobObject = require('./JobObject.js');
-const UpdateObject = require('../../dataLayer/UpdateObject.js');
-const log = require('../../utility/logging/harper_logger.js');
-const Insert_Object = require('../../dataLayer/InsertObject.js');
-const hdbUtil = require('../../utility/common_utils.js');
-const { promisify } = require('util');
-const moment = require('moment');
-const fileLoadValidator = require('../../validation/fileLoadValidator.js');
-const bulkDeleteValidator = require('../../validation/bulkDeleteValidator.js');
-const { deleteTransactionLogsBeforeValidator } = require('../../validation/transactionLogValidator.js');
-const { handleHDBError, hdbErrors, ClientError } = require('../../utility/errors/hdbError.js');
-const { HTTP_STATUS_CODES } = hdbErrors;
+import { v4 as uuidV4 } from 'uuid';
+import * as insert from '../../dataLayer/insert.ts';
+import * as search from '../../dataLayer/search.ts';
+import Search_Object from '../../dataLayer/SearchObject.ts';
+import searchByHashObj from '../../dataLayer/SearchByHashObject.ts';
+import SQL_Search_Object from '../../dataLayer/SqlSearchObject.ts';
+import * as hdbTerms from '../../utility/hdbTerms.ts';
+import JobObject from './JobObject.ts';
+import UpdateObject from '../../dataLayer/UpdateObject.ts';
+import log from '../../utility/logging/harper_logger.ts';
+import Insert_Object from '../../dataLayer/InsertObject.ts';
+import * as hdbUtil from '../../utility/common_utils.ts';
+import { promisify } from 'util';
+import moment from 'moment';
+import * as fileLoadValidator from '../../validation/fileLoadValidator.ts';
+import bulkDeleteValidator from '../../validation/bulkDeleteValidator.ts';
+import { deleteTransactionLogsBeforeValidator } from '../../validation/transactionLogValidator.ts';
+import { handleHDBError, ClientError } from '../../utility/errors/hdbError.ts';
+import { HTTP_STATUS_CODES } from '../../utility/errors/commonErrors.ts';
 
 //Promisified functions
 const pSearchByValue = search.searchByValue;
@@ -32,15 +32,7 @@ const pInsert = insert.insert;
 const pInsertUpdate = insert.update;
 let pSqlEvaluate;
 
-module.exports = {
-	addJob,
-	updateJob,
-	handleGetJob,
-	handleGetJobsByStartDate,
-	getJobById,
-};
-
-async function handleGetJob(jsonBody) {
+export async function handleGetJob(jsonBody: any) {
 	if (jsonBody.id === undefined) throw new ClientError("'id' is required");
 	let result = await getJobById(jsonBody.id);
 	if (!hdbUtil.isEmptyOrZeroLength(result)) {
@@ -53,7 +45,7 @@ async function handleGetJob(jsonBody) {
 	return result;
 }
 
-async function handleGetJobsByStartDate(jsonBody) {
+export async function handleGetJobsByStartDate(jsonBody: any) {
 	try {
 		let result = await getJobsInDateRange(jsonBody);
 		log.trace(`Searching for jobs from ${jsonBody.from_date} to ${jsonBody.to_date}`);
@@ -84,7 +76,7 @@ async function handleGetJobsByStartDate(jsonBody) {
  * @param jsonBody - job descriptor defined in the endpoint.
  * @returns {Promise<*>}
  */
-async function addJob(jsonBody) {
+export async function addJob(jsonBody: any) {
 	let result = { message: '', error: '', success: false, createdJob: undefined };
 	if (!jsonBody || Object.keys(jsonBody).length === 0 || hdbUtil.isEmptyOrZeroLength(jsonBody.operation)) {
 		let errMsg = `job parameter is invalid`;
@@ -144,14 +136,14 @@ async function addJob(jsonBody) {
 		);
 	}
 
-	let newJob = new JobObject();
+	let newJob = new (JobObject as any)();
 	newJob.type =
 		jsonBody.operation === hdbTerms.OPERATIONS_ENUM.DELETE_RECORDS_BEFORE
 			? hdbTerms.OPERATIONS_ENUM.DELETE_FILES_BEFORE
 			: jsonBody.operation;
 	newJob.type = jsonBody.operation;
 	newJob.user = jsonBody.hdb_user?.username;
-	let searchObj = new Search_Object(
+	let searchObj = new (Search_Object as any)(
 		hdbTerms.SYSTEM_SCHEMA_NAME,
 		hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME,
 		'id',
@@ -195,9 +187,12 @@ async function addJob(jsonBody) {
 	// Sending the request via IPC to the job process was causing some messages to be lost under load.
 	newJob.request = jsonBody;
 
-	let insertObject = new Insert_Object(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME, 'id', [
-		newJob,
-	]);
+	let insertObject = new (Insert_Object as any)(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME,
+		'id',
+		[newJob]
+	);
 	let insertResult;
 	try {
 		insertResult = await pInsert(insertObject);
@@ -224,7 +219,7 @@ async function addJob(jsonBody) {
  * @param jsonBody - The inbound message
  * @returns {Promise<*>}
  */
-async function getJobsInDateRange(jsonBody) {
+export async function getJobsInDateRange(jsonBody: any) {
 	let parsedFromDate = moment(jsonBody.from_date, moment.ISO_8601);
 	let parsedToDate = moment(jsonBody.to_date, moment.ISO_8601);
 
@@ -236,11 +231,11 @@ async function getJobsInDateRange(jsonBody) {
 	}
 
 	let jobSearchSql = `select * from system.hdb_job where start_datetime > '${parsedFromDate.valueOf()}' and start_datetime < '${parsedToDate.valueOf()}'`;
-	let sqlSearchObj = new SQL_Search_Object(jobSearchSql, jsonBody.hdb_user);
+	let sqlSearchObj = new (SQL_Search_Object as any)(jobSearchSql, jsonBody.hdb_user);
 
 	try {
 		if (!pSqlEvaluate) {
-			const hdbSql = require('../../sqlTranslator');
+			const hdbSql = require('../../sqlTranslator/index');
 			pSqlEvaluate = promisify(hdbSql.evaluateSQL);
 		}
 		return await pSqlEvaluate(sqlSearchObj);
@@ -257,12 +252,12 @@ async function getJobsInDateRange(jsonBody) {
  * @param jsonBody - The inbound message
  * @returns {Promise<*>}
  */
-async function getJobById(job_id) {
+export async function getJobById(job_id: any) {
 	if (hdbUtil.isEmptyOrZeroLength(job_id)) {
 		return hdbUtil.errorizeMessage('Invalid job ID specified.');
 	}
 
-	const searchObj = new searchByHashObj(
+	const searchObj = new (searchByHashObj as any)(
 		hdbTerms.SYSTEM_SCHEMA_NAME,
 		hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME,
 		[job_id],
@@ -283,7 +278,7 @@ async function getJobById(job_id) {
  * @param jobObject - The object representing the desired record.
  * @returns {Promise<*>}
  */
-async function updateJob(jobObject) {
+export async function updateJob(jobObject: any) {
 	if (Object.keys(jobObject).length === 0) {
 		throw new Error('invalid job object passed to updateJob');
 	}
@@ -295,9 +290,11 @@ async function updateJob(jobObject) {
 		jobObject.end_datetime = moment().valueOf();
 	}
 
-	let updateObject = new UpdateObject(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME, [
-		jobObject,
-	]);
+	let updateObject = new (UpdateObject as any)(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME,
+		[jobObject]
+	);
 	let updateResult = undefined;
 	updateResult = await pInsertUpdate(updateObject);
 	return updateResult;

package/server/loadRootComponents.js

@@ -4,7 +4,7 @@ const { loadComponentDirectories, loadComponent } = require('../components/compo
 const { resetResources } = require('../resources/Resources.ts');
 const configUtils = require('../config/configUtils.js');
 const { dirname } = require('path');
-const { loadCertificates } = require('../security/keys.js');
+const { loadCertificates } = require('../security/keys.ts');
 const { installApplications } = require('../components/Application.ts');
 
 let loadedComponents = new Map();