@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/core/dataLayer/{UpsertObject.js → UpsertObject.ts}

@@ -1,10 +1,11 @@
 'use strict';
-const OPERATIONS_ENUM = require('../utility/hdbTerms.ts').OPERATIONS_ENUM;
+import { OPERATIONS_ENUM } from '../utility/hdbTerms.ts';
 
 /**
  * object representing an upsert operation
  */
 class UpsertObject {
+	[key: string]: any;
 	/**
 	 * @param {String} schema
 	 * @param {string} table
@@ -20,4 +21,4 @@ class UpsertObject {
 	}
 }
 
-module.exports = UpsertObject;
+export default UpsertObject;

package/core/dataLayer/{bulkLoad.js → bulkLoad.ts}

@@ -1,34 +1,32 @@
-'use strict';
-
-const insert = require('./insert.js');
-const validator = require('../validation/fileLoadValidator.js');
-const needle = require('needle');
-const hdbTerms = require('../utility/hdbTerms.ts');
-const hdbUtils = require('../utility/common_utils.js');
-const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
-const { HTTP_STATUS_CODES, HDB_ERROR_MSGS, CHECK_LOGS_WRAPPER } = hdbErrors;
-const logger = require('../utility/logging/harper_logger.js');
-const papaParse = require('papaparse');
-hdbUtils.promisifyPapaParse();
-const fs = require('fs-extra');
-const path = require('path');
-const { chain } = require('stream-chain');
-const StreamArray = require('stream-json/streamers/StreamArray');
-const Batch = require('stream-json/utils/Batch');
-const comp = require('stream-chain/utils/comp');
-const { finished } = require('stream');
-const env = require('../utility/environment/environmentManager.js');
-const opFuncCaller = require('../utility/OperationFunctionCaller.js');
-const AWSConnector = require('../utility/AWS/AWSConnector.js');
-const { BulkLoadFileObject, BulkLoadDataObject } = require('./dataObjects/BulkLoadObjects.js');
-const PermissionResponseObject = require('../security/data_objects/PermissionResponseObject.js');
-const { verifyBulkLoadAttributePerms } = require('../utility/operation_authorization.js');
-const { databases } = require('../resources/databases.ts');
-const { coerceType } = require('../resources/Table.ts');
+import * as insert from './insert.ts';
+import * as validator from '../validation/fileLoadValidator.ts';
+import needle from 'needle';
+import * as hdbTerms from '../utility/hdbTerms.ts';
+import * as hdbUtils from '../utility/common_utils.ts';
+import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts';
+import { HTTP_STATUS_CODES, HDB_ERROR_MSGS, CHECK_LOGS_WRAPPER } from '../utility/errors/commonErrors.ts';
+
+import logger from '../utility/logging/harper_logger.ts';
+import * as papaParse from 'papaparse';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { chain } from 'stream-chain';
+import StreamArray from 'stream-json/streamers/StreamArray';
+import Batch from 'stream-json/utils/Batch';
+import comp from 'stream-chain/utils/comp';
+import { finished } from 'stream';
+import * as env from '../utility/environment/environmentManager.ts';
+import * as opFuncCaller from '../utility/OperationFunctionCaller.ts';
+import * as AWSConnector from '../utility/AWS/AWSConnector.js';
+import { BulkLoadFileObject, BulkLoadDataObject } from './dataObjects/BulkLoadObjects.js';
+import PermissionResponseObject from '../security/data_objects/PermissionResponseObject.ts';
+import { verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts';
+import { databases } from '../resources/databases.ts';
+import { coerceType } from '../resources/Table.ts';
 
 const CSV_NO_RECORDS_MSG = 'No records parsed from csv file.';
 const TEMP_DOWNLOAD_DIR = `${env.get('HDB_ROOT')}/tmp`;
-const { schemaRegex } = require('../validation/common_validators.js');
+import { schemaRegex } from '../validation/common_validators.ts';
 const HIGHWATERMARK = 1024 * 1024 * 2;
 const MAX_JSON_ARRAY_SIZE = 5000;
 
@@ -39,19 +37,12 @@ const ACCEPTABLE_URL_CONTENT_TYPE_ENUM = {
 	'application/vnd.ms-excel': true,
 };
 
-module.exports = {
-	csvDataLoad,
-	csvURLLoad,
-	csvFileLoad,
-	importFromS3,
-};
-
 /**
  * Load csv values specified as a string in the message 'data' field.
  * @param jsonMessage
  * @returns {Promise<string>}
  */
-async function csvDataLoad(jsonMessage) {
+export async function csvDataLoad(this: any, jsonMessage: any) {
 	let validationMsg = validator.dataObject(jsonMessage);
 	if (validationMsg) {
 		throw handleHDBError(
@@ -64,7 +55,7 @@ async function csvDataLoad(jsonMessage) {
 		);
 	}
 
-	let bulkLoadResult = {};
+	let bulkLoadResult: any = {};
 	try {
 		const mapOfTransforms = createTransformMap(jsonMessage.schema, jsonMessage.table);
 		let parseResults = papaParse.parse(jsonMessage.data, {
@@ -123,7 +114,7 @@ async function csvDataLoad(jsonMessage) {
  * @param jsonMessage
  * @returns {Promise<string>}
  */
-async function csvURLLoad(jsonMessage) {
+export async function csvURLLoad(this: any, jsonMessage: any) {
 	let validationMsg = validator.urlObject(jsonMessage);
 	if (validationMsg) {
 		throw handleHDBError(
@@ -175,7 +166,7 @@ async function csvURLLoad(jsonMessage) {
  * @param jsonMessage
  * @returns {Promise<string>}
  */
-async function csvFileLoad(jsonMessage) {
+export async function csvFileLoad(this: any, jsonMessage: any) {
 	let validationMsg = validator.fileObject(jsonMessage);
 	if (validationMsg) {
 		throw handleHDBError(
@@ -212,7 +203,7 @@ async function csvFileLoad(jsonMessage) {
  * @param jsonMessage
  * @returns {Promise<string>}
  */
-async function importFromS3(jsonMessage) {
+export async function importFromS3(this: any, jsonMessage: any) {
 	let validationMsg = validator.s3FileObject(jsonMessage);
 	if (validationMsg) {
 		throw handleHDBError(
@@ -288,9 +279,9 @@ async function downloadFileFromS3(s3FileName, jsonMessage) {
 		await fs.mkdirp(TEMP_DOWNLOAD_DIR);
 		await fs.writeFile(`${TEMP_DOWNLOAD_DIR}/${s3FileName}`, '', { flag: 'a+' });
 		let tempFileStream = await fs.createWriteStream(tempDownloadLocation);
-		let s3Stream = await AWSConnector.getFileStreamFromS3(jsonMessage);
+		let s3Stream: any = await AWSConnector.getFileStreamFromS3(jsonMessage);
 
-		await new Promise((resolve, reject) => {
+		await new Promise<void>((resolve, reject) => {
 			s3Stream.on('error', function (err) {
 				reject(err);
 			});
@@ -549,7 +540,7 @@ async function callPapaParse(jsonMessage) {
 		let stream = fs.createReadStream(jsonMessage.file_path, { highWaterMark: HIGHWATERMARK });
 		stream.setEncoding('utf8');
 
-		await papaParse.parsePromise(
+		await hdbUtils.parsePromise(
 			stream,
 			validateChunk.bind(null, jsonMessage, attrsPermsErrors),
 			typeFunction.bind(null, mapOfTransforms)
@@ -563,7 +554,7 @@ async function callPapaParse(jsonMessage) {
 		stream = fs.createReadStream(jsonMessage.file_path, { highWaterMark: HIGHWATERMARK });
 		stream.setEncoding('utf8');
 
-		await papaParse.parsePromise(
+		await hdbUtils.parsePromise(
 			stream,
 			insertChunk.bind(null, jsonMessage, insertResults),
 			typeFunction.bind(null, mapOfTransforms)
@@ -618,11 +609,11 @@ async function insertJson(jsonMessage) {
 			(data) => data.value,
 			new Batch({ batchSize: MAX_JSON_ARRAY_SIZE }),
 			comp(async (chunk) => {
-				await validateChunk(jsonMessage, attrsPermsErrors, throwErr, chunk);
+				await validateChunk(jsonMessage, attrsPermsErrors, throwErr, chunk, undefined);
 			}),
 		]);
 
-		await new Promise((resolve, reject) => {
+		await new Promise<void>((resolve, reject) => {
 			finished(jsonStreamer, (err) => {
 				if (err) {
 					reject(err);
@@ -644,11 +635,11 @@ async function insertJson(jsonMessage) {
 			(data) => data.value,
 			new Batch({ batchSize: MAX_JSON_ARRAY_SIZE }),
 			comp(async (chunk) => {
-				await insertChunk(jsonMessage, insertResults, throwErr, chunk);
+				await insertChunk(jsonMessage, insertResults, throwErr, chunk, undefined);
 			}),
 		]);
 
-		await new Promise((resolve, reject) => {
+		await new Promise<void>((resolve, reject) => {
 			finished(jsonStreamerInsert, (err) => {
 				if (err) {
 					reject(err);
@@ -672,7 +663,7 @@ async function insertJson(jsonMessage) {
 }
 
 async function callBulkFileLoad(jsonMsg) {
-	let bulkLoadResult = {};
+	let bulkLoadResult: any = {};
 	try {
 		if (jsonMsg.data && jsonMsg.data.length > 0 && validateColumnNames(jsonMsg.data[0])) {
 			bulkLoadResult = await bulkFileLoad(jsonMsg.data, jsonMsg.schema, jsonMsg.table, jsonMsg.action);

package/core/dataLayer/{delete.js → delete.ts}

@@ -1,31 +1,24 @@
 'use strict';
 
-const bulkDeleteValidator = require('../validation/bulkDeleteValidator.js');
-const deleteValidator = require('../validation/deleteValidator.js');
-const commonUtils = require('../utility/common_utils.js');
-const moment = require('moment');
-const harperLogger = require('../utility/logging/harper_logger.js');
-const { promisify, callbackify } = require('util');
-const terms = require('../utility/hdbTerms.ts');
-const globalSchema = require('../utility/globalSchema.js');
+import bulkDeleteValidator from '../validation/bulkDeleteValidator.ts';
+import deleteValidator from '../validation/deleteValidator.ts';
+import * as commonUtils from '../utility/common_utils.ts';
+import moment from 'moment';
+import harperLogger from '../utility/logging/harper_logger.ts';
+import { promisify, callbackify } from 'util';
+import * as terms from '../utility/hdbTerms.ts';
+import * as globalSchema from '../utility/globalSchema.ts';
 const pGlobalSchema = promisify(globalSchema.getTableSchema);
-const harperBridge = require('./harperBridge/harperBridge.js');
-const { DeleteResponseObject } = require('./DataLayerObjects.js');
-const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
-const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
-const DeleteAuditLogsBeforeResults = require('./harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js');
+const harperBridge = require('./harperBridge/harperBridge').default;
+import { DeleteResponseObject } from './DataLayerObjects.ts';
+import { handleHDBError } from '../utility/errors/hdbError.ts';
+import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
+
+import DeleteAuditLogsBeforeResults from './harperBridge/lmdbBridge/lmdbMethods/DeleteAuditLogsBeforeResults.js';
 
 const SUCCESS_MESSAGE = 'records successfully deleted';
 
 // Callbackified functions
-const cbDeleteRecord = callbackify(deleteRecord);
-
-module.exports = {
-	delete: cbDeleteRecord,
-	deleteRecord,
-	deleteFilesBefore,
-	deleteAuditLogsBefore,
-};
 
 /**
  * Deletes files that have a system date before the date parameter.
@@ -34,7 +27,7 @@ module.exports = {
  *
  * @param deleteObj - the request passed from chooseOperation.
  */
-async function deleteFilesBefore(deleteObj) {
+export async function deleteFilesBefore(deleteObj: any) {
 	let validation = bulkDeleteValidator(deleteObj, 'date');
 	if (validation) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
@@ -81,7 +74,7 @@ async function deleteFilesBefore(deleteObj) {
  *
  * @deprecated This has been deprecated in favor of deleteTransactionLogsBefore.
  */
-async function deleteAuditLogsBefore(deleteObj) {
+export async function deleteAuditLogsBefore(deleteObj: any) {
 	let validation = bulkDeleteValidator(deleteObj, 'timestamp');
 	if (validation) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
@@ -124,7 +117,7 @@ async function deleteAuditLogsBefore(deleteObj) {
  * @param deleteObject
  * @returns {Promise<string>}
  */
-async function deleteRecord(deleteObject) {
+export async function deleteRecord(deleteObject: any) {
 	if (deleteObject.ids) deleteObject.hash_values = deleteObject.ids;
 	let validation = deleteValidator(deleteObject);
 	if (validation) {
@@ -157,11 +150,13 @@ async function deleteRecord(deleteObject) {
 		if (err.message === terms.SEARCH_NOT_FOUND_MESSAGE) {
 			let returnMsg = new DeleteResponseObject();
 			returnMsg.message = terms.SEARCH_NOT_FOUND_MESSAGE;
-			returnMsg.skipped_hashes = deleteObject.hash_values.length;
-			returnMsg.deleted_hashes = 0;
+			returnMsg.skipped_hashes = [deleteObject.hash_values.length];
+			returnMsg.deleted_hashes = [];
 			return returnMsg;
 		}
 
 		throw err;
 	}
 }
+
+export const delete_ = callbackify(deleteRecord);

package/core/dataLayer/{export.js → export.ts}

@@ -1,19 +1,20 @@
 'use strict';
 
-const search = require('./search.js');
-const AWSConnector = require('../utility/AWS/AWSConnector.js');
-const stream = require('stream');
-const hdbUtils = require('../utility/common_utils.js');
-const fs = require('fs-extra');
-const path = require('path');
-const hdbLogger = require('../utility/logging/harper_logger.js');
-const { promisify } = require('util');
-const hdbCommon = require('../utility/common_utils.js');
-const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
-const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
-const { streamAsJSON } = require('../server/serverHelpers/JSONStream.ts');
-const { Upload } = require('@aws-sdk/lib-storage');
-const { toCsvStream } = require('../server/serverHelpers/contentTypes.ts');
+import * as search from './search.ts';
+import * as AWSConnector from '../utility/AWS/AWSConnector.js';
+import * as stream from 'stream';
+import * as hdbUtils from '../utility/common_utils.ts';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import hdbLogger from '../utility/logging/harper_logger.ts';
+import { promisify } from 'util';
+import * as hdbCommon from '../utility/common_utils.ts';
+import { handleHDBError } from '../utility/errors/hdbError.ts';
+import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
+
+import { streamAsJSON } from '../server/serverHelpers/JSONStream.ts';
+let { Upload } = require('@aws-sdk/lib-storage');
+import { toCsvStream } from '../server/serverHelpers/contentTypes.ts';
 
 const VALID_SEARCH_OPERATIONS = ['search_by_value', 'search_by_hash', 'sql', 'search_by_conditions'];
 const VALID_EXPORT_FORMATS = ['json', 'csv'];
@@ -29,17 +30,12 @@ const pSearchByHash = search.searchByHash;
 const pSearchByValue = search.searchByValue;
 const streamFinished = promisify(stream.finished);
 
-module.exports = {
-	export_to_s3,
-	export_local,
-};
-
 /**
  * Allows for exporting and saving to a file system the receiving system has access to
  *
  * @param exportObject
  */
-async function export_local(exportObject) {
+export async function export_local(exportObject: any) {
 	hdbLogger.trace(
 		`export_local request to path: ${exportObject.path}, filename: ${exportObject.filename}, format: ${exportObject.format}`
 	);
@@ -81,7 +77,7 @@ async function export_local(exportObject) {
  * stats the path sent in to verify the path exists, the user has access & the path is a directory
  * @param directoryPath
  */
-async function confirmPath(directoryPath) {
+async function confirmPath(directoryPath: string) {
 	hdbLogger.trace('in confirmPath');
 	if (hdbUtils.isEmptyOrZeroLength(directoryPath)) {
 		throw handleHDBError(
@@ -122,7 +118,7 @@ async function confirmPath(directoryPath) {
  * @param sourceDataFormat
  * @param data
  */
-async function saveToLocal(filePath, sourceDataFormat, data) {
+async function saveToLocal(filePath: string, sourceDataFormat: string, data: any) {
 	hdbLogger.trace('in saveToLocal');
 	if (hdbCommon.isEmptyOrZeroLength(filePath)) {
 		throw handleHDBError(
@@ -189,7 +185,7 @@ async function saveToLocal(filePath, sourceDataFormat, data) {
  * @param exportObject
  * @returns {*}
  */
-async function export_to_s3(exportObject) {
+export async function export_to_s3(exportObject: any) {
 	if (!exportObject.s3 || Object.keys(exportObject.s3).length === 0) {
 		throw handleHDBError(new Error(), HDB_ERROR_MSGS.MISSING_VALUE('S3 object'), HTTP_STATUS_CODES.BAD_REQUEST);
 	}
@@ -303,7 +299,7 @@ async function export_to_s3(exportObject) {
  * @param exportObject
  * @returns {string}
  */
-function exportCoreValidation(exportObject) {
+function exportCoreValidation(exportObject: any) {
 	hdbLogger.trace('in exportCoreValidation');
 	if (hdbUtils.isEmpty(exportObject.format)) {
 		return 'format missing';
@@ -328,7 +324,7 @@ let pSql;
  * determines which search operation to perform and executes it.
  * @param exportObject
  */
-async function getRecords(exportObject) {
+async function getRecords(exportObject: any) {
 	hdbLogger.trace('in getRecords');
 	let operation;
 	let errMsg = undefined;
@@ -350,7 +346,7 @@ async function getRecords(exportObject) {
 			break;
 		case 'sql': {
 			if (!pSql) {
-				const sql = require('../sqlTranslator/index.js');
+				const sql = require('../sqlTranslator/index');
 				pSql = promisify(sql.evaluateSQL);
 			}
 			operation = pSql;

package/core/dataLayer/{getBackup.js → getBackup.ts}

@@ -4,22 +4,20 @@
 
 'use strict';
 
-const harperBridge = require('./harperBridge/harperBridge.js');
+const harperBridge = require('./harperBridge/harperBridge').default;
 // eslint-disable-next-line no-unused-vars
-const GetBackupObject = require('./GetBackupObject.js');
-const hdbUtils = require('../utility/common_utils.js');
-const hdbTerms = require('../utility/hdbTerms.ts');
-const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
-const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
-
-module.exports = getBackup;
+import GetBackupObject from './GetBackupObject.ts';
+import * as hdbUtils from '../utility/common_utils.ts';
+import * as hdbTerms from '../utility/hdbTerms.ts';
+import { handleHDBError } from '../utility/errors/hdbError.ts';
+import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
 
 /**
  *
  * @param {GetBackupObject} getBackupObject
  * @returns {Promise<void>}
  */
-async function getBackup(getBackupObject) {
+export default async function getBackup(getBackupObject: any) {
 	if (hdbUtils.isEmpty(getBackupObject.schema)) {
 		throw new Error(HDB_ERROR_MSGS.SCHEMA_REQUIRED_ERR);
 	}

package/core/dataLayer/harperBridge/BridgeMethods.ts

@@ -0,0 +1,102 @@
+'use strict';
+
+/**
+ * BridgeMethods Class provides a framework for all HarperBridge method classes
+ */
+
+export class BridgeMethods {
+	/** @param {...any} _args */
+	createSchema(..._args: any[]): any {
+		throw new Error('createSchema bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	dropSchema(..._args: any[]): any {
+		throw new Error('dropSchema bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	createTable(..._args: any[]): any {
+		throw new Error('createTable bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	dropTable(..._args: any[]): any {
+		throw new Error('dropTable bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	createRecords(..._args: any[]): any {
+		throw new Error('createRecords bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	updateRecords(..._args: any[]): any {
+		throw new Error('updateRecords bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async upsertRecords(..._args: any[]): Promise<any> {
+		throw new Error('upsertRecords bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	deleteRecords(..._args: any[]): any {
+		throw new Error('deleteRecords bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	createAttribute(..._args: any[]): any {
+		throw new Error('createAttribute bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	dropAttribute(..._args: any[]): any {
+		throw new Error('dropAttribute bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	searchByConditions(..._args: any[]): any {
+		throw new Error('searchByConditions bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	searchByHash(..._args: any[]): any {
+		throw new Error('searchByHash bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	searchByValue(..._args: any[]): any {
+		throw new Error('searchByValue bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	getDataByHash(..._args: any[]): any {
+		throw new Error('getDataByHash bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async getDataByValue(..._args: any[]): Promise<any> {
+		throw new Error('getDataByValue bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async deleteRecordsBefore(..._args: any[]): Promise<any> {
+		throw new Error('deleteRecordsBefore bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async deleteAuditLogsBefore(..._args: any[]): Promise<any> {
+		throw new Error('deleteAuditLogsBefore bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async deleteTransactionLogsBefore(..._args: any[]): Promise<any> {
+		throw new Error('deleteTransactionLogsBefore bridge method is not defined');
+	}
+
+	/** @param {...any} _args */
+	async readAuditLog(..._args: any[]): Promise<any> {
+		throw new Error('readAuditLog bridge method is not defined');
+	}
+}