@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js

@@ -0,0 +1,24 @@
+'use strict';
+const LMDBTransactionObject = require('./LMDBTransactionObject.js');
+const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM;
+
+/**
+ * class to define an update transaction
+ */
+class LMDBUpdateTransactionObject extends LMDBTransactionObject {
+	/**
+	 * @param {Array.<Object>} records - records updated
+	 * @param {Array.<Object>} originalRecords - original state of records that were updated
+	 * @param {string} userName - username that executed the transaction
+	 * @param {number} timestamp - timestamp of transaction
+	 * @param {[String|Number]} hash_values
+	 * @param {any} origin
+	 */
+	constructor(records, originalRecords, userName, timestamp, hash_values, origin = undefined) {
+		super(OPERATIONS_ENUM.UPDATE, userName, timestamp, hash_values, origin);
+		this.records = records;
+		this.original_records = originalRecords;
+	}
+}
+
+module.exports = LMDBUpdateTransactionObject;

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js

@@ -0,0 +1,24 @@
+'use strict';
+const LMDBTransactionObject = require('./LMDBTransactionObject.js');
+const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM;
+
+/**
+ * class to define an update transaction
+ */
+class LMDBUpsertTransactionObject extends LMDBTransactionObject {
+	/**
+	 * @param {Array.<Object>} records - records updated
+	 * @param {Array.<Object>} originalRecords - original state of records that were updated
+	 * @param {string} userName - username that executed the transaction
+	 * @param {number} timestamp - timestamp of transaction
+	 * @param {[String|Number]} hash_values
+	 * @param {any} origin
+	 */
+	constructor(records, originalRecords, userName, timestamp, hash_values, origin = undefined) {
+		super(OPERATIONS_ENUM.UPSERT, userName, timestamp, hash_values, origin);
+		this.records = records;
+		this.original_records = originalRecords;
+	}
+}
+
+module.exports = LMDBUpsertTransactionObject;

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/TableSizeObject.js

@@ -0,0 +1,25 @@
+'use strict';
+
+/**
+ * represents the table size entry for a table
+ */
+class TableSizeObject {
+	/**
+	 * @param {String} schema
+	 * @param {String} table
+	 * @param {Number} tableSize - data size of the table in bytes
+	 * @param {Number} recordCount - number of entries in the table
+	 * @param {Number} transactionLogSize - number of entries in the transaction log
+	 * @param {Number} transactionLogRecordCount - data size of the transaction log in bytes
+	 */
+	constructor(schema, table, tableSize = 0, recordCount = 0, transactionLogSize = 0, transactionLogRecordCount = 0) {
+		this.schema = schema;
+		this.table = table;
+		this.table_size = tableSize;
+		this.record_count = recordCount;
+		this.transaction_log_size = transactionLogSize;
+		this.transaction_log_record_count = transactionLogRecordCount;
+	}
+}
+
+module.exports = TableSizeObject;

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+const searchValidator = require('../../../../validation/searchValidator.js');
+const { getSchemaPath } = require('./initializePaths.js');
+
+module.exports = initialize;
+
+/**
+ *
+ * @param searchObject
+ * @returns {*}
+ */
+function initialize(searchObject) {
+	const validationError = searchValidator(searchObject, 'hashes');
+	if (validationError) {
+		throw validationError;
+	}
+	let envBasePath = getSchemaPath(searchObject.schema, searchObject.table);
+	return environmentUtility.openEnvironment(envBasePath, searchObject.table);
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const hdbUtils = require('../../../../utility/common_utils.js');
+const env = require('../../../../utility/environment/environmentManager.js');
+const path = require('path');
+const minimist = require('minimist');
+const fs = require('fs-extra');
+const _ = require('lodash');
+env.initSync();
+
+const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms;
+let BASE_SCHEMA_PATH = undefined;
+let SYSTEM_SCHEMA_PATH = undefined;
+let TRANSACTION_STORE_PATH = undefined;
+
+/**
+ * when HDB is not yet installed we do not yet know the base path and an error is thrown if we do a standard const, so we create a getter
+ * @returns {string|*}
+ */
+function getBaseSchemaPath() {
+	if (BASE_SCHEMA_PATH !== undefined) {
+		return BASE_SCHEMA_PATH;
+	}
+
+	if (env.getHdbBasePath() !== undefined) {
+		BASE_SCHEMA_PATH =
+			env.get(CONFIG_PARAMS.STORAGE_PATH) || path.join(env.getHdbBasePath(), hdbTerms.DATABASES_DIR_NAME);
+		return BASE_SCHEMA_PATH;
+	}
+}
+
+/**
+ * when HDB is not yet installed we do not yet know the base path and an error is thrown if we do a standard const, so we create a getter
+ * @returns {string|*}
+ */
+function getSystemSchemaPath() {
+	if (SYSTEM_SCHEMA_PATH !== undefined) {
+		return SYSTEM_SCHEMA_PATH;
+	}
+
+	if (env.getHdbBasePath() !== undefined) {
+		SYSTEM_SCHEMA_PATH = getSchemaPath(SYSTEM_SCHEMA_NAME);
+		return SYSTEM_SCHEMA_PATH;
+	}
+}
+
+function getTransactionAuditStoreBasePath() {
+	if (TRANSACTION_STORE_PATH !== undefined) {
+		return TRANSACTION_STORE_PATH;
+	}
+
+	if (env.getHdbBasePath() !== undefined) {
+		TRANSACTION_STORE_PATH =
+			env.get(hdbTerms.CONFIG_PARAMS.STORAGE_AUDIT_PATH) ||
+			path.join(env.getHdbBasePath(), hdbTerms.TRANSACTIONS_DIR_NAME);
+		return TRANSACTION_STORE_PATH;
+	}
+}
+
+function getTransactionAuditStorePath(schema, table) {
+	let schemaConfig = env.get(CONFIG_PARAMS.DATABASES)?.[schema];
+	return (
+		(table && schemaConfig?.tables?.[table]?.auditPath) ||
+		schemaConfig?.auditPath ||
+		path.join(getTransactionAuditStoreBasePath(), schema.toString())
+	);
+}
+
+function getSchemaPath(schema, table) {
+	schema = schema.toString();
+	table = table ? table.toString() : table;
+	let schemaConfig = env.get(hdbTerms.CONFIG_PARAMS.DATABASES)?.[schema];
+	return (table && schemaConfig?.tables?.[table]?.path) || schemaConfig?.path || path.join(getBaseSchemaPath(), schema);
+}
+
+/**
+ * It is possible to set where the system schema/table files reside. This function will check for CLI/env vars
+ * on install and update accordingly.
+ * @param schema
+ * @param table
+ * @returns {string|string|*}
+ */
+function initSystemSchemaPaths(schema, table) {
+	schema = schema.toString();
+	table = table.toString();
+
+	// Check to see if there are any CLI or env args related to schema/table path
+	const args = process.env;
+	Object.assign(args, minimist(process.argv));
+
+	const schemaConfJson = args[CONFIG_PARAMS.DATABASES.toUpperCase()];
+	if (schemaConfJson) {
+		let schemasConf;
+		try {
+			schemasConf = JSON.parse(schemaConfJson);
+		} catch (err) {
+			if (!hdbUtils.isObject(schemaConfJson)) throw err;
+			schemasConf = schemaConfJson;
+		}
+
+		for (const schemaConf of schemasConf) {
+			const systemSchemaConf = schemaConf[SYSTEM_SCHEMA_NAME];
+			if (!systemSchemaConf) continue;
+			let schemasObj = env.get(CONFIG_PARAMS.DATABASES);
+			schemasObj = schemasObj ?? {};
+
+			// If path var exists for system table add it to schemas prop and return path.
+			const systemTablePath = systemSchemaConf?.tables?.[table]?.[DATABASES_PARAM_CONFIG.PATH];
+			if (systemTablePath) {
+				_.set(
+					schemasObj,
+					[SYSTEM_SCHEMA_NAME, DATABASES_PARAM_CONFIG.TABLES, table, DATABASES_PARAM_CONFIG.PATH],
+					systemTablePath
+				);
+				env.setProperty(CONFIG_PARAMS.DATABASES, schemasObj);
+				return systemTablePath;
+			}
+
+			// If path exists for system schema add it to schemas prop and return path.
+			const systemSchemaPath = systemSchemaConf?.[DATABASES_PARAM_CONFIG.PATH];
+			if (systemSchemaPath) {
+				_.set(schemasObj, [SYSTEM_SCHEMA_NAME, DATABASES_PARAM_CONFIG.PATH], systemSchemaPath);
+				env.setProperty(CONFIG_PARAMS.DATABASES, schemasObj);
+				return systemSchemaPath;
+			}
+		}
+	}
+
+	// If storagePath is passed use that to determine location
+	const storagePath = args[CONFIG_PARAMS.STORAGE_PATH.toUpperCase()];
+	if (storagePath) {
+		if (!fs.pathExistsSync(storagePath)) throw new Error(storagePath + ' does not exist');
+		const storageSchemaPath = path.join(storagePath, schema);
+		fs.mkdirsSync(storageSchemaPath);
+		env.setProperty(CONFIG_PARAMS.STORAGE_PATH, storagePath);
+
+		return storageSchemaPath;
+	}
+
+	// Default to default location
+	return getSystemSchemaPath();
+}
+function resetPaths() {
+	BASE_SCHEMA_PATH = undefined;
+	SYSTEM_SCHEMA_PATH = undefined;
+	TRANSACTION_STORE_PATH = undefined;
+}
+module.exports = {
+	getBaseSchemaPath,
+	getSystemSchemaPath,
+	getTransactionAuditStorePath,
+	getTransactionAuditStoreBasePath,
+	getSchemaPath,
+	initSystemSchemaPaths,
+	resetPaths,
+};

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const hUtils = require('../../../../utility/common_utils.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const logger = require('../../../../utility/logging/harper_logger.js');
+const lmdbCreateAttribute = require('../lmdbMethods/lmdbCreateAttribute.js');
+const LMDBCreateAttributeObject = require('./LMDBCreateAttributeObject.js');
+const signalling = require('../../../../utility/signalling.js');
+const { SchemaEventMsg } = require('../../../../server/threads/itc.js');
+
+const ATTRIBUTE_ALREADY_EXISTS = 'already exists in';
+
+module.exports = lmdbCheckForNewAttributes;
+
+/**
+ * Uses a utility function to check if there are any new attributes that dont exist. Utility function
+ * references the global schema.
+ * @param hdbAuthHeader
+ * @param tableSchema
+ * @param dataAttributes
+ */
+async function lmdbCheckForNewAttributes(hdbAuthHeader, tableSchema, dataAttributes) {
+	if (hUtils.isEmptyOrZeroLength(dataAttributes)) {
+		return dataAttributes;
+	}
+
+	let rawAttributes = [];
+	if (!hUtils.isEmptyOrZeroLength(tableSchema.attributes)) {
+		tableSchema.attributes.forEach((attribute) => {
+			rawAttributes.push(attribute.attribute);
+		});
+	}
+
+	let new_attributes = dataAttributes.filter((attribute) => rawAttributes.indexOf(attribute) < 0);
+
+	if (new_attributes.length === 0) {
+		return new_attributes;
+	}
+
+	await Promise.all(
+		new_attributes.map(async (attribute) => {
+			await createNewAttribute(hdbAuthHeader, tableSchema.schema, tableSchema.name, attribute);
+		})
+	);
+
+	return new_attributes;
+}
+
+/**
+ * check the existing schema and creates new attributes based on what the incoming records have
+ * @param hdbAuthHeader
+ * @param schema
+ * @param table
+ * @param attribute
+ */
+async function createNewAttribute(hdbAuthHeader, schema, table, attribute) {
+	let attributeObject = new LMDBCreateAttributeObject(schema, table, attribute, undefined, true);
+
+	if (hdbAuthHeader) {
+		attributeObject.hdb_auth_header = hdbAuthHeader;
+	}
+
+	try {
+		await createAttribute(attributeObject);
+	} catch (e) {
+		//if the attribute already exists we do not want to stop the insert
+		if (typeof e === 'object' && e.message !== undefined && e.message.includes(ATTRIBUTE_ALREADY_EXISTS)) {
+			logger.warn(`attribute ${schema}.${table}.${attribute} already exists`);
+		} else {
+			throw e;
+		}
+	}
+}
+
+/**
+ *
+ * @param {LMDBCreateAttributeObject} createAttributeObject
+ * @returns {Promise<*>}
+ */
+async function createAttribute(createAttributeObject) {
+	let attributeStructure;
+	attributeStructure = await lmdbCreateAttribute(createAttributeObject);
+	signalling.signalSchemaChange(
+		new SchemaEventMsg(
+			process.pid,
+			hdbTerms.OPERATIONS_ENUM.CREATE_ATTRIBUTE,
+			createAttributeObject.schema,
+			createAttributeObject.table,
+			createAttributeObject.attribute
+		)
+	);
+
+	return attributeStructure;
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const fs = require('fs-extra');
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js');
+const lmdbTerms = require('../../../../utility/lmdb/terms.js');
+// eslint-disable-next-line no-unused-vars
+const CreateTableObject = require('../../../CreateTableObject.js');
+
+module.exports = createTransactionsAuditEnvironment;
+
+/**
+ * Creates the environment to hold transactions
+ * @param {CreateTableObject} tableCreateObj
+ * @returns {Promise<lmdb.RootDatabase>}
+ */
+async function createTransactionsAuditEnvironment(tableCreateObj) {
+	let env;
+	try {
+		//create transactions environment for table
+		let transactionPath = getTransactionAuditStorePath(tableCreateObj.schema, tableCreateObj.table);
+		await fs.mkdirp(transactionPath);
+		env = await environmentUtility.createEnvironment(transactionPath, tableCreateObj.table, true);
+	} catch (e) {
+		e.message = `unable to create transactions audit environment for ${tableCreateObj.schema}.${tableCreateObj.table} due to: ${e.message}`;
+		throw e;
+	}
+
+	try {
+		//create dbis for transactions environment
+		environmentUtility.createDBI(env, lmdbTerms.TRANSACTIONS_DBI_NAMES_ENUM.TIMESTAMP, false, true);
+		environmentUtility.createDBI(env, lmdbTerms.TRANSACTIONS_DBI_NAMES_ENUM.HASH_VALUE, true, false);
+		environmentUtility.createDBI(env, lmdbTerms.TRANSACTIONS_DBI_NAMES_ENUM.USER_NAME, true, false);
+	} catch (e) {
+		e.message = `unable to create dbi for ${tableCreateObj.schema}.${tableCreateObj.table} due to: ${e.message}`;
+		throw e;
+	}
+	return env;
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const TableSizeObject = require('./TableSizeObject.js');
+const log = require('../../../../utility/logging/harper_logger.js');
+const { getDatabases } = require('../../../../resources/databases.ts');
+
+module.exports = lmdbGetTableSize;
+
+/**
+ * calculates the number of entries & data size in bytes for a table & its transaction log
+ * @param tableObject
+ * @returns {Promise<TableSizeObject>}
+ */
+async function lmdbGetTableSize(tableObject) {
+	let tableStats = new TableSizeObject();
+	try {
+		//get the table record count
+		let table = getDatabases()[tableObject.schema]?.[tableObject.name];
+
+		let dbiStat = table.primaryStore.getStats();
+
+		//get the txn log record count
+		let txnDbiStat = table.auditStore?.getStats();
+
+		tableStats.schema = tableObject.schema;
+		tableStats.table = tableObject.name;
+		tableStats.record_count = dbiStat.entryCount;
+		tableStats.transaction_log_record_count = txnDbiStat.entryCount;
+	} catch (e) {
+		log.warn(`unable to stat table dbi due to ${e}`);
+	}
+
+	return tableStats;
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js

@@ -0,0 +1,100 @@
+'use strict';
+
+// eslint-disable-next-line no-unused-vars
+const InsertObject = require('../../../InsertObject.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const hdbUtils = require('../../../../utility/common_utils.js');
+const log = require('../../../../utility/logging/harper_logger.js');
+const uuid = require('uuid');
+const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.js');
+const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
+
+module.exports = processRows;
+
+/**
+ * parses the records and validates the hash value for each row as well as adding updated/created time stamps
+ * @param {InsertObject} insertObj
+ * @param {Array.<String>} attributes
+ * @param {String} hash_attribute
+ */
+function processRows(insertObj, attributes, hash_attribute) {
+	for (let x = 0; x < attributes.length; x++) {
+		validateAttribute(attributes[x]);
+	}
+
+	let { records } = insertObj;
+
+	// Iterates through array of record objects and validates their hash
+	for (let x = 0; x < records.length; x++) {
+		let record = records[x];
+		validateHash(record, hash_attribute, insertObj.operation);
+	}
+}
+processRows.validateAttribute = validateAttribute;
+
+/**
+ * Validates that attribute is under max size and is not null, undefined or empty.
+ * @param attribute
+ */
+function validateAttribute(attribute) {
+	if (Buffer.byteLength(String(attribute)) > hdbTerms.INSERT_MAX_CHARACTER_SIZE) {
+		throw handleHDBError(
+			new Error(),
+			HDB_ERROR_MSGS.ATTR_NAME_LENGTH_ERR(attribute),
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+
+	if (hdbUtils.isEmptyOrZeroLength(attribute) || hdbUtils.isEmpty(attribute.trim())) {
+		throw handleHDBError(
+			new Error(),
+			HDB_ERROR_MSGS.ATTR_NAME_NULLISH_ERR,
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+}
+
+/**
+ * Validates hash value exists and under max char size. If the operation is 'insert' and the hash doesn't exist it
+ * will create one.
+ * @param record
+ * @param hash_attribute
+ * @param operation
+ */
+function validateHash(record, hash_attribute, operation) {
+	if (!record.hasOwnProperty(hash_attribute) || hdbUtils.isEmptyOrZeroLength(record[hash_attribute])) {
+		if (operation === hdbTerms.OPERATIONS_ENUM.INSERT || operation === hdbTerms.OPERATIONS_ENUM.UPSERT) {
+			record[hash_attribute] = uuid.v4();
+			//return here since the rest of the validations do not apply
+			return;
+		}
+
+		log.error('Update transaction aborted due to record with no hash value:', record);
+		throw handleHDBError(
+			new Error(),
+			HDB_ERROR_MSGS.RECORD_MISSING_HASH_ERR,
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+
+	if (Buffer.byteLength(String(record[hash_attribute])) > hdbTerms.INSERT_MAX_CHARACTER_SIZE) {
+		log.error(record);
+		throw handleHDBError(
+			new Error(),
+			HDB_ERROR_MSGS.HASH_VAL_LENGTH_ERR,
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+}