@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/validation/fileLoadValidator.js

@@ -0,0 +1,153 @@
+const clone = require('clone');
+const validator = require('./validationWrapper.js');
+const commonUtils = require('../utility/common_utils.js');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const fs = require('fs');
+const joi = require('joi');
+const { string } = joi.types();
+const { hdbErrors, handleHDBError } = require('../utility/errors/hdbError.js');
+const { HTTP_STATUS_CODES } = hdbErrors;
+
+const { commonValidators } = require('./common_validators.js');
+
+const isRequiredString = ' is required';
+
+const actions = ['insert', 'update', 'upsert'];
+const constraints = {
+	database: {
+		presence: false,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	schema: {
+		presence: false,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	table: {
+		presence: true,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	action: {
+		inclusion: {
+			within: actions,
+			message: 'is required and must be either insert, update, or upsert',
+		},
+	},
+	file_path: {},
+	csv_url: {
+		url: {
+			allowLocal: true,
+		},
+	},
+	data: {},
+	passthrough_headers: {},
+};
+
+const baseJoiSchema = {
+	schema: string.required(),
+	table: string.required(),
+	action: string.valid('insert', 'update', 'upsert'),
+};
+
+const { AWS_ACCESS_KEY, AWS_SECRET, AWS_BUCKET, AWS_FILE_KEY, REGION } = hdbTerms.S3_BUCKET_AUTH_KEYS;
+
+const s3Constraints = {
+	s3: {
+		presence: true,
+	},
+	[`s3.${AWS_ACCESS_KEY}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_SECRET}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_BUCKET}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_FILE_KEY}`]: {
+		presence: true,
+		type: 'String',
+		hasValidFileExt: ['.csv', '.json'],
+	},
+	[`s3.${REGION}`]: {
+		presence: true,
+		type: 'String',
+	},
+};
+
+const dataConstraints = clone(constraints);
+dataConstraints.data.presence = {
+	message: isRequiredString,
+};
+
+const fileConstraints = clone(constraints);
+fileConstraints.file_path.presence = {
+	message: isRequiredString,
+};
+
+const s3FileConstraints = Object.assign(clone(constraints), s3Constraints);
+
+const urlSchema = clone(baseJoiSchema);
+urlSchema.csv_url = string.uri().messages({ 'string.uri': "'csv_url' must be a valid url" }).required();
+urlSchema.passthrough_headers = joi.object();
+
+function dataObject(object) {
+	let validateRes = validator.validateObject(object, dataConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+function urlObject(object) {
+	let validateRes = validator.validateBySchema(object, joi.object(urlSchema));
+	return postValidateChecks(object, validateRes);
+}
+
+function fileObject(object) {
+	let validateRes = validator.validateObject(object, fileConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+function s3FileObject(object) {
+	let validateRes = validator.validateObject(object, s3FileConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+/**
+ * Post validate module checks, confirms schema and table exist.
+ * If file upload - checks that it exists, permissions and size.
+ */
+function postValidateChecks(object, validateRes) {
+	if (!validateRes) {
+		let msg = commonUtils.checkGlobalSchemaTable(object.schema, object.table);
+		if (msg) {
+			return handleHDBError(new Error(), msg, HTTP_STATUS_CODES.BAD_REQUEST);
+		}
+
+		if (object.operation === hdbTerms.OPERATIONS_ENUM.CSV_FILE_LOAD) {
+			try {
+				fs.accessSync(object.file_path, fs.constants.R_OK | fs.constants.F_OK);
+			} catch (err) {
+				if (err.code === hdbTerms.NODE_ERROR_CODES.ENOENT) {
+					return handleHDBError(err, `No such file or directory ${err.path}`, HTTP_STATUS_CODES.BAD_REQUEST);
+				}
+
+				if (err.code === hdbTerms.NODE_ERROR_CODES.EACCES) {
+					return handleHDBError(err, `Permission denied ${err.path}`, HTTP_STATUS_CODES.BAD_REQUEST);
+				}
+				return handleHDBError(err);
+			}
+		}
+	}
+	return validateRes;
+}
+
+module.exports = {
+	dataObject,
+	urlObject,
+	fileObject,
+	s3FileObject,
+};

package/validation/insertValidator.js

@@ -0,0 +1,40 @@
+const { hdbTable, hdbDatabase } = require('./common_validators.js');
+const validator = require('./validationWrapper.js');
+const Joi = require('joi');
+const INVALID_ATTRIBUTE_NAMES = {
+	undefined: 'undefined',
+	null: 'null',
+};
+
+const customRecordsVal = (value, helpers) => {
+	const attributes = Object.keys(value);
+	const attributesLength = attributes.length;
+	let errorMsg = undefined;
+	for (let i = 0; i < attributesLength; i++) {
+		const attribute = attributes[i];
+		if (!attribute || attribute.length === 0 || INVALID_ATTRIBUTE_NAMES[attribute] !== undefined) {
+			if (errorMsg === undefined) {
+				errorMsg = `Invalid attribute name: '${attribute}'`;
+			} else {
+				errorMsg += `. Invalid attribute name: '${attribute}'`;
+			}
+		}
+	}
+
+	if (errorMsg) {
+		return helpers.message(errorMsg);
+	}
+
+	return value;
+};
+
+const insertSchema = Joi.object({
+	database: hdbDatabase,
+	schema: hdbDatabase,
+	table: hdbTable,
+	records: Joi.array().items(Joi.object().custom(customRecordsVal)).required(),
+});
+
+module.exports = function (insertObject) {
+	return validator.validateBySchema(insertObject, insertSchema);
+};

package/validation/installValidator.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const Joi = require('joi');
+const { string, number } = Joi.types();
+const fs = require('fs-extra');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const path = require('path');
+const validator = require('../validation/validationWrapper.js');
+
+module.exports = installValidator;
+
+/**
+ * Used to validate any command or environment variables used passed to install.
+ * @param param
+ * @returns {*}
+ */
+function installValidator(param) {
+	const installSchema = Joi.object({
+		[hdbTerms.INSTALL_PROMPTS.ROOTPATH]: Joi.custom(validateRootAvailable),
+		[hdbTerms.INSTALL_PROMPTS.OPERATIONSAPI_NETWORK_PORT]: Joi.alternatives([number.min(0), string]).allow(
+			'null',
+			null
+		),
+		[hdbTerms.INSTALL_PROMPTS.TC_AGREEMENT]: string.valid('yes', 'YES', 'Yes'),
+	});
+
+	return validator.validateBySchema(param, installSchema);
+}
+
+function validateRootAvailable(value, helpers) {
+	if (
+		fs.existsSync(path.join(value, 'system/hdb_user/data.mdb')) ||
+		fs.existsSync(path.join(value, 'system/hdb_user.mdb'))
+	) {
+		return helpers.message(`'${value}' is already in use. Please enter a different path.`);
+	}
+}

package/validation/readLogValidator.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const envMangr = require('../utility/environment/environmentManager.js');
+const Joi = require('joi');
+const validator = require('./validationWrapper.js');
+const moment = require('moment');
+const fs = require('fs-extra');
+const path = require('path');
+const _ = require('lodash');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const { LOG_LEVELS } = require('../utility/hdbTerms.ts');
+
+const LOG_DATE_FORMAT = 'YYYY-MM-DD hh:mm:ss';
+const INSTALL_LOG_LOCATION = path.resolve(__dirname, `../logs`);
+
+module.exports = function (object) {
+	return validator.validateBySchema(object, readLogSchema);
+};
+
+const readLogSchema = Joi.object({
+	from: Joi.custom(validateDatetime),
+	until: Joi.custom(validateDatetime),
+	to: Joi.custom(validateDatetime),
+	level: Joi.valid(
+		LOG_LEVELS.NOTIFY,
+		LOG_LEVELS.FATAL,
+		LOG_LEVELS.ERROR,
+		LOG_LEVELS.WARN,
+		LOG_LEVELS.INFO,
+		LOG_LEVELS.DEBUG,
+		LOG_LEVELS.TRACE
+	),
+	order: Joi.valid('asc', 'desc'),
+	limit: Joi.number().min(1),
+	start: Joi.number().min(0),
+	log_name: Joi.custom(validateReadLogPath),
+	filter: Joi.string(),
+});
+
+function validateDatetime(value, helpers) {
+	if (moment(value, moment.ISO_8601).format(LOG_DATE_FORMAT) === 'Invalid date') {
+		return helpers.message(`'${helpers.state.path[0]}' date '${value}' is invalid.`);
+	}
+}
+
+function validateReadLogPath(value, helpers) {
+	const processLogName = _.invert(hdbTerms.LOG_NAMES);
+	if (processLogName[value] === undefined) {
+		return helpers.message(`'log_name' '${value}' is invalid.`);
+	}
+
+	const logPath = envMangr.get(hdbTerms.HDB_SETTINGS_NAMES.LOG_PATH_KEY);
+	const logName = value === undefined ? hdbTerms.LOG_NAMES.HDB : value;
+	const readLogPath =
+		logName === hdbTerms.LOG_NAMES.INSTALL
+			? path.join(INSTALL_LOG_LOCATION, hdbTerms.LOG_NAMES.INSTALL)
+			: path.join(logPath, logName);
+
+	let exists = fs.existsSync(readLogPath);
+	if (exists) {
+		return null;
+	}
+	return helpers.message(`'log_name' '${value}' does not exist.`);
+}

package/validation/role_validation.js

@@ -0,0 +1,320 @@
+const validate = require('validate.js'),
+	validator = require('./validationWrapper.js'),
+	terms = require('../utility/hdbTerms.ts'),
+	{ validateOperations } = require('../utility/operationPermissions.ts'),
+	{ handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
+
+const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
+
+const constraintsTemplate = () => ({
+	role: {
+		presence: true,
+		format: '[\\w\\-\\_]+',
+	},
+	id: {
+		presence: true,
+		format: '[\\w\\-\\_]+',
+	},
+	permission: {
+		presence: true,
+	},
+});
+
+const STRUCTURE_USER_ENUM = {
+	STRUCTURE_USER: 'structure_user',
+};
+
+const OPERATIONS_KEY = 'operations';
+
+const ROLE_TYPES = Object.values(terms.ROLE_TYPES_ENUM);
+const ATTR_PERMS_KEY = 'attribute_permissions';
+const ATTR_NAME_KEY = 'attribute_name';
+const { PERMS_CRUD_ENUM } = terms;
+const TABLE_PERM_KEYS = [ATTR_PERMS_KEY, ...Object.values(PERMS_CRUD_ENUM)];
+const ATTR_CRU_KEYS = [PERMS_CRUD_ENUM.READ, PERMS_CRUD_ENUM.INSERT, PERMS_CRUD_ENUM.UPDATE];
+const ATTR_PERMS_KEYS = [ATTR_NAME_KEY, ...ATTR_CRU_KEYS];
+
+function addRoleValidation(object) {
+	const constraints = constraintsTemplate();
+	constraints.role.presence = true;
+	constraints.id.presence = false;
+	constraints.permission.presence = true;
+	return customValidate(object, constraints);
+}
+
+function alterRoleValidation(object) {
+	const constraints = constraintsTemplate();
+	constraints.role.presence = false;
+	constraints.id.presence = true;
+	constraints.permission.presence = true;
+	return customValidate(object, constraints);
+}
+
+function dropRoleValidation(object) {
+	const constraints = constraintsTemplate();
+	constraints.role.presence = false;
+	constraints.id.presence = true;
+	constraints.permission.presence = false;
+	return validator.validateObject(object, constraints);
+}
+
+const ALLOWED_JSON_KEYS = ['operation', 'role', 'id', 'permission', 'hdb_user', 'access'];
+
+function customValidate(object, constraints) {
+	let validationErrors = {
+		main_permissions: [],
+		schema_permissions: {},
+	};
+
+	const jsonMsgKeys = Object.keys(object);
+
+	//Check to confirm that keys in JSON body are valid
+	const invalidKeys = [];
+	for (let i = 0, arrLength = jsonMsgKeys.length; i < arrLength; i++) {
+		if (!ALLOWED_JSON_KEYS.includes(jsonMsgKeys[i])) {
+			invalidKeys.push(jsonMsgKeys[i]);
+		}
+	}
+	if (invalidKeys.length > 0) {
+		addPermError(HDB_ERROR_MSGS.INVALID_ROLE_JSON_KEYS(invalidKeys), validationErrors);
+	}
+
+	let validateResult = validator.validateObject(object, constraints);
+	if (validateResult) {
+		validateResult.message.split(',').forEach((validationErr) => {
+			addPermError(validationErr, validationErrors);
+		});
+	}
+
+	//need this check to avoid unexpected errors if someone doesn't have permissions key included in request
+	if (object.permission) {
+		//check if role is SU or CU and has perms included
+		const suPermsError = validateNoSUPerms(object);
+		if (suPermsError) {
+			addPermError(suPermsError, validationErrors);
+		}
+		//check if cu or su values, if included, are booleans
+		ROLE_TYPES.forEach((role) => {
+			if (object.permission[role] && !validate.isBoolean(object.permission[role])) {
+				addPermError(HDB_ERROR_MSGS.SU_CU_ROLE_BOOLEAN_ERROR(role), validationErrors);
+			}
+		});
+	}
+
+	for (let item in object.permission) {
+		if (ROLE_TYPES.indexOf(item) < 0) {
+			//validate the user type 'structure_user'.  acceptable data type is boolean or array of strings (this would be array of accepted schemas to interact with)
+			if (item === STRUCTURE_USER_ENUM.STRUCTURE_USER) {
+				let structureUserPerm = object.permission[item];
+
+				//boolean is valid, move on
+				if (typeof structureUserPerm === 'boolean') {
+					continue;
+				}
+
+				//array is valid check to make sure each entry is actually a schema.
+				if (Array.isArray(structureUserPerm)) {
+					for (let k = 0, length = structureUserPerm.length; k < length; k++) {
+						let schemaPerm = structureUserPerm[k];
+						if (!global.hdb_schema[schemaPerm]) {
+							addPermError(HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(schemaPerm), validationErrors);
+						}
+					}
+					continue;
+				}
+
+				//if we end up here then this is an invalid data type
+				addPermError(HDB_ERROR_MSGS.STRUCTURE_USER_ROLE_TYPE_ERROR(item), validationErrors);
+				continue;
+			}
+
+			// validate operations: must be an array of valid operation names and/or group names
+			if (item === OPERATIONS_KEY) {
+				const opUserPerm = object.permission[item];
+
+				if (!Array.isArray(opUserPerm)) {
+					addPermError(HDB_ERROR_MSGS.OPERATIONS_MUST_BE_ARRAY, validationErrors);
+					continue;
+				}
+
+				const invalidOp = validateOperations(opUserPerm);
+				if (invalidOp !== null) {
+					addPermError(HDB_ERROR_MSGS.INVALID_OPERATIONS_OP(invalidOp), validationErrors);
+				}
+				continue;
+			}
+
+			let schema = object.permission[item];
+			//validate that schema exists
+			if (!item || !global.hdb_schema[item]) {
+				addPermError(HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(item), validationErrors);
+				continue;
+			}
+			if (schema.tables) {
+				for (let t in schema.tables) {
+					let table = schema.tables[t];
+					//validate that table exists in schema
+					if (!t || !global.hdb_schema[item][t]) {
+						addPermError(HDB_ERROR_MSGS.TABLE_NOT_FOUND(item, t), validationErrors);
+						continue;
+					}
+
+					//validate all table perm keys are valid
+					Object.keys(table).forEach((tableKey) => {
+						if (!TABLE_PERM_KEYS.includes(tableKey)) {
+							addPermError(HDB_ERROR_MSGS.INVALID_PERM_KEY(tableKey), validationErrors, item, t);
+						}
+					});
+
+					//validate table CRUD perms
+					Object.values(PERMS_CRUD_ENUM).forEach((permKey) => {
+						if (!validate.isDefined(table[permKey])) {
+							addPermError(HDB_ERROR_MSGS.TABLE_PERM_MISSING(permKey), validationErrors, item, t);
+						} else if (!validate.isBoolean(table[permKey])) {
+							addPermError(HDB_ERROR_MSGS.TABLE_PERM_NOT_BOOLEAN(permKey), validationErrors, item, t);
+						}
+					});
+
+					//validate table ATTRIBUTE_PERMISSIONS perm
+					if (table.attribute_permissions === undefined) {
+						addPermError(HDB_ERROR_MSGS.ATTR_PERMS_ARRAY_MISSING, validationErrors, item, t);
+						continue;
+					} else if (!(Array.isArray(table.attribute_permissions) || table.attribute_permissions === null)) {
+						addPermError(HDB_ERROR_MSGS.ATTR_PERMS_NOT_ARRAY, validationErrors, item, t);
+						continue;
+					}
+
+					//need this check here to ensure no unexpected errors if key is missing in table perms obj
+					if (table.attribute_permissions) {
+						let tableAttributeNames = global.hdb_schema[item][t].attributes.map(({ attribute }) => attribute);
+						const attrPermsCheck = {
+							read: false,
+							insert: false,
+							update: false,
+						};
+
+						for (let r in table.attribute_permissions) {
+							let permission = table.attribute_permissions[r];
+
+							Object.keys(permission).forEach((key) => {
+								//Leaving this second check for "DELETE" in for now since we've decided to silently
+								// allow it to remain in the attr permission object even though we do not use it
+								if (!ATTR_PERMS_KEYS.includes(key) && key !== PERMS_CRUD_ENUM.DELETE) {
+									addPermError(HDB_ERROR_MSGS.INVALID_ATTR_PERM_KEY(key), validationErrors, item, t);
+								}
+							});
+
+							//validate that attribute_name is included
+							if (!validate.isDefined(permission.attribute_name)) {
+								addPermError(HDB_ERROR_MSGS.ATTR_PERM_MISSING_NAME, validationErrors, item, t);
+								continue;
+							}
+
+							const attrName = permission.attribute_name;
+							//validate that attr exists in schema for table
+							if (!tableAttributeNames.includes(attrName)) {
+								addPermError(HDB_ERROR_MSGS.INVALID_ATTRIBUTE_IN_PERMS(attrName), validationErrors, item, t);
+								continue;
+							}
+
+							//validate table attribute CRU perms
+							ATTR_CRU_KEYS.forEach((permKey) => {
+								if (!validate.isDefined(permission[permKey])) {
+									addPermError(HDB_ERROR_MSGS.ATTR_PERM_MISSING(permKey, attrName), validationErrors, item, t);
+								} else if (!validate.isBoolean(permission[permKey])) {
+									addPermError(HDB_ERROR_MSGS.ATTR_PERM_NOT_BOOLEAN(permKey, attrName), validationErrors, item, t);
+								}
+							});
+
+							//confirm that false table perms are not set to true for an attribute
+							if (!attrPermsCheck.read && permission.read === true) {
+								attrPermsCheck.read = true;
+							}
+							if (!attrPermsCheck.insert && permission.insert === true) {
+								attrPermsCheck.insert = true;
+							}
+							if (!attrPermsCheck.update && permission.update === true) {
+								attrPermsCheck.update = true;
+							}
+						}
+						//validate that there is no mismatching perms between table and attrs
+						if (
+							(table.read === false && attrPermsCheck.read === true) ||
+							(table.insert === false && attrPermsCheck.insert === true) ||
+							(table.update === false && attrPermsCheck.update === true)
+						) {
+							const schemaName = `${item}.${t}`;
+							addPermError(HDB_ERROR_MSGS.MISMATCHED_TABLE_ATTR_PERMS(schemaName), validationErrors, item, t);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return generateRolePermResponse(validationErrors);
+}
+
+module.exports = {
+	addRoleValidation,
+	alterRoleValidation,
+	dropRoleValidation,
+};
+
+/**
+ * Validates that permissions object for CU or SU roles do not also include permissions
+ * @param obj
+ * @returns {string|null}
+ */
+function validateNoSUPerms(obj) {
+	const { operation, permission } = obj;
+	if (operation === terms.OPERATIONS_ENUM.ADD_ROLE || operation === terms.OPERATIONS_ENUM.ALTER_ROLE) {
+		//Check if role type is super user
+		const isSuRole = permission.super_user === true;
+		const hasPerms = Object.keys(permission).length > 1;
+		if (hasPerms && isSuRole) {
+			return HDB_ERROR_MSGS.SU_CU_ROLE_NO_PERMS_ALLOWED(terms.ROLE_TYPES_ENUM.SUPER_USER);
+		}
+	}
+	return null;
+}
+
+/**
+ * Builds final permissions object error response to return if validation fails
+ *
+ * @param validationErrors
+ * @returns {null|HdbError}
+ */
+function generateRolePermResponse(validationErrors) {
+	const { main_permissions, schema_permissions } = validationErrors;
+	if (main_permissions.length > 0 || Object.keys(schema_permissions).length > 0) {
+		let validationMessage = {
+			error: HDB_ERROR_MSGS.ROLE_PERMS_ERROR,
+			...validationErrors,
+		};
+
+		return handleHDBError(new Error(), validationMessage, HTTP_STATUS_CODES.BAD_REQUEST);
+	} else {
+		return null;
+	}
+}
+
+/**
+ * Adds perm validation error to the correct category for the final validation error response
+ * @param err
+ * @param invalidPermsObj
+ * @param schema
+ * @param table
+ */
+function addPermError(err, invalidPermsObj, schema, table) {
+	if (!schema) {
+		invalidPermsObj.main_permissions.push(err);
+	} else {
+		const schemaKey = table ? schema + '_' + table : schema;
+		if (!invalidPermsObj.schema_permissions[schemaKey]) {
+			invalidPermsObj.schema_permissions[schemaKey] = [err];
+		} else {
+			invalidPermsObj.schema_permissions[schemaKey].push(err);
+		}
+	}
+}

package/validation/schemaMetadataValidator.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const schemaDescribe = require('../dataLayer/schemaDescribe.js');
+const { hdbErrors } = require('../utility/errors/hdbError.js');
+const { getDatabases } = require('../resources/databases.ts');
+
+module.exports = {
+	checkSchemaExists,
+	checkSchemaTableExists,
+	schemaDescribe,
+};
+
+/**
+ * Checks the global hdbSchema for a schema and table
+ * @param schemaName
+ * @param tableName
+ * @returns string returns a thrown message if schema and or table does not exist
+ */
+async function checkSchemaExists(schemaName) {
+	let databases = getDatabases();
+	if (!databases[schemaName]) {
+		return hdbErrors.HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(schemaName);
+	}
+}
+
+/**
+ * Checks the global hdbSchema for a schema and table
+ * @param schemaName
+ * @param tableName
+ * @returns string returns a thrown message if schema and or table does not exist
+ */
+async function checkSchemaTableExists(schemaName, tableName) {
+	let invalidSchema = await checkSchemaExists(schemaName);
+	if (invalidSchema) {
+		return invalidSchema;
+	}
+	let databases = getDatabases();
+
+	if (!databases[schemaName][tableName]) {
+		return hdbErrors.HDB_ERROR_MSGS.TABLE_NOT_FOUND(schemaName, tableName);
+	}
+}