@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/validation/common_validators.js

@@ -0,0 +1,95 @@
+'use strict';
+
+const hdbUtils = require('../utility/common_utils.js');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const schemaRegex = /^[\x20-\x2E|\x30-\x5F|\x61-\x7E]*$/;
+const Joi = require('joi');
+
+const commonValidators = {
+	schema_format: {
+		pattern: schemaRegex,
+		message: 'names cannot include backticks or forward slashes',
+	},
+	schema_length: {
+		minimum: 1,
+		maximum: 250,
+		tooLong: 'cannot exceed 250 characters',
+	},
+};
+
+// A Joi schema that can be used to validate hdb schemas and tables.
+const hdbSchemaTable = Joi.alternatives(
+	Joi.string()
+		.min(1)
+		.max(commonValidators.schema_length.maximum)
+		.pattern(schemaRegex)
+		.messages({ 'string.pattern.base': '{:#label} ' + commonValidators.schema_format.message }),
+	Joi.number(),
+	Joi.array()
+).required();
+
+const hdbDatabase = Joi.alternatives(
+	Joi.string()
+		.min(1)
+		.max(commonValidators.schema_length.maximum)
+		.pattern(schemaRegex)
+		.messages({ 'string.pattern.base': '{:#label} ' + commonValidators.schema_format.message }),
+	Joi.number()
+);
+
+const hdbTable = Joi.alternatives(
+	Joi.string()
+		.min(1)
+		.max(commonValidators.schema_length.maximum)
+		.pattern(schemaRegex)
+		.messages({ 'string.pattern.base': '{:#label} ' + commonValidators.schema_format.message }),
+	Joi.number()
+).required();
+
+function checkValidTable(propertyName, value) {
+	if (!value) return `'${propertyName}' is required`;
+	if (typeof value !== 'string') return `'${propertyName}' must be a string`;
+	if (!value.length) return `'${propertyName}' must be at least one character`;
+	if (value.length > commonValidators.schema_length.maximum) return `'${propertyName}' maximum of 250 characters`;
+	if (!schemaRegex.test(value)) return `'${propertyName}' has illegal characters`;
+	return '';
+}
+
+function validateSchemaExists(value, helpers) {
+	if (!hdbUtils.doesSchemaExist(value)) {
+		return helpers.message(`Database '${value}' does not exist`);
+	}
+
+	return value;
+}
+
+function validateTableExists(value, helpers) {
+	const schema = helpers.state.ancestors[0].schema;
+	if (!hdbUtils.doesTableExist(schema, value)) {
+		return helpers.message(`Table '${value}' does not exist`);
+	}
+
+	return value;
+}
+
+function validateSchemaName(value, helpers) {
+	if (value.toLowerCase() === hdbTerms.SYSTEM_SCHEMA_NAME) {
+		return helpers.message(
+			`'subscriptions[${helpers.state.path[1]}]' invalid database name, '${hdbTerms.SYSTEM_SCHEMA_NAME}' name is reserved`
+		);
+	}
+
+	return value;
+}
+
+module.exports = {
+	commonValidators,
+	schemaRegex,
+	hdbSchemaTable,
+	validateSchemaExists,
+	validateTableExists,
+	validateSchemaName,
+	checkValidTable,
+	hdbDatabase,
+	hdbTable,
+};

package/validation/configValidator.js

@@ -0,0 +1,331 @@
+'use strict';
+
+const fs = require('fs-extra');
+const Joi = require('joi');
+const os = require('os');
+const { boolean, string, number, array } = Joi.types();
+const { totalmem } = require('os');
+const path = require('path');
+const hdbLogger = require('../utility/logging/harper_logger.js');
+const hdbUtils = require('../utility/common_utils.js');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const validator = require('./validationWrapper.js');
+
+const DEFAULT_LOG_FOLDER = 'log';
+const DEFAULT_COMPONENTS_FOLDER = 'components';
+const INVALID_SIZE_UNIT_MSG = 'Invalid logging.rotation.maxSize unit. Available units are G, M or K';
+const INVALID_INTERVAL_UNIT_MSG = 'Invalid logging.rotation.interval unit. Available units are D, H or M (minutes)';
+const INVALID_MAX_SIZE_VALUE_MSG =
+	"Invalid logging.rotation.maxSize value. Value should be a number followed by unit e.g. '10M'";
+const INVALID_INTERVAL_VALUE_MSG =
+	"Invalid logging.rotation.interval value. Value should be a number followed by unit e.g. '10D'";
+const UNDEFINED_OPS_API = 'rootPath config parameter is undefined';
+
+const portConstraints = Joi.alternatives([number.min(0), string])
+	.optional()
+	.empty(null);
+const routeConstraints = Joi.alternatives([
+	array
+		.items(
+			string,
+			{
+				host: string.required(),
+				port: portConstraints,
+			},
+			{
+				hostname: string.required(),
+				port: portConstraints,
+			}
+		)
+		.empty(null),
+	array.items(string),
+]);
+
+let hdbRoot;
+let skipFsVal = false;
+
+module.exports = {
+	configValidator,
+	routesValidator,
+	routeConstraints,
+};
+
+function configValidator(configJson, skipFsValidation = false) {
+	skipFsVal = skipFsValidation;
+	hdbRoot = configJson.rootPath;
+	if (hdbUtils.isEmpty(hdbRoot)) {
+		throw UNDEFINED_OPS_API;
+	}
+
+	const enabledConstraints = boolean.optional();
+	const threadsConstraints = number.min(0).max(1000).empty(null).default(setDefaultThreads);
+	const rootConstraints = string
+		.pattern(/^[\\/]$|([\\/a-zA-Z_0-9:-]+)+$/, 'directory path')
+		.empty(null)
+		.default(setDefaultRoot);
+	const pemFileConstraints = string.optional().empty(null);
+
+	const storagePathConstraints = Joi.custom(validatePath).empty(null).default(setDefaultRoot);
+	const tlsConstraints = Joi.object({
+		certificate: pemFileConstraints,
+		certificateAuthority: pemFileConstraints,
+		privateKey: pemFileConstraints,
+	});
+
+	const configSchema = Joi.object({
+		authentication: Joi.alternatives(
+			Joi.object({
+				authorizeLocal: boolean,
+				cacheTTL: number.required(),
+				cookie: Joi.object({
+					domains: array.items(string).optional(),
+					expires: string.optional(),
+				}),
+				enableSessions: boolean,
+				hashFunction: string.valid('md5', 'sha256', 'argon2id').optional().empty(null),
+			}),
+			boolean
+		).optional(),
+		analytics: Joi.object({
+			aggregatePeriod: number,
+			replicate: boolean.optional(),
+		}),
+		replication: Joi.object({
+			hostname: Joi.alternatives(string, number).optional().empty(null),
+			url: string.optional().empty(null),
+			port: portConstraints,
+			securePort: portConstraints,
+			routes: array.optional().empty(null),
+			databases: Joi.alternatives(string, array),
+			enableRootCAs: boolean.optional(),
+			copyTablesToCatchUp: boolean.optional(),
+		}).optional(),
+		componentsRoot: rootConstraints.optional(),
+		localStudio: Joi.object({
+			enabled: enabledConstraints,
+		}).required(),
+		logging: Joi.object({
+			auditAuthEvents: Joi.object({
+				logFailed: boolean,
+				logSuccessful: boolean,
+			}),
+			file: boolean.required(),
+			level: Joi.valid('notify', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'),
+			rotation: Joi.object({
+				enabled: boolean.optional(),
+				compress: boolean.optional(),
+				interval: string.custom(validateRotationInterval).optional().empty(null),
+				maxSize: string.custom(validateRotationMaxSize).optional().empty(null),
+				path: string.optional().empty(null).default(setDefaultRoot),
+			}).required(),
+			root: rootConstraints,
+			stdStreams: boolean.required(),
+			auditLog: boolean.required(),
+		}).required(),
+		operationsApi: Joi.object({
+			network: Joi.object({
+				cors: boolean.optional(),
+				corsAccessList: array.optional(),
+				headersTimeout: number.min(1).optional(),
+				keepAliveTimeout: number.min(1).optional(),
+				port: portConstraints,
+				domainSocket: Joi.optional().empty('hdb/operations-server').default(setDefaultRoot),
+				securePort: portConstraints,
+				timeout: number.min(1).optional(),
+			}).optional(),
+			tls: Joi.alternatives([Joi.array().items(tlsConstraints), tlsConstraints]),
+		}).required(),
+		rootPath: string.pattern(/^[\\/]$|([\\/a-zA-Z_0-9:-]+)+$/, 'directory path').required(),
+		mqtt: Joi.object({
+			network: Joi.object({
+				port: portConstraints,
+				securePort: portConstraints,
+				mtls: Joi.alternatives([
+					boolean.optional(),
+					Joi.object({
+						user: string.optional(),
+						certificateAuthority: pemFileConstraints,
+						required: boolean.optional(),
+					}),
+				]),
+			}).required(),
+			webSocket: boolean.optional(),
+			requireAuthentication: boolean.optional(),
+		}),
+		http: Joi.object({
+			compressionThreshold: number.optional(),
+			cors: boolean.optional(),
+			corsAccessList: array.optional(),
+			headersTimeout: number.min(1).optional(),
+			port: portConstraints,
+			securePort: portConstraints,
+			maxHeaderSize: number.optional(),
+			mtls: Joi.alternatives([
+				boolean.optional(),
+				Joi.object({
+					user: string.optional(),
+					certificateAuthority: pemFileConstraints,
+					required: boolean.optional(),
+				}),
+			]),
+			threadRange: Joi.alternatives([array.optional(), string.optional()]),
+		}).required(),
+		threads: Joi.alternatives(
+			threadsConstraints.optional(),
+			Joi.object({
+				count: threadsConstraints.optional(),
+				debug: Joi.alternatives(
+					boolean.optional(),
+					Joi.object({
+						startingPort: number.min(1).optional(),
+						host: string.optional(),
+						waitForDebugger: boolean.optional(),
+					})
+				),
+				maxHeapMemory: number.min(0).optional(),
+			})
+		),
+		storage: Joi.object({
+			writeAsync: boolean.required(),
+			overlappingSync: boolean.optional(),
+			caching: boolean.optional(),
+			compression: Joi.alternatives([
+				boolean.optional(),
+				Joi.object({ dictionary: string.optional(), threshold: number.optional() }),
+			]),
+			compactOnStart: boolean.optional(),
+			compactOnStartKeepBackup: boolean.optional(),
+			noReadAhead: boolean.optional(),
+			path: storagePathConstraints,
+			prefetchWrites: boolean.optional(),
+			maxFreeSpaceToLoad: number.optional(),
+			maxFreeSpaceToRetain: number.optional(),
+		}).required(),
+		ignoreScripts: boolean.optional(),
+		tls: Joi.alternatives([Joi.array().items(tlsConstraints), tlsConstraints]),
+	});
+
+	// Not using the validation wrapper here because we need the result if validation is successful because
+	// there is default values set as part of validation.
+	return configSchema.validate(configJson, {
+		allowUnknown: true,
+		abortEarly: false,
+		errors: { wrap: { label: "'" } },
+	});
+}
+
+// This function is used to validate existence of paths passed as an argument
+function doesPathExist(pathToCheck) {
+	if (skipFsVal) return null;
+	let exists = fs.existsSync(pathToCheck);
+	if (exists) {
+		return null;
+	}
+
+	return `Specified path ${pathToCheck} does not exist.`;
+}
+
+function validatePath(value, helpers) {
+	Joi.assert(value, string.pattern(/^[\\/]$|([\\/a-zA-Z_0-9:-]+)+$/, 'directory path'));
+
+	const doesExistMsg = doesPathExist(value);
+	if (doesExistMsg) {
+		return helpers.message(doesExistMsg);
+	}
+}
+
+function validateRotationMaxSize(value, helpers) {
+	const unit = value.slice(-1);
+	if (unit !== 'G' && unit !== 'M' && unit !== 'K') {
+		return helpers.message(INVALID_SIZE_UNIT_MSG);
+	}
+
+	const size = value.slice(0, -1);
+	if (isNaN(parseInt(size))) {
+		return helpers.message(INVALID_MAX_SIZE_VALUE_MSG);
+	}
+
+	return value;
+}
+
+function validateRotationInterval(value, helpers) {
+	const unit = value.slice(-1);
+	if (unit !== 'D' && unit !== 'H' && unit !== 'M') {
+		return helpers.message(INVALID_INTERVAL_UNIT_MSG);
+	}
+
+	const size = value.slice(0, -1);
+	if (isNaN(parseInt(size))) {
+		return helpers.message(INVALID_INTERVAL_VALUE_MSG);
+	}
+
+	return value;
+}
+
+function setDefaultThreads(parent, helpers) {
+	const configParam = helpers.state.path.join('.');
+	let processors = os.cpus().length;
+
+	// default to one less than the number of logical CPU/processors so we can have good concurrency with the
+	// ingest process and any extra processes (jobs, reply, etc.).
+	let numProcesses = processors - 1;
+	// But if only two or less processors, keep two processes so we have some level of concurrency fairness
+	if (numProcesses <= 2) numProcesses = 2;
+	let availableMemory = process.constrainedMemory?.() || totalmem(); // used constrained memory if it is available
+	// and lower than total memory
+	availableMemory = Math.round(Math.min(availableMemory, totalmem()) / 1000000);
+	// (available memory -750MB) / 300MB
+	numProcesses = Math.max(Math.min(numProcesses, Math.round((availableMemory - 750) / 300)), 1);
+	hdbLogger.info(
+		`Detected ${processors} cores and ${availableMemory}MB on this machine, defaulting ${configParam} to ${numProcesses}`
+	);
+	return numProcesses;
+}
+
+/**
+ * Sets a default root for a config param.
+ * @param parent
+ * @param helpers
+ * @returns {string}
+ */
+function setDefaultRoot(parent, helpers) {
+	// For some reason Joi is still calling set default when value is not null.
+	// For that reason we do this check.
+	const configParam = helpers.state.path.join('.');
+	if (!hdbUtils.isEmpty(helpers.original) && configParam !== 'operationsApi.network.domainSocket') {
+		return helpers.original;
+	}
+
+	if (hdbUtils.isEmpty(hdbRoot)) {
+		throw new Error(`Error setting default root for: ${configParam}. HDB root is not defined`);
+	}
+
+	switch (configParam) {
+		case 'componentsRoot':
+			return path.join(hdbRoot, DEFAULT_COMPONENTS_FOLDER);
+		case 'logging.root':
+			return path.join(hdbRoot, DEFAULT_LOG_FOLDER);
+		case 'storage.path':
+			const legacyStoragePath = path.join(hdbRoot, hdbTerms.LEGACY_DATABASES_DIR_NAME);
+			if (fs.existsSync(legacyStoragePath)) return legacyStoragePath;
+			return path.join(hdbRoot, hdbTerms.DATABASES_DIR_NAME);
+		case 'logging.rotation.path':
+			return path.join(hdbRoot, DEFAULT_LOG_FOLDER);
+		case 'operationsApi.network.domainSocket':
+			return configParam == null ? null : path.join(hdbRoot, 'operations-server');
+		default:
+			throw new Error(`Error setting default root for config parameter: ${configParam}. Unrecognized config parameter`);
+	}
+}
+
+/**
+ * Validates just the routes array.
+ * @param routesArray
+ * @returns {*}
+ */
+function routesValidator(routesArray) {
+	const schema = Joi.object({
+		routes: routeConstraints,
+	});
+	return validator.validateBySchema({ routes: routesArray }, schema);
+}

package/validation/deleteValidator.js

@@ -0,0 +1,15 @@
+const validator = require('./validationWrapper.js');
+const Joi = require('joi');
+const { hdbTable, hdbDatabase } = require('./common_validators.js');
+
+const deleteSchema = Joi.object({
+	schema: hdbDatabase,
+	database: hdbDatabase,
+	table: hdbTable,
+	hash_values: Joi.array().required(),
+	ids: Joi.array(),
+});
+
+module.exports = function (deleteObject) {
+	return validator.validateBySchema(deleteObject, deleteSchema);
+};

package/validation/fileLoadValidator.js

@@ -0,0 +1,153 @@
+const clone = require('clone');
+const validator = require('./validationWrapper.js');
+const commonUtils = require('../utility/common_utils.js');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const fs = require('fs');
+const joi = require('joi');
+const { string } = joi.types();
+const { hdbErrors, handleHDBError } = require('../utility/errors/hdbError.js');
+const { HTTP_STATUS_CODES } = hdbErrors;
+
+const { commonValidators } = require('./common_validators.js');
+
+const isRequiredString = ' is required';
+
+const actions = ['insert', 'update', 'upsert'];
+const constraints = {
+	database: {
+		presence: false,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	schema: {
+		presence: false,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	table: {
+		presence: true,
+		format: commonValidators.schema_format,
+		length: commonValidators.schema_length,
+	},
+	action: {
+		inclusion: {
+			within: actions,
+			message: 'is required and must be either insert, update, or upsert',
+		},
+	},
+	file_path: {},
+	csv_url: {
+		url: {
+			allowLocal: true,
+		},
+	},
+	data: {},
+	passthrough_headers: {},
+};
+
+const baseJoiSchema = {
+	schema: string.required(),
+	table: string.required(),
+	action: string.valid('insert', 'update', 'upsert'),
+};
+
+const { AWS_ACCESS_KEY, AWS_SECRET, AWS_BUCKET, AWS_FILE_KEY, REGION } = hdbTerms.S3_BUCKET_AUTH_KEYS;
+
+const s3Constraints = {
+	s3: {
+		presence: true,
+	},
+	[`s3.${AWS_ACCESS_KEY}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_SECRET}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_BUCKET}`]: {
+		presence: true,
+		type: 'String',
+	},
+	[`s3.${AWS_FILE_KEY}`]: {
+		presence: true,
+		type: 'String',
+		hasValidFileExt: ['.csv', '.json'],
+	},
+	[`s3.${REGION}`]: {
+		presence: true,
+		type: 'String',
+	},
+};
+
+const dataConstraints = clone(constraints);
+dataConstraints.data.presence = {
+	message: isRequiredString,
+};
+
+const fileConstraints = clone(constraints);
+fileConstraints.file_path.presence = {
+	message: isRequiredString,
+};
+
+const s3FileConstraints = Object.assign(clone(constraints), s3Constraints);
+
+const urlSchema = clone(baseJoiSchema);
+urlSchema.csv_url = string.uri().messages({ 'string.uri': "'csv_url' must be a valid url" }).required();
+urlSchema.passthrough_headers = joi.object();
+
+function dataObject(object) {
+	let validateRes = validator.validateObject(object, dataConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+function urlObject(object) {
+	let validateRes = validator.validateBySchema(object, joi.object(urlSchema));
+	return postValidateChecks(object, validateRes);
+}
+
+function fileObject(object) {
+	let validateRes = validator.validateObject(object, fileConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+function s3FileObject(object) {
+	let validateRes = validator.validateObject(object, s3FileConstraints);
+	return postValidateChecks(object, validateRes);
+}
+
+/**
+ * Post validate module checks, confirms schema and table exist.
+ * If file upload - checks that it exists, permissions and size.
+ */
+function postValidateChecks(object, validateRes) {
+	if (!validateRes) {
+		let msg = commonUtils.checkGlobalSchemaTable(object.schema, object.table);
+		if (msg) {
+			return handleHDBError(new Error(), msg, HTTP_STATUS_CODES.BAD_REQUEST);
+		}
+
+		if (object.operation === hdbTerms.OPERATIONS_ENUM.CSV_FILE_LOAD) {
+			try {
+				fs.accessSync(object.file_path, fs.constants.R_OK | fs.constants.F_OK);
+			} catch (err) {
+				if (err.code === hdbTerms.NODE_ERROR_CODES.ENOENT) {
+					return handleHDBError(err, `No such file or directory ${err.path}`, HTTP_STATUS_CODES.BAD_REQUEST);
+				}
+
+				if (err.code === hdbTerms.NODE_ERROR_CODES.EACCES) {
+					return handleHDBError(err, `Permission denied ${err.path}`, HTTP_STATUS_CODES.BAD_REQUEST);
+				}
+				return handleHDBError(err);
+			}
+		}
+	}
+	return validateRes;
+}
+
+module.exports = {
+	dataObject,
+	urlObject,
+	fileObject,
+	s3FileObject,
+};

package/validation/insertValidator.js

@@ -0,0 +1,40 @@
+const { hdbTable, hdbDatabase } = require('./common_validators.js');
+const validator = require('./validationWrapper.js');
+const Joi = require('joi');
+const INVALID_ATTRIBUTE_NAMES = {
+	undefined: 'undefined',
+	null: 'null',
+};
+
+const customRecordsVal = (value, helpers) => {
+	const attributes = Object.keys(value);
+	const attributesLength = attributes.length;
+	let errorMsg = undefined;
+	for (let i = 0; i < attributesLength; i++) {
+		const attribute = attributes[i];
+		if (!attribute || attribute.length === 0 || INVALID_ATTRIBUTE_NAMES[attribute] !== undefined) {
+			if (errorMsg === undefined) {
+				errorMsg = `Invalid attribute name: '${attribute}'`;
+			} else {
+				errorMsg += `. Invalid attribute name: '${attribute}'`;
+			}
+		}
+	}
+
+	if (errorMsg) {
+		return helpers.message(errorMsg);
+	}
+
+	return value;
+};
+
+const insertSchema = Joi.object({
+	database: hdbDatabase,
+	schema: hdbDatabase,
+	table: hdbTable,
+	records: Joi.array().items(Joi.object().custom(customRecordsVal)).required(),
+});
+
+module.exports = function (insertObject) {
+	return validator.validateBySchema(insertObject, insertSchema);
+};

package/validation/installValidator.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const Joi = require('joi');
+const { string, number } = Joi.types();
+const fs = require('fs-extra');
+const hdbTerms = require('../utility/hdbTerms.ts');
+const path = require('path');
+const validator = require('../validation/validationWrapper.js');
+
+module.exports = installValidator;
+
+/**
+ * Used to validate any command or environment variables used passed to install.
+ * @param param
+ * @returns {*}
+ */
+function installValidator(param) {
+	const installSchema = Joi.object({
+		[hdbTerms.INSTALL_PROMPTS.ROOTPATH]: Joi.custom(validateRootAvailable),
+		[hdbTerms.INSTALL_PROMPTS.OPERATIONSAPI_NETWORK_PORT]: Joi.alternatives([number.min(0), string]).allow(
+			'null',
+			null
+		),
+		[hdbTerms.INSTALL_PROMPTS.TC_AGREEMENT]: string.valid('yes', 'YES', 'Yes'),
+	});
+
+	return validator.validateBySchema(param, installSchema);
+}
+
+function validateRootAvailable(value, helpers) {
+	if (
+		fs.existsSync(path.join(value, 'system/hdb_user/data.mdb')) ||
+		fs.existsSync(path.join(value, 'system/hdb_user.mdb'))
+	) {
+		return helpers.message(`'${value}' is already in use. Please enter a different path.`);
+	}
+}