@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/server/throttle.ts

@@ -0,0 +1,73 @@
+import { logger } from '../utility/logging/logger.ts';
+const MAX_EVENT_DELAY_TIME = 3000;
+const DEFAULT_MAX_QUEUE_TIME = 20_000; // 20 seconds
+let lastWarning = 0;
+const WARNING_INTERVAL = 30000;
+const EVENT_QUEUE_MONITORING_INTERVAL = 3000;
+let lastEventQueueCheck = performance.now() + EVENT_QUEUE_MONITORING_INTERVAL;
+let averageEventCycleTime = 0;
+/**
+ * Throttle function to limit the number of calls to a function so that the event queue doesn't get overwhelmed.
+ * @param fn
+ * @param onLimitExceeded
+ * @param maxQueueTimeLimit
+ */
+export function throttle(
+	fn: (...args: any) => any,
+	onLimitExceeded?: (...args: any) => any,
+	maxQueueTimeLimit = DEFAULT_MAX_QUEUE_TIME
+) {
+	let queuedCalls: any[];
+	return function (...args: any[]) {
+		if (queuedCalls) {
+			// this is an estimate of the time an event will take to process, based on the average event cycle time and the queue depth
+			if (queuedCalls.length * averageEventCycleTime > maxQueueTimeLimit) {
+				return onLimitExceeded(...args);
+			}
+			return new Promise((resolve, reject) => {
+				queuedCalls.push({
+					args,
+					fn() {
+						try {
+							const result = fn(...args);
+							resolve(result);
+						} catch (e) {
+							reject(e);
+						}
+					},
+				});
+			});
+		}
+		queuedCalls = [];
+		waitForNextCycle(performance.now());
+		return fn(...args);
+	};
+	function waitForNextCycle(startTime: number) {
+		setImmediate(() => {
+			const now = performance.now();
+			// get the decaying/running average of the event cycle time
+			averageEventCycleTime = (averageEventCycleTime * 4 + now - startTime) / 5;
+			const nextCall = queuedCalls.shift();
+			if (nextCall) {
+				const { fn: nextFunction } = nextCall;
+				nextFunction();
+				waitForNextCycle(now);
+			} else {
+				queuedCalls = null;
+			}
+		});
+	}
+}
+setInterval(() => {
+	const now = performance.now();
+	if (
+		now - lastEventQueueCheck - EVENT_QUEUE_MONITORING_INTERVAL > MAX_EVENT_DELAY_TIME &&
+		lastWarning + WARNING_INTERVAL < now
+	) {
+		logger.warn?.(
+			`JavaScript execution has taken too long and is not allowing proper event queue cycling, consider using 'await new Promise(setImmediate)' in code that will execute for a long duration`
+		);
+		lastWarning = now;
+	}
+	lastEventQueueCheck = now;
+}, EVENT_QUEUE_MONITORING_INTERVAL).unref();

package/sqlTranslator/SelectValidator.js

@@ -0,0 +1,330 @@
+'use strict';
+
+const RecursiveIterator = require('recursive-iterator');
+const alasql = require('alasql');
+const clone = require('clone');
+const commonUtils = require('../utility/common_utils.js');
+const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
+const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
+const { getDatabases } = require('../resources/databases.ts');
+
+//exclusion list for validation on group bys
+const customAggregators = ['DISTINCT_ARRAY'];
+
+const validateTables = Symbol('validateTables'),
+	validateTable = Symbol('validateTable'),
+	validateAllColumns = Symbol('validateAllColumns'),
+	findColumn = Symbol('findColumn'),
+	validateOrderBy = Symbol('validateOrderBy'),
+	validateSegment = Symbol('validateSegment'),
+	validateColumn = Symbol('validateColumn'),
+	setColumnsForTable = Symbol('setColumnsForTable'),
+	checkColumnsForAsterisk = Symbol('checkColumnsForAsterisk'),
+	validateGroupBy = Symbol('validateGroupBy'),
+	hasColumns = Symbol('hasColumns');
+
+/**
+ * Validates the tables and attributes against the actual schema
+ * Validates general SQL rules
+ */
+class SelectValidator {
+	constructor(statement) {
+		this.statement = statement;
+		this.attributes = [];
+	}
+
+	/**
+	 * entry point for validation
+	 * @returns {*}
+	 */
+	validate() {
+		if (!this.statement) {
+			throw new Error('invalid sql statement');
+		}
+
+		this[validateTables]();
+		this[checkColumnsForAsterisk]();
+		this[validateAllColumns]();
+	}
+
+	/**
+	 * if the statement has columns in it:
+	 * loops thru the from and join arrays of the AST and passes individual entries into validateTable
+	 */
+	[validateTables]() {
+		if (this[hasColumns]()) {
+			if (!this.statement.from || this.statement.from.length === 0) {
+				throw `no from clause`;
+			}
+
+			this.statement.from.forEach((table) => {
+				this[validateTable](table);
+			});
+
+			if (this.statement.joins) {
+				this.statement.joins.forEach((join) => {
+					join.table.as = join.as;
+					this[validateTable](join.table);
+				});
+			}
+		}
+	}
+
+	/**
+	 * check to see if there are columns in any part of the select
+	 * @returns {boolean}
+	 */
+	[hasColumns]() {
+		let hasColumns = false;
+		let iterator = new RecursiveIterator(this.statement);
+		for (let { node } of iterator) {
+			if (node && node.columnid) {
+				hasColumns = true;
+				break;
+			}
+		}
+		return hasColumns;
+	}
+
+	/**
+	 * Checks that the table exists in the schema, adds all of it's attributes to the class level collection this.attributes
+	 * @param table
+	 */
+	[validateTable](table) {
+		if (!table.databaseid) {
+			throw `schema not defined for table ${table.tableid}`;
+		}
+		let databases = getDatabases();
+		if (!databases[table.databaseid]) {
+			throw handleHDBError(new Error(), HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(table.databaseid), HTTP_STATUS_CODES.NOT_FOUND);
+		}
+
+		if (!databases[table.databaseid][table.tableid]) {
+			throw handleHDBError(
+				new Error(),
+				HDB_ERROR_MSGS.TABLE_NOT_FOUND(table.databaseid, table.tableid),
+				HTTP_STATUS_CODES.NOT_FOUND
+			);
+		}
+
+		//let theTable = clone(table);
+		let schemaTable = databases[table.databaseid][table.tableid];
+		/*TODO rather than putting every attribute in an array we will create a Map there will be a map element for every table and every table alias
+ (this will create duplicate map elements) this will have downstream effects in comparison functions like findColumn*/
+		schemaTable.attributes.forEach((attribute) => {
+			let attributeClone = clone(attribute);
+			attributeClone.table = clone(table);
+			this.attributes.push(attributeClone);
+		});
+	}
+
+	/**
+	 * validates the column against the schema
+	 * @param column
+	 * @returns {*[]}
+	 */
+	[findColumn](column) {
+		//look to see if this attribute exists on one of the tables we are selecting from
+		return this.attributes.filter((attribute) => {
+			if (column.tableid) {
+				return (
+					(attribute.table.as === column.tableid || attribute.table.tableid === column.tableid) &&
+					attribute.attribute === column.columnid
+				);
+			} else {
+				return attribute.attribute === column.columnid;
+			}
+		});
+	}
+
+	/**
+	 * detects * in the select, if found adds all columns to the select
+	 */
+	[checkColumnsForAsterisk]() {
+		let iterator = new RecursiveIterator(this.statement.columns);
+
+		for (let { node, path } of iterator) {
+			//we check the path to make sure the '*' is not wrapped in some form of expression like count(*)
+			if (node && node.columnid === '*' && path.indexOf('expression') < 0) {
+				this[setColumnsForTable](node.tableid);
+			}
+		}
+	}
+
+	/**
+	 * takes a table and adds all of it's columns to the select. if no table it adds every column from every table in the select
+	 * @param tableName
+	 */
+	[setColumnsForTable](tableName) {
+		this.attributes.forEach((attribute) => {
+			if (
+				(!tableName || (tableName && (attribute.table.tableid === tableName || attribute.table.as === tableName))) &&
+				!attribute.relation
+			) {
+				this.statement.columns.push(
+					new alasql.yy.Column({
+						columnid: attribute.attribute,
+						tableid: attribute.table.as ? attribute.table.as : attribute.table.tableid,
+					})
+				);
+			}
+		});
+	}
+
+	/**
+	 * passes segments to ValidateSegment for validation
+	 */
+	[validateAllColumns]() {
+		this[validateSegment](this.statement.columns, false);
+		this[validateSegment](this.statement.joins, false);
+		this[validateSegment](this.statement.where, false);
+		this[validateGroupBy](this.statement.group, false);
+		this[validateSegment](this.statement.order, true);
+	}
+
+	/**
+	 * iterates the attributes in a segment and validates them against the schema
+	 * @param segment
+	 * @param isOrderBy
+	 * @returns {*}
+	 */
+	[validateSegment](segment, isOrderBy) {
+		if (!segment) {
+			return;
+		}
+
+		let iterator = new RecursiveIterator(segment);
+		let attributes = [];
+		for (let { node } of iterator) {
+			if (!commonUtils.isEmpty(node) && !commonUtils.isEmpty(node.columnid) && node.columnid !== '*') {
+				if (isOrderBy) {
+					this[validateOrderBy](node);
+				} else {
+					attributes.push(this[validateColumn](node));
+				}
+			}
+		}
+
+		return attributes;
+	}
+
+	/**
+	 * validation specific for GROUP BY
+	 * makes sure that the non-aggregate functions and columns from the select are represented in the group by and the columns match the schema
+	 * @param segment
+	 */
+	[validateGroupBy](segment) {
+		if (!segment) {
+			return;
+		}
+		//check select for aggregates and non-aggregates, if it has both non-aggregates need to be in group by
+		let selectColumns = [];
+		//here we are pulling out all non-aggregate functions into an array for comparison to the group by
+		this.statement.columns.forEach((column) => {
+			//this keeps white listed custom functions from being validated
+			if (column.funcid && customAggregators.indexOf(column.funcid.toUpperCase()) >= 0) {
+				return;
+			}
+
+			if (!column.aggregatorid && !column.columnid) {
+				//this is to make sure functions or any type of evaluation statement is being compared to the select.
+				//i.e. "GROUP BY UPPER(name)" needs to have UPPER(name) in the select
+				let columnClone = clone(column);
+				delete columnClone.as;
+				selectColumns.push(columnClone);
+			} else if (column.columnid) {
+				let found = this[findColumn](column)[0];
+				if (found) {
+					selectColumns.push(found);
+				}
+			}
+		});
+
+		//here we iterate the group by and compare to what is in the select and make sure they match appropriately
+		this.statement.group.forEach((groupColumn) => {
+			let foundColumn = null;
+
+			if (!groupColumn.columnid) {
+				//TODO can use for of to break out of the loop rather than this janky way
+				selectColumns.forEach((column, x) => {
+					if (column.toString() === groupColumn.toString()) {
+						foundColumn = column;
+						selectColumns.splice(x, 1);
+						return;
+					}
+				});
+			} else {
+				let foundGroupColumn = this[findColumn](groupColumn);
+
+				if (!foundGroupColumn || foundGroupColumn.length === 0) {
+					throw `unknown column '${groupColumn.toString()}' in group by`;
+				}
+
+				if (foundGroupColumn.length > 1) {
+					throw `ambiguously defined column '${groupColumn.toString()}' in group by`;
+				}
+
+				//TODO can use for of to break out of the loop rather than this janky way
+				selectColumns.forEach((column, x) => {
+					if (
+						column.attribute === foundGroupColumn[0].attribute &&
+						column.table.tableid === foundGroupColumn[0].table.tableid
+					) {
+						foundColumn = column;
+						selectColumns.splice(x, 1);
+						return;
+					}
+				});
+			}
+
+			if (!foundColumn) {
+				throw `group by column '${groupColumn.toString()}' must be in select`;
+			}
+		});
+
+		if (selectColumns.length > 0) {
+			throw `select column '${
+				selectColumns[0].attribute ? selectColumns[0].attribute : selectColumns[0].toString()
+			}' must be in group by`;
+		}
+	}
+
+	/**
+	 * Order BY specific validation
+	 *
+	 * @param column
+	 */
+	[validateOrderBy](column) {
+		let foundColumns = this.statement.columns.filter((col) => col.as === column.columnid);
+
+		if (foundColumns.length > 1) {
+			let columnName = (column.tableid ? column.tableid + '.' : '') + column.columnid;
+			throw `ambiguous column reference ${columnName} in order by`;
+		} else if (foundColumns.length === 0) {
+			this[validateColumn](column);
+		}
+	}
+
+	/**
+	 * validates a column to the schema
+	 * @param column
+	 * @returns {*}
+	 */
+	[validateColumn](column) {
+		let foundColumns = this[findColumn](column);
+
+		let columnName = (column.tableid ? column.tableid + '.' : '') + column.columnid;
+
+		if (foundColumns.length === 0) {
+			throw `unknown column ${columnName}`;
+		}
+
+		if (foundColumns.length > 1) {
+			throw `ambiguous column reference ${columnName}`;
+		}
+
+		return foundColumns[0];
+	}
+}
+
+module.exports = SelectValidator;

package/sqlTranslator/alasqlFunctionImporter.js

@@ -0,0 +1,62 @@
+/***
+ * alasqlFunctionImporter.js
+ *
+ * PUrpose of this is to set up a central module to define and import custom functions into alasql
+ */
+
+const alasqlExtension = require('../utility/functions/sql/alaSQLExtension.js'),
+	dateFunctions = require('../utility/functions/date/dateFunctions.js'),
+	geo = require('../utility/functions/geo.js');
+
+//import the custom function, need to define an upper and lower case version of the function so it is parsed properly in alasql
+module.exports = (alasql) => {
+	/*
+    AGGREGATE FUNCTIONS
+     */
+
+	alasql.aggr.mad = alasql.aggr.MAD = alasqlExtension.mad;
+	alasql.aggr.mean = alasql.aggr.MEAN = alasqlExtension.mean;
+	alasql.aggr.mode = alasql.aggr.MODE = alasqlExtension.mode;
+	alasql.aggr.prod = alasql.aggr.PROD = alasqlExtension.prod;
+	//we are overriding alasql's median function as their algorithm is incorrect
+	alasql.aggr.median = alasql.aggr.MEDIAN = alasqlExtension.median;
+
+	/*
+    CUSTOM FUNCTIONS
+     */
+	alasql.fn.distinct_array = alasql.fn.DISTINCT_ARRAY = alasqlExtension.distinct_array;
+	alasql.fn.search_json = alasql.fn.SEARCH_JSON = alasqlExtension.searchJSON;
+	alasql.fn.__ala__ = alasql;
+
+	//Date Functions...
+
+	alasql.fn.current_date = alasql.fn.CURRENT_DATE = dateFunctions.current_date;
+	alasql.fn.current_time = alasql.fn.CURRENT_TIME = dateFunctions.current_time;
+	alasql.fn.extract = alasql.fn.EXTRACT = dateFunctions.extract;
+	alasql.fn.date = alasql.fn.DATE = dateFunctions.date;
+	alasql.fn.date_format = alasql.fn.DATE_FORMAT = dateFunctions.date_format;
+	alasql.fn.date_add = alasql.fn.DATE_ADD = dateFunctions.date_add;
+	alasql.fn.date_sub = alasql.fn.DATE_SUB = dateFunctions.date_sub;
+	alasql.fn.date_diff = alasql.fn.DATE_DIFF = alasql.fn.datediff = alasql.fn.DATEDIFF = dateFunctions.date_diff;
+	alasql.fn.now = alasql.fn.NOW = dateFunctions.now;
+	alasql.fn.offset_utc = alasql.fn.OFFSET_UTC = dateFunctions.offset_utc;
+	alasql.fn.get_server_time = alasql.fn.GET_SERVER_TIME = dateFunctions.get_server_time;
+	//GETDATE() and CURRENT_TIMESTAMP reference the date/time value from NOW() in alasql but we need to monkey patch
+	// them here as well with the new now logic
+	alasql.fn.getdate = alasql.fn.GETDATE = dateFunctions.now;
+	alasql.fn.current_timestamp = alasql.fn.CURRENT_TIMESTAMP = dateFunctions.now;
+
+	/*
+    CUSTOM GEO FUNCTIONS
+     */
+	alasql.fn.geoarea = alasql.fn.GEOAREA = alasql.fn.geoArea = geo.geoArea;
+	alasql.fn.geocircle = alasql.fn.GEOCIRCLE = alasql.fn.geoCircle = geo.geoCircle;
+	alasql.fn.geocontains = alasql.fn.GEOCONTAINS = alasql.fn.geoContains = geo.geoContains;
+	alasql.fn.geoconvert = alasql.fn.GEOCONVERT = alasql.fn.geoConvert = geo.geoConvert;
+	alasql.fn.geocrosses = alasql.fn.GEOCROSSES = alasql.fn.geoCrosses = geo.geoCrosses;
+	alasql.fn.geodifference = alasql.fn.GEODIFFERENCE = alasql.fn.geoDifference = geo.geoDifference;
+	alasql.fn.geodistance = alasql.fn.GEODISTANCE = alasql.fn.geoDistance = geo.geoDistance;
+	alasql.fn.geoequal = alasql.fn.GEOEQUAL = alasql.fn.geoEqual = geo.geoEqual;
+	alasql.fn.geolength = alasql.fn.GEOLENGTH = alasql.fn.geoLength = geo.geoLength;
+	alasql.fn.geonear = alasql.fn.GEONEAR = alasql.fn.geoNear = geo.geoNear;
+};

package/sqlTranslator/deleteTranslator.js

@@ -0,0 +1,67 @@
+const alasql = require('alasql');
+const search = require('../dataLayer/search.js');
+const log = require('../utility/logging/harper_logger.js');
+const harperBridge = require('../dataLayer/harperBridge/harperBridge.js');
+const util = require('util');
+const hdbUtils = require('../utility/common_utils.js');
+const terms = require('../utility/hdbTerms.ts');
+const globalSchema = require('../utility/globalSchema.js');
+
+const RECORD = 'record';
+const SUCCESS = 'successfully deleted';
+
+const cbConvertDelete = util.callbackify(convertDelete);
+const pSearchSearch = util.promisify(search.search);
+const pGetTableSchema = util.promisify(globalSchema.getTableSchema);
+
+module.exports = {
+	convertDelete: cbConvertDelete,
+};
+
+function generateReturnMessage(deleteResultsObject) {
+	return `${deleteResultsObject.deleted_hashes.length} ${RECORD}${
+		deleteResultsObject.deleted_hashes.length === 1 ? `` : `s`
+	} ${SUCCESS}`;
+}
+
+async function convertDelete({ statement, hdb_user }) {
+	//convert this update statement to a search capable statement
+	let tableInfo = await pGetTableSchema(statement.table.databaseid, statement.table.tableid);
+
+	//convert this delete statement to a SQL search capable statement
+	hdbUtils.backtickASTSchemaItems(statement);
+	let { table: from, where } = statement;
+
+	let whereString = hdbUtils.isEmpty(where) ? '' : ` WHERE  ${where.toString()}`;
+	let selectString = `SELECT ${tableInfo.hash_attribute} FROM ${from.toString()} ${whereString}`;
+	let searchStatement = alasql.parse(selectString).statements[0];
+
+	let deleteObj = {
+		operation: terms.OPERATIONS_ENUM.DELETE,
+		schema: from.databaseid_orig,
+		table: from.tableid_orig,
+		hdb_user,
+	};
+
+	try {
+		//let result = await transaction.writeTransaction(tableInfo.schema, tableInfo.name, async () => {
+		deleteObj.records = await pSearchSearch(searchStatement);
+		let result = await harperBridge.deleteRecords(deleteObj);
+		//});
+		//await write.flush({ schema: tableInfo.schema, table: tableInfo.name });
+
+		if (hdbUtils.isEmptyOrZeroLength(result.message)) {
+			result.message = generateReturnMessage(result);
+		}
+
+		delete result.txn_time;
+
+		return result;
+	} catch (err) {
+		log.error(err);
+		if (err.hdb_code) {
+			throw err.message;
+		}
+		throw err;
+	}
+}