@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/utility/functions/geo.js

@@ -0,0 +1,355 @@
+'use strict';
+
+/***
+ * geo.js
+ *
+ * Module created as a wrapper for our implementation of turf.js into the sql parser
+ * turf.js has very robust internal validation as such we offload the validation to turf.js
+ */
+
+const turfArea = require('@turf/area');
+const turfLength = require('@turf/length');
+const turfCircle = require('@turf/circle');
+const turfDifference = require('@turf/difference');
+const turfDistance = require('@turf/distance');
+const turfBooleanContains = require('@turf/boolean-contains');
+const turfBooleanEqual = require('@turf/boolean-equal');
+const turfBooleanDisjoint = require('@turf/boolean-disjoint');
+const turfHelpers = require('@turf/helpers');
+const hdbTerms = require('../hdbTerms.ts');
+const commonUtils = require('../common_utils.js');
+const hdbLog = require('../logging/harper_logger.js');
+
+module.exports = {
+	geoArea,
+	geoLength,
+	geoCircle,
+	geoDifference,
+	geoDistance,
+	geoNear,
+	geoContains,
+	geoEqual,
+	geoCrosses,
+	geoConvert,
+};
+
+/***
+ * Takes one or more features and returns the area in square meters
+ * @param geoJSON
+ * @returns {number}
+ */
+function geoArea(geoJSON) {
+	if (commonUtils.isEmpty(geoJSON)) {
+		return NaN;
+	}
+
+	if (typeof geoJSON === 'string') {
+		geoJSON = commonUtils.autoCastJSON(geoJSON);
+	}
+	try {
+		return turfArea.default(geoJSON);
+	} catch (err) {
+		hdbLog.trace(err, geoJSON);
+		return NaN;
+	}
+}
+
+/***
+ * Takes a GeoJSON and measures its length in the specified units (default is kilometers)
+ * @param geoJSON
+ * @param units
+ * @returns {number}
+ */
+function geoLength(geoJSON, units) {
+	if (commonUtils.isEmpty(geoJSON)) {
+		return NaN;
+	}
+
+	if (typeof geoJSON === 'string') {
+		geoJSON = commonUtils.autoCastJSON(geoJSON);
+	}
+
+	try {
+		return turfLength.default(geoJSON, { units: units ? units : 'kilometers' });
+	} catch (err) {
+		hdbLog.trace(err, geoJSON);
+		return NaN;
+	}
+}
+
+/***
+ * Takes a Point and calculates the circle polygon given a radius in units (default units are kilometers)
+ * @param point
+ * @param radius
+ * @param units
+ * @returns {Feature<Polygon>}
+ */
+function geoCircle(point, radius, units) {
+	if (commonUtils.isEmpty(point)) {
+		return NaN;
+	}
+
+	if (commonUtils.isEmpty(radius)) {
+		return NaN;
+	}
+
+	if (typeof point === 'string') {
+		point = commonUtils.autoCastJSON(point);
+	}
+
+	try {
+		return turfCircle.default(point, radius, { units: units ? units : 'kilometers' });
+	} catch (err) {
+		hdbLog.trace(err, point, radius);
+		return NaN;
+	}
+}
+
+/***
+ * returns a new polygon with the difference of the second polygon clipped from the first polygon
+ * @param poly1
+ * @param poly2
+ * @returns {Feature<Polygon | MultiPolygon> | null}
+ */
+function geoDifference(poly1, poly2) {
+	if (commonUtils.isEmpty(poly1)) {
+		return NaN;
+	}
+
+	if (commonUtils.isEmpty(poly2)) {
+		return NaN;
+	}
+
+	if (typeof poly1 === 'string') {
+		poly1 = commonUtils.autoCastJSON(poly1);
+	}
+
+	if (typeof poly2 === 'string') {
+		poly2 = commonUtils.autoCastJSON(poly2);
+	}
+
+	try {
+		return turfDifference(poly1, poly2);
+	} catch (err) {
+		hdbLog.trace(err, poly1, poly2);
+		return NaN;
+	}
+}
+
+/***
+ * Calculates the distance between two points, default unit is kilometers
+ * @param point1
+ * @param point2
+ * @param units
+ * @returns {number}
+ */
+function geoDistance(point1, point2, units) {
+	if (commonUtils.isEmpty(point1)) {
+		return NaN;
+	}
+
+	if (commonUtils.isEmpty(point2)) {
+		return NaN;
+	}
+
+	if (typeof point1 === 'string') {
+		point1 = commonUtils.autoCastJSON(point1);
+	}
+	if (typeof point2 === 'string') {
+		point2 = commonUtils.autoCastJSON(point2);
+	}
+
+	try {
+		return turfDistance.default(point1, point2, { units: units ? units : 'kilometers' });
+	} catch (err) {
+		hdbLog.trace(err, point1, point2);
+		return NaN;
+	}
+}
+
+/***
+ * determines if point1 and point2 are within a specified distance from each other, default units are kilometers
+ * @param point1
+ * @param point2
+ * @param distance
+ * @param units
+ * @returns {boolean}
+ */
+function geoNear(point1, point2, distance, units) {
+	if (commonUtils.isEmpty(point1)) {
+		return false;
+	}
+
+	if (commonUtils.isEmpty(point2)) {
+		return false;
+	}
+
+	if (commonUtils.isEmpty(distance)) {
+		throw new Error('distance is required');
+	}
+
+	if (typeof point1 === 'string') {
+		point1 = commonUtils.autoCastJSON(point1);
+	}
+	if (typeof point2 === 'string') {
+		point2 = commonUtils.autoCastJSON(point2);
+	}
+
+	if (isNaN(distance)) {
+		throw new Error('distance must be a number');
+	}
+
+	try {
+		let pointsDistance = geoDistance(point1, point2, units);
+		return pointsDistance <= distance;
+	} catch (err) {
+		hdbLog.trace(err, point1, point2);
+		return false;
+	}
+}
+
+/***
+ * Determines if geo2 is completely contained by geo1
+ * @param geo1
+ * @param geo2
+ * @returns {boolean}
+ */
+function geoContains(geo1, geo2) {
+	if (commonUtils.isEmpty(geo1)) {
+		return false;
+	}
+
+	if (commonUtils.isEmpty(geo2)) {
+		return false;
+	}
+
+	if (geo1.coordinates && geo1.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (geo2.coordinates && geo2.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (typeof geo1 === 'string') {
+		geo1 = commonUtils.autoCastJSON(geo1);
+	}
+	if (typeof geo2 === 'string') {
+		geo2 = commonUtils.autoCastJSON(geo2);
+	}
+
+	try {
+		return turfBooleanContains.default(geo1, geo2);
+	} catch (err) {
+		hdbLog.trace(err, geo1, geo2);
+		return false;
+	}
+}
+
+/***
+ * Determines if geo1 & geo2 are the same type and have identical x,y coordinate values based on: http://edndoc.esri.com/arcsde/9.0/generalTopics/understandSpatialRelations.htm
+ * @param geo1
+ * @param geo2
+ * @returns {boolean}
+ */
+function geoEqual(geo1, geo2) {
+	if (commonUtils.isEmpty(geo1)) {
+		return false;
+	}
+
+	if (commonUtils.isEmpty(geo2)) {
+		return false;
+	}
+
+	if (geo1.coordinates && geo1.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (geo2.coordinates && geo2.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (typeof geo1 === 'string') {
+		geo1 = commonUtils.autoCastJSON(geo1);
+	}
+	if (typeof geo2 === 'string') {
+		geo2 = commonUtils.autoCastJSON(geo2);
+	}
+
+	try {
+		return turfBooleanEqual.default(geo1, geo2);
+	} catch (err) {
+		hdbLog.trace(err, geo1, geo2);
+		return false;
+	}
+}
+
+/***
+ * Determines if the geometries cross over each other
+ * @param geo1
+ * @param geo2
+ * @returns {boolean}
+ */
+function geoCrosses(geo1, geo2) {
+	if (commonUtils.isEmpty(geo1)) {
+		return false;
+	}
+
+	if (commonUtils.isEmpty(geo2)) {
+		return false;
+	}
+
+	if (geo1.coordinates && geo1.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (geo2.coordinates && geo2.coordinates.includes?.(null)) {
+		return false;
+	}
+
+	if (typeof geo1 === 'string') {
+		geo1 = commonUtils.autoCastJSON(geo1);
+	}
+	if (typeof geo2 === 'string') {
+		geo2 = commonUtils.autoCastJSON(geo2);
+	}
+
+	try {
+		//need to do ! as this checks for non-intersections of geometries
+		return !turfBooleanDisjoint.default(geo1, geo2);
+	} catch (err) {
+		hdbLog.trace(err, geo1, geo2);
+		return false;
+	}
+}
+
+/***
+ * Converts a series of coordinates into the desired type
+ * @param coordinates
+ * @param geoType
+ * @param properties
+ * @returns {*}
+ */
+function geoConvert(coordinates, geoType, properties) {
+	if (commonUtils.isEmptyOrZeroLength(coordinates)) {
+		throw new Error('coordinates is required');
+	}
+
+	if (commonUtils.isEmpty(geoType)) {
+		throw new Error('geo_type is required');
+	}
+
+	if (commonUtils.isEmpty(hdbTerms.GEO_CONVERSION_ENUM[geoType])) {
+		throw new Error(
+			`geoType of ${geoType} is invalid please use one of the following types: ${Object.keys(
+				hdbTerms.GEO_CONVERSION_ENUM
+			).join(',')}`
+		);
+	}
+
+	if (typeof coordinates === 'string') {
+		coordinates = commonUtils.autoCastJSON(coordinates);
+	}
+
+	return turfHelpers[geoType](coordinates, properties);
+}

package/utility/functions/sql/alaSQLExtension.js

@@ -0,0 +1,104 @@
+'use strict';
+
+/***
+ * alaSQLExtension.js
+ * purpose of this module is to hold custom functions for alasql
+ */
+
+const _ = require('lodash');
+const mathjs = require('mathjs');
+const jsonata = require('jsonata');
+const hdbUtils = require('../../common_utils.js');
+
+module.exports = {
+	/***
+	 * distinctArray takes in an array an dedupes its values using lodash. this works on complex as well as simple datatypes
+	 * @param array
+	 * @returns array
+	 */
+	distinct_array: (array) => {
+		if (Array.isArray(array) && array.length > 1) {
+			return _.uniqWith(array, _.isEqual);
+		}
+
+		return array;
+	},
+	searchJSON,
+	/***
+	 * median absolute deviation aggregate function based on http://mathjs.org/docs/reference/functions/mad.html
+	 */
+	mad: aggregateFunction.bind(null, mathjs.mad),
+	/***
+	 * mean aggregate function based on http://mathjs.org/docs/reference/functions/mean.html
+	 */
+	mean: aggregateFunction.bind(null, mathjs.mean),
+	/***
+	 * computes the mode of values on http://mathjs.org/docs/reference/functions/mode.html
+	 */
+	mode: aggregateFunction.bind(null, mathjs.mode),
+	/***
+	 * compute the product based on http://mathjs.org/docs/reference/functions/prod.html
+	 */
+	prod: aggregateFunction.bind(null, mathjs.prod),
+	/***
+	 * compute the median based on http://mathjs.org/docs/reference/functions/median.html
+	 */
+	median: aggregateFunction.bind(null, mathjs.median),
+};
+
+/***
+ * handles the 3 pass loop for aggregates and executes the final calc with the passed in aggregator function
+ * alasql's stages work like the following:
+ * stage 1 is the very first record and requires you to return what is the array variable from then on
+ * stage 2 occurs for every following row
+ * stage 3 is where the processing occurs and returns the final result
+ * @param calculationFunction - function to execute to perform the calculation
+ * @param value - value per row
+ * @param array - the aggregate list of values
+ * @param stage - defines the stage in processing see description above
+ * @returns {*}
+ */
+function aggregateFunction(calculationFunction, value, array, stage) {
+	if (stage === 1) {
+		if (value === null || value === undefined) {
+			return [];
+		}
+
+		return [value];
+	} else if (stage === 2) {
+		if (value !== null && value !== undefined) {
+			array.push(value);
+		}
+		return array;
+	} else {
+		if (array !== null && array !== undefined && array.length > 0) {
+			return calculationFunction(array);
+		}
+
+		return null;
+	}
+}
+
+/**
+ * wrapper function that implements the JSONata library, which performs searches, transforms, etc... on JSON
+ * @param {String} jsonataExpression - the JSONata expression to execute
+ * @param {any} data - data which will be evaluated
+ * @returns {any}
+ */
+function searchJSON(jsonataExpression, data) {
+	if (typeof jsonataExpression !== 'string' || jsonataExpression.length === 0) {
+		throw new Error('search json expression must be a non-empty string');
+	}
+
+	let alias = '__' + jsonataExpression + '__';
+
+	if (hdbUtils.isEmpty(this.__ala__.res)) {
+		this.__ala__.res = {};
+	}
+
+	if (hdbUtils.isEmpty(this.__ala__.res[alias])) {
+		let expression = jsonata(jsonataExpression);
+		this.__ala__.res[alias] = expression;
+	}
+	return this.__ala__.res[alias].evaluate(data);
+}

package/utility/globalSchema.js

@@ -0,0 +1,35 @@
+const systemSchema = require('../json/systemSchema.json');
+const { promisify } = require('util');
+const { getDatabases } = require('../resources/databases.ts');
+
+module.exports = {
+	setSchemaDataToGlobal,
+	getTableSchema,
+	getSystemSchema,
+	setSchemaDataToGlobalAsync: promisify(setSchemaDataToGlobal),
+};
+
+function setSchemaDataToGlobal(callback) {
+	global.hdb_schema = getDatabases();
+	if (callback) callback();
+}
+
+function getTableSchema(schemaName, tableName, callback) {
+	const database = getDatabases()[schemaName];
+	if (!database) {
+		return callback(`schema ${schemaName} does not exist`);
+	}
+	const table = database[tableName];
+	if (!table) {
+		return callback(`table ${schemaName}.${tableName} does not exist`);
+	}
+	return callback(null, {
+		schema: schemaName,
+		name: tableName,
+		hash_attribute: table.primaryKey,
+	});
+}
+
+function getSystemSchema() {
+	return systemSchema;
+}