@blazedpath/commons 0.3.1 → 0.4.0

package/blz-rds-postgres/connection-manager.js

@@ -0,0 +1,225 @@
+"use strict";
+const pg = require("pg");
+const types = require('pg').types
+const BlzBase = require("./base");
+const _ = require("underscore");
+const dayjs = require("dayjs");
+//https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
+
+types.setTypeParser(types.builtins.TIMESTAMPTZ, (value) => {
+    if (isUtcOffset(value)) {
+        value = dayjs(value).format("YYYY-MM-DDTHH:mm:ss");
+    }
+    return value;
+});
+
+types.setTypeParser(types.builtins.DATE,  (value) => {
+    return dayjs(value).format("YYYY-MM-DDTHH:mm:ss");
+})
+
+types.setTypeParser(types.builtins.TIMESTAMP,  (value) => {
+    return dayjs(value).format("YYYY-MM-DDTHH:mm:ss");
+})
+/*
+types.setTypeParser(types.builtins.TIME,  (value) => {
+})
+*/
+
+let poolsByName = new Map();
+
+function getURI(config) {
+    //postgresql://[user[:password]@][netloc][:port][,...][/dbname][?param1=value1&...]
+    let uri = ["postgresql://"];
+    if(config.user) {
+        uri.push(config.user);
+        if(config.password) {
+            uri.push(`:${config.password}`)
+        }
+        uri.push("@")
+    }
+    if(config.host) {
+        uri.push(config.host);
+    }
+    if(config.port) {
+        uri.push(`:${config.port}`);
+    }
+    if(config.database) {
+        uri.push(`/${config.database}`);
+    }
+    // if(config.ssl) {
+    //     uri.push(`?ssl=${config.ssl}`);
+    // }
+    return uri.join("");
+}
+
+function getCustomURI(config) {
+    if (config.url) {
+        let indexStart = config.url.indexOf('?schema=');
+        if (indexStart === -1) {
+            indexStart = config.url.indexOf('&schema=');
+        }
+        if (indexStart > -1) {
+            let indexEnd = config.url.indexOf('&', indexStart + 1);
+            let schemaAttribute = null;
+            if (indexEnd === -1) {
+                schemaAttribute = config.url.substring(indexStart + 1);
+            }
+            else {
+                schemaAttribute = config.url.substring(indexStart + 1, indexEnd);
+            }
+            let schemaAttributeParts = schemaAttribute.split('=');
+            if (schemaAttributeParts.length === 2) {
+                config.schema = schemaAttributeParts[1];
+            }
+        }
+    }
+    return `postgresql://${config.url}`
+}
+
+function getParameters(config = {}) {
+    let properties = _.reduce(config.properties, (memo, value) => {
+        return memo + "," + (value.key === 'statementTimeout' ? 'statement_timeout' : value.key) + "=" + value.value;
+    }, '');
+    return properties.slice("1");
+}
+
+function getObjectConfig(config = {}) {
+    if(!config.name) {
+        throw BlzBase.error('RdsPostgresException', { name: config.name });
+    }
+    if(config.customUrl) {
+        return getCustomURI(config);
+    }
+    return getURI(config);
+}
+
+function getConnectionString(config = {}) {
+    let connectionString = [ getObjectConfig(config) ];
+    let parameters = getParameters(config);
+    if(!_.isEmpty(parameters)) {
+        connectionString.push(`?${parameters}`);
+    }
+    return connectionString.join("");
+}
+
+async function getPool(config = {}) {
+    let key = config.name, poolConfig;
+    let pool = poolsByName.get(key);
+    if(pool) {
+        return pool;
+    }
+    let connectionString = getConnectionString(config);
+    poolConfig = _.pick(config, "max", "min", "connectionTimeoutMillis", "idleTimeoutMillis", "keepAlive", "parseInputDatesAsUTC", "ssl", "keepAliveInitialDelayMillis")
+    if (config.queryTimeout)
+        poolConfig.query_timeout = config.queryTimeout;
+    if (config.idleInTransactionSessionTimeout)
+        poolConfig.idle_in_transaction_session_timeout = config.idleInTransactionSessionTimeout;
+    poolConfig.connectionString = connectionString;
+    if(config.ssl) {
+        let ssl = { require: true, rejectUnauthorized: false } // Set to false if self-signed certificate is used
+        if(config.sslCertPath) {
+            ssl.cert = await BlzBase.readFile(config.sslCertPath);
+        }
+        if(config.sslKeyPath) {
+            ssl.key = await BlzBase.readFile(config.sslKeyPath);
+        }
+        if(config.sslPemPath) {
+            ssl.ca = await BlzBase.readFile(config.sslPemPath);
+        }
+        poolConfig.ssl = ssl;
+    }
+
+    try {
+        pool = new pg.Pool(poolConfig);
+        if (config.schema) {
+            pool.on('connect', (client) => {
+                let sqlSet = `SET search_path TO ${config.schema}`;
+                client.query(sqlSet);
+            });
+        }
+    } catch(error)  {
+        throw BlzBase.error('RdsPostgresCannotCreatePool', { config, uri }, error);
+    }
+    pool.name = config.name;
+    poolsByName.set(key, pool);
+    return pool;
+}
+
+async function getConnection(pool) {
+    let session;
+    try {
+        session = await pool.connect();
+    } catch(error) {
+        throw BlzBase.error('RdsPostgresCannotConnectToDatabase', { name: pool.name }, error);
+    }
+    session.name = pool.name;
+    return session;
+}
+
+async function terminate() {
+    let reject = [];
+    for (const [, pool] of poolsByName) {
+        try {
+            await pool.end();
+        } catch(error) {
+            reject.push(error);
+        }
+    }
+    poolsByName.clear();
+    if(_.isEmpty(reject)) {
+        return true;
+    }
+    throw new Error(reject);
+}
+
+const createConnection = async (config) => await getConnection(await getPool(config));
+
+
+const beginTransaction = async function(connection) {
+    try {
+        await connection.query('BEGIN');
+    } catch(error) {
+        throw BlzBase.error('RdsPostgresBeginTransactionError', { name: connection.name }, error);
+    }
+};
+
+const commitTransaction = async function(connection) {
+    try {
+        await connection.query('COMMIT');
+    } catch(error) {
+        throw BlzBase.error('RdsPostgresCommitTransactionError', { name: connection.name }, error);
+    }
+};
+
+const rollbackTransaction = async function(connection) {
+    try {
+        await connection.query('ROLLBACK');
+    } catch(error) {
+        throw BlzBase.error('RdsPostgresRollbackTransactionError', { name: connection.name }, error);
+    }
+};
+
+const close = async function(connection) {
+    try {
+        await connection.release(true);
+    } catch(error) {
+        throw BlzBase.error('RdsPostgresCloseError', { name: connection.name }, error);
+    }
+}
+
+function isUtcOffset(date) {
+    if (_.isNull(date) || _.isUndefined(date) || !_.isString(date)) {
+        return false;
+    }
+    const match = date.match(/^(.*)([+-])(\d{2})?:?(\d{2})$/);
+    return (!_.isNull(match)) && ((!_.isUndefined(match[3]) && (match[3] !== '00')) || (!_.isUndefined(match[4]) && (match[4] !== '00')));
+}
+
+module.exports = {
+    beginTransaction,
+    close,
+    commitTransaction,
+    createConnection,
+    rollbackTransaction,
+    terminate
+}

package/blz-rds-postgres/execute-bulk-insert.js

@@ -0,0 +1,81 @@
+"use strict";
+const SqlString = require('sqlstring')
+const { DateTime } = require('luxon')
+const Base = require('./base');
+
+
+
+const dateFormat = function  (value, format) {
+    const iso = new Date(value).toISOString()
+    if (format === 'ISO') {
+        return DateTime.fromISO(iso).toISO()
+    } else {
+        return DateTime.fromISO(iso).toFormat(format)
+    }
+}
+
+
+const getValue = function (source, type) {   
+    if (source == null || source === undefined) {
+        return 'null'
+    } else {
+        const _type =  type ? type.toLowerCase() : typeof source
+        switch (_type) {
+        case 'boolean':
+            return source ? 'true' : 'false'
+        case 'string':
+            if (source.includes('\'')) {
+               return `'${Base.replace(source, '\'', '\'\'')}'`
+            } else {
+                return `'${source}'`
+            }
+        case 'datetime':
+            return SqlString.escape(dateFormat(source, 'yyyy-LL-dd HH:mm:ss'))
+        case 'date':
+            return SqlString.escape(dateFormat(source,'yyyy-LL-dd'))
+        case 'time':
+            return SqlString.escape(dateFormat(source, 'HH:mm:ss' ))
+        default:
+            return source 
+        }
+    }
+}
+
+async function executeBulkInsert(connection, sql, rows, options) {
+    let autoincremental = null
+    if (options.autoincrementalData && options.autoincrementalData.pkColumnName) { 
+        autoincremental = { name: options.autoincrementalData.pkColumnName ,
+                     alias: options.autoincrementalData.pkColumnAlias,
+                    sequence: options.autoincrementalData.sequenceName
+                   }
+    }
+    const list = []
+    for(let i = 0; i < rows.length; i++) {
+        const row = rows[i]
+        const values = []
+        if (autoincremental) {
+            values.push('nextval(\''+autoincremental.sequence+'\')' )
+        }
+        for(const column of options.columns) {
+            const value = getValue(row[column.alias], column.type)
+            values.push(value)
+        }
+        list.push(`(${values.join(',')})`)
+    }
+    const _sql = autoincremental ? 'INSERT INTO ' + options.rdsTable.tableName + ' (' + autoincremental.name + ',' + options.setsFields.join(', ') + ') VALUES ' : sql
+    const returning = autoincremental ? 'RETURNING ' + autoincremental.name + ' AS "' + autoincremental.alias + '"' : ''
+    const query = `${_sql} ${list.join(',')} ${returning};`
+    const result = await connection.query(query);
+    const ids = []
+    if (autoincremental) {
+        for (const p in result.rows) {
+            const id = result.rows[p][autoincremental.alias]
+            ids.push(id)
+        }
+    }   
+    return {
+        rowsAffected: result.rowCount,
+        ids
+    };
+}
+module.exports = executeBulkInsert;

package/blz-rds-postgres/execute-bulk-merge.js

@@ -0,0 +1,93 @@
+"use strict";
+const SqlString = require('sqlstring')
+const { DateTime } = require('luxon')
+const Base = require('./base');
+
+
+
+const dateFormat = function  (value, format) {
+    const iso = new Date(value).toISOString()
+    if (format === 'ISO') {
+        return DateTime.fromISO(iso).toISO()
+    } else {
+        return DateTime.fromISO(iso).toFormat(format)
+    }
+}
+
+
+const getValue = function (source, type) {   
+    if (source == null || source === undefined) {
+        return 'null'
+    } else {
+        const _type =  type ? type.toLowerCase() : typeof source
+        switch (_type) {
+        case 'boolean':
+            return source ? 'true' : 'false'
+        case 'string':
+            if (source.includes('\'')) {
+               return `'${Base.replace(source, '\'', '\'\'')}'`
+            } else {
+                return `'${source}'`
+            }
+        case 'datetime':
+            return SqlString.escape(dateFormat(source, 'yyyy-LL-dd HH:mm:ss')) + ':: timestamp'
+        case 'date':
+            return SqlString.escape(dateFormat(source,'yyyy-LL-dd')) + ':: date'
+        case 'time':
+            return SqlString.escape(dateFormat(source, 'HH:mm:ss' )) + ':: time'
+        default:
+            return source 
+        }
+    }
+}
+
+async function executeBulkMerge(connection, sql, rows, options) {
+    let autoincremental = null
+    if (options.autoincrementalData && options.autoincrementalData.pkColumnName) { 
+        autoincremental = { name: options.autoincrementalData.pkColumnName ,
+                     alias: options.autoincrementalData.pkColumnAlias,
+                    sequence: options.autoincrementalData.sequenceName
+                   }
+    }
+
+    const list = []
+    for(let i = 0; i < rows.length; i++) {
+        const row = rows[i]
+        const values = []          
+        for(let j = 0; j < row.length; j++) {
+            const column = options.columns[j]
+            const value = getValue(row[j],column.type)
+            values.push(value)
+        }
+        list.push(`(${values.join(',')})`)
+    }
+    const _sql = []
+    _sql.push('INSERT INTO ' + options.rdsTable.tableName.toLowerCase() + ' (')
+    if (autoincremental) {
+        _sql.push(autoincremental.name.toLowerCase() + ',')
+    } 
+    _sql.push(options.setsFields.join(', ').toLowerCase())    
+    _sql.push(') SELECT ')
+    if (autoincremental) {
+        _sql.push('nextval(\''+autoincremental.sequence.toLowerCase()+'\'), ' )
+    }
+    _sql.push(options.setsFields.join(', ').toLowerCase())
+    _sql.push(' FROM (VALUES ')
+    _sql.push(list.join(','))
+    _sql.push(') AS data(')
+    _sql.push(options.setsFields.join(', ').toLowerCase())
+    _sql.push(') ON CONFLICT (')
+    _sql.push(options.mergeFields.join(', ').toLowerCase())
+    _sql.push(') DO UPDATE SET ')
+    let first = true
+    for( const field of options.setsFields) {
+        _sql.push((first? ' ': ', ') + field.toLowerCase() + ' = EXCLUDED.' + field.toLowerCase())
+        first = false
+    }
+    _sql.push(';')
+    const query = _sql.join('')
+    const result = await connection.query(query);
+    return { rowsAffected: result.rowCount };
+}
+
+module.exports = executeBulkMerge;

package/blz-rds-postgres/execute-non-query.js

@@ -0,0 +1,23 @@
+"use strict";
+const BlzBase = require('./base');
+/**
+ * @param {pg.PoolClient} connection
+ * @param {string} sql
+ * @param {[{name: string, value}]} [rdsParameters = []]
+ * @param {boolean} [withGeneratedKey = false]
+ * @return {Promise}
+ */
+async function executeNonQuery(connection, sql, rdsParameters = [], options , withGeneratedKey = false) {
+    let result;
+    if(!BlzBase.isEmpty(rdsParameters)) {
+        const { query, parameters } = BlzBase.convertSqlAndParameters(sql, rdsParameters, options);
+        result = await connection.query(query, parameters);
+    } else {
+        result = await connection.query(sql);
+    }
+    return {
+        rowsAffected: result.rowCount,
+        id: BlzBase.scalarOutput(result)
+    };
+}
+module.exports = { executeNonQuery, convertValueToDb: BlzBase.convertValueToDb }

package/blz-rds-postgres/execute-query.js

@@ -0,0 +1,37 @@
+"use strict";
+const Base = require('./base');
+
+async function executeQuery(connection, sql, rdsParameters = [], options) {    
+    if(Base.isEmpty(rdsParameters)) {
+        const checked = Base.checkPostgresSQL(sql)
+        if (!checked.safe) {
+            const err = new Error('Potential SQL injection');
+            err.code = 'SQLInjection';
+            err.data = checked.reason;
+            throw err;      
+        }
+        let data = await connection.query({
+            text : sql,
+            rowMode : 'array'
+        });
+        return { data : data.rows }
+    } else {
+        const { query, parameters } = Base.convertSqlAndParameters(sql, rdsParameters, options);
+        const checked = Base.checkPostgresSQL(query)
+        if (!checked.safe) {
+            const err = new Error('Potential SQL injection');
+            err.code = 'SQLInjection';
+            err.data = checked.reason;
+            throw err;      
+        }        
+        let data = await connection.query({
+            text : query,
+            rowMode : 'array',
+            values : parameters
+        });
+        return {
+            data: data.rows
+        }
+    }
+}
+module.exports = executeQuery;

package/blz-rds-postgres/index.js

@@ -0,0 +1,41 @@
+const _ = require("underscore");
+const { executeNonQuery } = require("./execute-non-query");
+const executeBulkInsert = require("./execute-bulk-insert");
+const executeBulkMerge = require("./execute-bulk-merge");
+const executeQuery = require("./execute-query");
+const executeProcedure = require("./stored-procedure");
+const {
+    beginTransaction,
+    close,
+    commitTransaction,
+    createConnection: createRdsConnection,
+    rollbackTransaction,
+    terminate
+} = require("./connection-manager");
+
+process.on('exit', async function () {
+    await terminate();
+});
+
+module.exports = {
+    syntaxis: require('./syntaxis.json'),    
+    executeProcedure,    
+    executeSql: async function (connection, sql, parameters, options = {}) {
+        if(sql.indexOf("call") != -1 ) {
+            return await executeProcedure(connection, sql, parameters);
+        } else if(_.has(options, "bulkInsert")) {
+            return await executeBulkInsert(connection, sql, parameters, options);     
+        } else if(_.has(options, "bulkMerge")) {
+            return await executeBulkMerge(connection, sql, parameters, options);     
+        } else if(_.has(options, "queryOne") || _.has(options, "query")) {
+            return await executeQuery(connection, sql, parameters, options);
+        } else {
+            return await executeNonQuery(connection, sql, parameters, options );
+        }
+    },
+    beginTransaction,    
+    close,
+    createRdsConnection,
+    commitTransaction,
+    rollbackTransaction,
+}

package/blz-rds-postgres/result-set.js

@@ -0,0 +1,51 @@
+"use strict";
+
+function createResultSet() {
+    let buffer = [];
+    let isFinished = false;
+    let pendingResolves = [];
+    return {        
+        getRow() {            
+            return new Promise((resolve, reject) => {
+                if (buffer.length > 0) {                    
+                    resolve(buffer.shift());
+                } else if (isFinished) {                    
+                    resolve(null);
+                } else {
+                    pendingResolves.push(resolve);
+                }                    
+            });
+        },
+        close() {
+            return new Promise((resolve, reject) => {
+                if (isFinished) {
+                    resolve(null);
+                } else {
+                    pendingResolves.push(resolve);
+                }                    
+            });
+        },
+        addRow(rowObj) {
+            if (isFinished) {
+                return;
+            }
+            let row = rowObj;// convertToArray(queryFields, rowObj);
+            if (pendingResolves.length > 0) {                
+                let resolve = pendingResolves.shift();
+                resolve(row);
+            } else {
+                buffer.push(row);
+            }                
+        },
+        end() {            
+            isFinished = true;
+            if (pendingResolves.length > 0) {
+                pendingResolves.forEach(resolve => {
+                    resolve(null);
+                });
+            }
+        }
+    }
+}
+
+module.exports = createResultSet;

package/blz-rds-postgres/stored-procedure.js

@@ -0,0 +1,116 @@
+"use strict";
+const BlzBase = require("./base");
+
+/** 
+ * @typedef {Object} Parameter
+ * @property {boolean} isResultSet
+ * @property {string} name
+ * @property {*} value
+ * @property { 'in' | 'out' | 'in/out' } direction
+ * @property {'string' | 'integer' | 'decimal' | 'boolean' | 'datetime' | 'date' | 'time' | 'binary'} type
+ */
+
+ /**
+  * 
+  * @param {mysqlx.Session} session
+  * @param {Parameter} parameter
+  * @param {*} result 
+  */
+async function parseOutputParameters(session, parameter, result, resultSet, cursors, ordinalParameter) {
+    if(parameter.isResultSet) {        
+        result[parameter.name] = cursors.shift().rows;        
+    } else {
+        let index = ordinalParameter.indexOf(parameter.name);
+        if(index == -1) {
+            throw BlzBase.error('RdsPosgreSqlIndex', { parameter, result });
+        }
+        let value = BlzBase.first(resultSet.rows)[index];
+        switch (parameter.type) {
+            case 'string':
+                result[parameter.name] = BlzBase.convertToString(value);
+                break;
+            case 'integer':
+                result[parameter.name] = BlzBase.convertToInteger(value);
+                break;
+            case 'decimal':
+                result[parameter.name] = BlzBase.convertToDecimal(value);
+                break;
+            case 'boolean':
+                result[parameter.name] = BlzBase.convertToBoolean(value);
+                break;
+            case 'datetime':
+                result[parameter.name] = BlzBase.convertToDatetime(value);
+                break;
+            case 'date':
+                result[parameter.name] = BlzBase.convertToDate(value);
+                break;
+            case 'time':
+                result[parameter.name] = BlzBase.convertToTime(value);
+                break;
+            case 'binary':
+                result[parameter.name] = BlzBase.convertToBinary(value);
+                break;
+            default:
+                throw BlzBase.error('RdsMySqlUnexpectedType', { parameter, result, value });
+        }
+    }
+}
+/**
+ * @param {mysqlx.Session} session 
+ * @param {string} procedure 
+ * @param {[Parameter]} [parameters=[]]
+ * @param {number} time_stamp
+ * @returns {string}
+ */
+async function parseInputParameters(session, procedure, parameters, list) {
+    let procedureParameters = [], timestamp = Date.now();
+    for(const parameter of parameters) {
+        if(parameter.isResultSet) {
+            procedureParameters.push(`FETCH ALL IN ${parameter.name.toLowerCase()}_${timestamp}`);
+        }
+        if (parameter.direction === 'in') {
+            let index = -1;
+            while((index = list.indexOf(parameter.name)) != -1) {
+                list.splice(index, 1);
+            }
+        }
+        if(parameter.type === 'string' && parameter.value)
+            procedure = procedure.replace(`:${parameter.name}`, `'${BlzBase.convertValueToDb(parameter.value, parameter, timestamp)}'`);
+        else
+            procedure = procedure.replace(`:${parameter.name}`, BlzBase.convertValueToDb(parameter.value, parameter, timestamp));
+    }
+    procedureParameters.unshift(procedure);    
+    return procedureParameters.join(";");
+}
+/**
+ * @param {mysqlx.Session} client 
+ * @param {string} procedure 
+ * @param {[Parameter]} [parameters=[]]
+ */
+async function executeProcedure(client, procedure, parameters) {    
+    let outParameters = {}, sql, resultSet, cursors, [, ordinalParameter ] = /\(([^\(\)]*)\)/mg.exec(procedure);
+    if(BlzBase.isArray(parameters)) {
+        ordinalParameter = ordinalParameter.replace(/[:\s]*/g, "").split(",");
+        sql = await parseInputParameters(client, procedure, parameters, ordinalParameter);
+    }
+    try {        
+        resultSet = await client.query({
+            text : sql,
+            rowMode : 'array'
+        });
+    } catch (error) {
+        throw BlzBase.error('RdsPostgresCannotExecuteProcedure', { client, procedure, parameters }, error);
+    }
+    if(BlzBase.isArray(resultSet)) {
+        [resultSet, ... cursors] = resultSet;
+    }    
+    for(const parameter of parameters) { 
+        if(parameter.direction == "in") {
+            continue;
+        }
+        await parseOutputParameters(client, parameter, outParameters, resultSet, cursors, ordinalParameter);
+    }
+    return { outParameters };
+}
+
+module.exports = executeProcedure;