@blazedpath/commons 0.3.1 → 0.4.0

package/blz-rds-mysql/connection-manager.js

@@ -0,0 +1,129 @@
+"use strict";
+const BlzBase = require("./base");
+const mysql = require("mysql");
+const poolsByName = new Map();
+const _ = require("underscore");
+const dayjs = require("dayjs");
+
+async function getPool(config = {}){
+    let key = config.name;
+    let pool = poolsByName.get(key);
+    if(pool) {
+        return pool;
+    }
+    try {
+        config.typeCast = function typeCast(field, next) {
+            if (field.type === 'TINY' && field.length === 1) {
+                return (next() === 1); // 1 = true, 0 = false
+            } else if(field.type === 'DATETIME' || field.type === 'DATE') {
+                let value = next();
+                if (isUtcOffset(value)) {
+                    value = dayjs(value).format("YYYY-MM-DDTHH:mm:ss");
+                }
+                return value;
+            } else {
+                return next();
+            }
+        };
+        config.waitForConnections = config.waitForConnections || true;
+        config.connectionLimit = config.connectionLimit || 100;
+        config.queueLimit = config.queueLimit || 100;
+        pool = mysql.createPool(config);
+        poolsByName.set(key, pool);
+    } catch(error) {
+        throw BlzBase.error('MySqlCannotCreateConnection', { config }, error);
+    }
+    return pool;
+};
+
+function getConnection(pool) {
+    return new Promise(function(resolve, reject) {
+        pool.getConnection(function (error, connection) {
+            if(error) {
+                console.error(error);
+                reject(BlzBase.error('MySqlCannotCreateConnection', { connection }, error));
+                return;
+            }
+            connection.name = pool.name;
+            resolve(connection);
+        });
+    });
+}
+
+const createConnection = async (config) => await getConnection(await getPool(config));
+
+function beginTransaction(connection) {
+    return new Promise(function(resolve, reject) {
+        connection.beginTransaction(function(err){
+            if(err) {
+                reject(BlzBase.error('MySqlCannotBeginTransaction', { connection }, err));
+                return;
+            }
+            resolve();
+        });
+    });
+};
+
+function commitTransaction(connection) {
+    return new Promise(function(resolve, reject) {
+        connection.commit(function(err){
+            if(err) {
+                reject(BlzBase.error('MySqlCannotCommitTransaction', { connection }, err));
+                return;
+            }
+            resolve();
+        });
+    });
+};
+
+function rollbackTransaction(connection) {
+    return new Promise(function(resolve, reject) {
+        connection.rollback(function(err){
+            if(err) {
+                reject(BlzBase.error('MySqlCannotRollBackTransaction', { connection }, err));
+                return;
+            }
+            resolve();
+        });
+    });
+};
+
+async function close(connection) {
+    connection.release();
+    connection.destroy();
+}
+
+async function terminate() {
+    let reject = [];
+    for (const [, pool] of poolsByName) {
+        try {
+            await pool.end();
+        } catch(error) {
+            reject.push(error);
+        }
+    }
+    poolsByName.clear();
+    if(_.isEmpty(reject)) {
+        return true;
+    }
+    throw new Error(reject);
+}
+
+function isUtcOffset(date) {
+    if (_.isNull(date) || _.isUndefined(date) || !_.isString(date)) {
+        return false;
+    }
+    const match = date.match(/^(.*)([+-])(\d{2})?:?(\d{2})$/);
+    return (!_.isNull(match)) && ((!_.isUndefined(match[3]) && (match[3] !== '00')) || (!_.isUndefined(match[4]) && (match[4] !== '00')));
+}
+
+module.exports = {
+    beginTransaction,
+    close,
+    commitTransaction,
+    createConnection,
+    getConnection,
+    getPool,
+    rollbackTransaction,
+    terminate
+};

package/blz-rds-mysql/execute-bulk-insert.js

@@ -0,0 +1,35 @@
+"use strict";
+const {convertValueToDb } = require("./execute-non-query");
+const BlzBase = require("./base");
+
+async function executeBulkInsert(connection, sql, rows, options) {
+    const query = sql + ' ?'    
+    const list = []
+    for(let i = 0; i < rows.length; i++) {
+        const row = rows[i]
+        const values = []
+        for(let j = 0; j < row.length; j++) {
+            const value = convertValueToDb(row[j])
+            values.push(value)
+        }
+        list.push(values)
+    }
+    return new Promise(function(resolve, reject){
+        connection.query(query, [list], function(error, result, fields) {
+            if(error) {
+                reject(BlzBase.error('RdsMySqlExecuteError', { name: connection.name, query, rows }, error));
+                return;
+            }
+            if (options.autoincrementalData) { 
+                const start = result.insertId
+                const end = result.insertId + (result.affectedRows - 1)
+                const ids= []
+                for (let i = start; i <= end; i++)ids.push(i)
+                resolve({ ids, affectedRows: result.affectedRows });
+            } else {
+                resolve({affectedRows: result.affectedRows});
+            }            
+        })
+    })
+}
+module.exports = executeBulkInsert;

package/blz-rds-mysql/execute-bulk-merge.js

@@ -0,0 +1,45 @@
+"use strict";
+const {convertValueToDb } = require("./execute-non-query");
+const BlzBase = require("./base");
+const SqlString = require('sqlstring')
+
+async function executeBulkMerge(connection, sql, rows, options) {
+    const list = []
+    for(let i = 0; i < rows.length; i++) {
+        const row = rows[i]
+        const values = []          
+        for(let j = 0; j < row.length; j++) {
+            const column = options.columns[j]
+            let value = convertValueToDb(row[j])
+            if ( ['string','date','datetime','time'].includes(column.type)) {
+                value = SqlString.escape(value)
+            }
+            values.push(value)
+        }
+        list.push(`(${values.join(',')})`)
+    }
+    const _sql = []
+    _sql.push('INSERT INTO ' + options.rdsTable.tableName + ' (')
+    _sql.push(options.setsFields.map(p=> '`'+p+'`').join(', '))    
+    _sql.push(')\n\rVALUES\n\r')
+    _sql.push(list.join(','))
+    _sql.push('\n\rON DUPLICATE KEY UPDATE\n\r')
+    let first = true
+    for( const field of options.setsFields.map(p=> '`'+p+'`')) {
+        _sql.push((first? ' ': ',\n\r ') + field + ' = VALUES(' + field + ')')
+        first = false
+    }
+    _sql.push(';')
+    const query = _sql.join('')
+
+    return new Promise(function(resolve, reject){
+        connection.query(query, null, function(error, result, fields) {
+            if(error) {
+                reject(BlzBase.error('RdsMySqlExecuteError', { name: connection.name, query, rows }, error));
+                return;
+            }
+            resolve({affectedRows: result.affectedRows});
+        })
+    })
+}
+module.exports = executeBulkMerge;

package/blz-rds-mysql/execute-non-query.js

@@ -0,0 +1,34 @@
+"use strict";
+const _ = require('underscore');
+const BlzBase = require("./base");
+const toUnnamed = require('named-placeholders')();
+/**
+ * @param {*} connection
+ * @param {string} sql
+ * @param {[{name: string, value: *}]} [rdsParameters = []]
+ * @returns {Promise<{row: number, id: number}>}
+ * @public
+ */
+async function executeNonQuery(connection, sql, rdsParameters, options) {
+    let query = sql, parameters = [], rs = null;
+    try {
+        if(!_.isEmpty(rdsParameters)) {
+            [query, parameters] = toUnnamed(sql, BlzBase.reduceParameters(rdsParameters, options));
+        }
+    } catch(error) {
+        throw BlzBase.error('RdsAuroraExecuteError', { name: connection.name, sql, parameters }, error);
+    }
+    return new Promise(function(resolve, reject) {
+        connection.query(query, parameters, function(err, result, fields) {
+            if(err) {
+                reject(BlzBase.error('RdsAuroraExecuteError', { name: connection.name, sql, parameters }, err));
+                return;
+            }
+            resolve({
+                rowsAffected : result.affectedRows,
+                id : result.insertId
+            });
+        });
+    })
+}
+module.exports = { executeNonQuery }

package/blz-rds-mysql/execute-query.js

@@ -0,0 +1,50 @@
+"use strict";
+const _ = require('underscore');
+const toUnnamed = require('named-placeholders')();
+const BlzBase = require("./base");
+
+/**
+ * @param {*} connection
+ * @param {string} sql
+ * @param {[{name: string, value: *}]} [rdsParameters = []]
+ * @public
+ */
+async function executeQuery(connection, sql, rdsParameters = [], options) {
+    let query = sql, parameters = [];
+    try {
+        if(!_.isEmpty(rdsParameters)) {
+            [query, parameters] = toUnnamed(sql, BlzBase.reduceParameters(rdsParameters, options));
+        }
+    } catch(error) {
+        throw BlzBase.error('RdsMySqlExecuteError', { name: connection.name, sql, parameters }, error);
+    }
+    return new Promise(function(resolve, reject){
+        connection.query(query, parameters, function(error, results, fields) {
+            if(error) {
+                reject(BlzBase.error('RdsMySqlExecuteError', { name: connection.name, sql, parameters }, error));
+                return;
+            }
+            resolve({resultSet: new ResultSet(results, fields)});
+        });
+    });
+}
+
+function ResultSet(results, fields) {
+    return {
+        getRow: async function() {
+            let data = results.shift();
+            if(!data) {
+                return null;
+            }
+            return _.map(fields, field =>
+                data[field.name]
+            );
+        },
+        close: async function() {
+            results = null;
+            fields  = null;
+            return true;
+        }
+    }
+}
+module.exports = executeQuery;

package/blz-rds-mysql/index.js

@@ -0,0 +1,41 @@
+const _ = require("underscore");
+const {executeNonQuery } = require("./execute-non-query");
+const executeQuery = require("./execute-query");
+const executeBulkInsert = require("./execute-bulk-insert");
+const executeBulkMerge = require("./execute-bulk-merge");
+const executeProcedure = require("./stored-procedure");
+const {
+    beginTransaction,
+    close,
+    commitTransaction,
+    createConnection: createRdsConnection,
+    rollbackTransaction,
+    terminate
+} = require("./connection-manager");
+
+process.on('exit', async function () {
+    await terminate();
+});
+
+module.exports = {
+    syntaxis: require("./syntaxis.json"),    
+    executeProcedure,    
+    executeSql: async function (connection, sql, parameters, options = {}) {
+        if(sql.indexOf("call") != -1 ) {
+            return await executeProcedure(connection, sql, parameters);
+        } else if(_.has(options, "bulkInsert")) {
+            return await executeBulkInsert(connection, sql, parameters, options);
+        } else if(_.has(options, "bulkMerge")) {
+            return await executeBulkMerge(connection, sql, parameters, options);        
+        } else if(_.has(options, "queryOne") || _.has(options, "query")) {
+            return await executeQuery(connection, sql, parameters, options);
+        } else {
+            return await executeNonQuery(connection, sql, parameters, options);
+        }
+    },
+    beginTransaction,    
+    close,
+    createRdsConnection,
+    commitTransaction,
+    rollbackTransaction,
+}

package/blz-rds-mysql/stored-procedure.js

@@ -0,0 +1,207 @@
+"use strict";
+const mysql = require("mysql");
+const _ = require('underscore');
+const BlzBase = require("./base");
+const toString = Object.prototype.toString;
+//https://dev.mysql.com/doc/dev/connector-nodejs/8.0/tutorial-Getting_Started.html
+//https://dev.mysql.com/doc/x-devapi-userguide/en/using-sql.html
+//https://dev.mysql.com/doc/refman/8.0/en/faqs-stored-procs.html#faq-mysql-can-pass-cursor-in
+
+function tryDatetime(value) {
+    let convertMilliseconds = function(strMilliseconds) {
+        if (strMilliseconds) {
+            if (strMilliseconds.length < 3)
+                strMilliseconds = strMilliseconds.padEnd(3, '0');
+            if (strMilliseconds.length > 3)
+                strMilliseconds = strMilliseconds.substr(0, 3);
+            return Number(strMilliseconds);
+        }
+        else
+            return 0;
+    };
+    let strType = toString.call(value);
+    if (strType === '[object Date]') {
+        return value;
+    }
+    else if (strType === '[object String]') {
+        let match = /^(\d{4})-(\d{1,2})-(\d{1,2})[T,\s](\d{1,2})\:(\d{1,2})\:(\d{1,2})\.?(\d+)?Z?$/.exec(value);
+        if (match)
+            return new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]), Number(match[4]), Number(match[5]), Number(match[6] || 0), convertMilliseconds(match[7])));
+    }
+    return null;
+}
+
+function convertValueToDb (value) {
+    if (_.isUndefined(value)) {
+        return null;
+    }
+    if(_.isNull(value)) {
+        return null;
+    }
+    if (value === true) {
+        return 1;
+    }
+    if (value === false) {
+        return 0;
+    }
+    let datetime = tryDatetime(value);
+    if (datetime) {
+        let result = datetime.toJSON();
+        if (result.length === 24) {
+            result = result.substring(0, 19);
+        }
+        return result;
+    }
+    return value;
+}
+
+/**
+ * @typedef {Object} Parameter
+ * @property {boolean} isResultSet
+ * @property {string} name
+ * @property {*} value
+ * @property { 'in' | 'out' | 'in/out' } direction
+ * @property {'string' | 'integer' | 'decimal' | 'boolean' | 'datetime' | 'date' | 'time' | 'binary'} type
+ */
+
+ /**
+  *
+  * @param {mysqlx.Session} session
+  * @param {Parameter} parameter
+  * @param {*} result
+  */
+async function parseOutputParameters(session, parameter, result, time_stamp) {
+    let resultSet = null, value = null;
+    try {
+        resultSet = await (new Promise(function(resolve, reject) {
+            let sqlSelect = `SELECT @${parameter.name}_${time_stamp};`;
+            session.query(sqlSelect,(err, result, field) => {
+                if(err) {
+                    return reject(err);
+                }
+                resolve(result);
+            })
+        }));
+    } catch(error) {
+        throw BlzBase.error('RdsMysqlCannotConvertParameters', { parameter, result, time_stamp }, error);
+    }
+    let property = _.property(`@${parameter.name}_${time_stamp}`);
+    value = property(_.first(resultSet));
+    if(parameter.isResultSet) {
+        result[parameter.name] = JSON.parse(value);
+        return;
+    }
+    switch (parameter.type) {
+        case 'string':
+            result[parameter.name] = BlzBase.convertToString(value);
+            break;
+        case 'integer':
+            result[parameter.name] = BlzBase.convertToInteger(value);
+            break;
+        case 'decimal':
+            result[parameter.name] = BlzBase.convertToDecimal(value);
+            break;
+        case 'boolean':
+            result[parameter.name] = BlzBase.convertToBoolean(value);
+            break;
+        case 'datetime':
+            result[parameter.name] = BlzBase.convertToDatetime(value);
+            break;
+        case 'date':
+            result[parameter.name] = BlzBase.convertToDate(value);
+            break;
+        case 'time':
+            result[parameter.name] = BlzBase.convertToTime(value);
+            break;
+        case 'binary':
+            result[parameter.name] = BlzBase.convertToBinary(value);
+            break;
+        default:
+            throw BlzBase.error('RdsMySqlUnexpectedType', { parameter, result, value });
+    }
+
+}
+
+/**
+ * @param {mysqlx.Session} session
+ * @param {string} procedure
+ * @param {[Parameter]} [parameters=[]]
+ * @param {number} time_stamp
+ * @returns {string}
+ */
+async function parseInputParameters(session, procedure, parameters, time_stamp) {
+    for(const parameter of parameters) {
+        switch(parameter.direction) {
+            case "in":
+            case "in/out":
+                try {
+                    await (new Promise(function(resolve, reject) {
+                        let sqlSet = `SET @${parameter.name}_${time_stamp} = ?;`;
+                        session.query(sqlSet, [convertValueToDb(parameter.value)], (err, result, field) => {
+                            if(err) {
+                                return reject(err);
+                            }
+                            resolve();
+                        })
+                    }));
+                } catch (error) {
+                    throw BlzBase.error('RdsMySqlInputParameterType', { parameters, time_stamp });
+                }
+                procedure = procedure
+                    .replace(`:${parameter.name}`, `@${parameter.name}_${time_stamp}`);
+                break;
+            case "out":
+                try {
+                    await (new Promise(function(resolve, reject) {
+                        let sqlSet = `SET @${parameter.name}_${time_stamp} = NULL;`;
+                        session.query(sqlSet, (err, result, field) => {
+                            if(err) {
+                                return reject(err);
+                            }
+                            resolve();
+                        });
+                    }));
+                } catch(error) {
+                    throw BlzBase.error('RdsMySqlInputParameterType', { parameters, time_stamp });
+                }
+                procedure = procedure
+                    .replace(`:${parameter.name}`, `@${parameter.name}_${time_stamp}`);
+                break;
+        }
+    }
+    return procedure;
+}
+
+/**
+ * @param {mysqlx.Session} session
+ * @param {string} procedure
+ * @param {[Parameter]} [parameters=[]]
+ */
+async function executeProcedure(session, procedure, parameters) {
+    const time_stamp = Date.now();
+    if(_.isArray(parameters)) {
+        procedure = await parseInputParameters(session, procedure, parameters, time_stamp);
+    }
+    try {
+        await (new Promise(function(resolve, reject) {
+            session.query(procedure, (err, result, field) => {
+                if(err) {
+                    return reject(err);
+                }
+                resolve(result);
+            })
+        }));
+    } catch (error) {
+        throw BlzBase.error('RdsMysqlCannotExecuteProcedure', { session, procedure, parameters }, error);
+    }
+    let outParameters = {};
+    for(const parameter of parameters) {
+        if(parameter.direction == "in") {
+            continue;
+        }
+        await parseOutputParameters(session, parameter, outParameters, time_stamp);
+    }
+    return { outParameters };
+}
+
+module.exports = executeProcedure;