@igojs/db 6.0.0-beta.1

package/src/Sql.js

@@ -0,0 +1,311 @@
+
+const _       = require('lodash');
+
+module.exports = class Sql {
+  
+  constructor(query, dialect) {
+    this.query    = query;
+    this.dialect  = dialect;
+    this.i        = 1;
+  }
+
+  // SELECT SQL
+  selectSQL() {
+    const { query, dialect } = this;
+    const { esc } = dialect;
+
+    let sql       = 'SELECT ';
+    const params  = [];
+
+    if (query.distinct) {
+      const joined = query.distinct.join(`${esc},${esc}`);
+      sql += `DISTINCT ${esc}${joined}${esc} `;
+    } else if (query.select) {
+      let select_sql = query.select;
+      _.each(query.joins, join => {
+        const [assoc_type, name, Obj] = join.association;
+        select_sql = select_sql.replace(new RegExp(`\\b${Obj.schema.table}\\b`, 'g'), name);
+      });
+      sql += select_sql + ' ';
+    } else {
+      sql += `${esc}${query.table}${esc}.*`;
+
+      _.each(query.joins, join => {
+        const { src_schema, association } = join;
+        const [ assoc_type, name, Obj, src_column, column] = association;
+        const table_alias = name;
+        sql += ', ';
+        if (join.columns) {
+          // add only specified columns
+          sql += _.map(join.columns, (column) => {
+            if (column.indexOf('.') > -1 || column.toLowerCase().indexOf(' as ') > -1) {
+              return column; // already qualified or aliased
+            }
+            // 
+            return `${esc}${table_alias}${esc}.${esc}${column}${esc} as ${esc}${table_alias}__${column}${esc}`;
+          }).join(', ');
+        } else {
+          // add all columns from joined table          
+          sql += _.map(Obj.schema.columns, (column) => {
+            return `${esc}${table_alias}${esc}.${esc}${column.name}${esc} as ${esc}${table_alias}__${column.name}${esc}`;
+          }).join(', ');
+        }
+      });
+      sql += ' ';
+    }
+
+    // from
+    sql += `FROM ${esc}${query.table}${esc} `;
+
+    // joins
+    sql += this.addJoins();
+
+    // where
+    sql += this.whereSQL(params);
+
+    // whereNot
+    sql += this.whereNotSQL(params);
+
+    // group
+    sql += this.groupSQL();
+
+    // order by
+    sql += this.orderSQL();
+
+
+    // limit
+    if (query.limit) {
+      sql += dialect.limit(this.i++, this.i++);
+      params.push(query.offset || 0);
+      params.push(query.limit);
+    }
+
+    const ret = {
+      sql: sql.trim(),
+      params: params
+    };
+    // console.dir(ret);
+    return ret;
+  };
+
+  // COUNT SQL
+  countSQL() {
+    const { query, dialect } = this;
+    const { esc } = dialect;
+
+    // select
+    let sql = `SELECT COUNT(0) as ${esc}count${esc} `;
+    const params = [];
+
+    // from
+    sql += `FROM ${esc}${query.table}${esc} `;
+
+    // joins
+    sql += this.addJoins();
+
+    // where
+    sql += this.whereSQL(params);
+
+    // whereNot
+    sql += this.whereNotSQL(params);
+
+    var ret = {
+      sql: sql.trim(),
+      params: params
+    };
+    // console.dir(ret);
+    return ret;
+  };
+
+  // JOINS
+  addJoins() {
+    const { query, dialect } = this;
+    const { esc }   = dialect;
+
+    let sql = '';
+    _.each(query.joins, join => {
+      const { src_schema, type, association, src_alias } = join;
+      const [ assoc_type, name, Obj, src_column, column] = association;
+      const src_table_alias = src_alias || src_schema.table;
+      const table       = Obj.schema.table;
+      sql += `${type.toUpperCase()} JOIN ${esc}${table}${esc} AS ${esc}${name}${esc} ON ${esc}${name}${esc}.${esc}${column}${esc} = ${esc}${src_table_alias}${esc}.${esc}${src_column}${esc} `;
+    });
+    return sql;
+  }
+
+  // WHERE
+  whereSQL(params, not) {
+    const { query, dialect } = this;
+    const { esc } = dialect;
+
+    const sqlwhere = [];
+    const wheres = not ? query.whereNot : query.where;
+    _.forEach(wheres, (where) => {
+      if (_.isArray(where)) {
+        // where is an array with sql string and params
+        if (not) {
+          console.warn('Where clause contains a string with whereNot, this is not supported');
+          return;
+        }
+        // where is an array
+        let s = where[0];
+        while (s.indexOf('$?') > -1) {
+          s = s.replace('$?', dialect.param(this.i++));
+        }
+        sqlwhere.push(s + ' ');
+        if (_.isArray(where[1])) {
+          Array.prototype.push.apply(params, where[1]);
+        } else {
+          params.push(where[1]);
+        }
+      } else if (_.isString(where)) {
+        // where is a string
+        if (not) {
+          console.warn('Where clause is a string with whereNot, this is not supported');
+          return;
+        }
+        sqlwhere.push(where + ' ');
+      } else {
+        // where is an object
+        _.forOwn(where, (value, key) =>{
+          let column_alias;
+          if (key.indexOf('.') > -1 && key.indexOf('`') === -1) {
+            column_alias = _.map(key.split('.'), (part) => (`${esc}${part}${esc}`)).join('.');
+          } else {
+            column_alias = `${esc}${query.table}${esc}.${esc}${key}${esc}`
+          }
+          if (value === null || value === undefined) {
+            sqlwhere.push(`${column_alias} IS ${not ? 'NOT NULL' : 'NULL'} `);
+          } else if (_.isArray(value) && value.length === 0) {
+            // where in empty array
+            sqlwhere.push(not ? 'TRUE ': 'FALSE ');
+          } else if (_.isArray(value)) {
+            sqlwhere.push(`${column_alias} ${not ? dialect.notin : dialect.in} (${dialect.param(this.i++)}) `);
+            params.push(value);
+          } else {
+            sqlwhere.push(`${column_alias} ${not ? '!=' : '='} ${dialect.param(this.i++)} `);
+            params.push(value);
+          }
+        });
+      }
+    });
+    
+    if (sqlwhere.length) {
+      const ret = (not && query.where.length > 0) ? 'AND ' : 'WHERE ';
+      return ret + sqlwhere.join('AND ');
+    }
+    return '';
+  };
+
+  //
+  whereNotSQL(params) {
+    return this.whereSQL(params, true);
+  };
+
+  // INSERT SQL
+  insertSQL() {
+    const { query, dialect } = this;
+    const { esc } = dialect;
+
+    // insert into
+    let sql = `INSERT INTO ${esc}${query.table}${esc}`;
+
+    // columns
+    const columns = [], values = [], params = [];
+    _.forOwn(query.values, function(value, key) {
+      columns.push(`${esc}${key}${esc}`);
+      values.push(dialect.param(this.i++));
+      params.push(value);
+    }.bind(this));
+
+    if (!columns.length && dialect.emptyInsert) {
+      sql += dialect.emptyInsert;
+    } else {
+      sql += '(' + columns.join(',') + ') ';
+      sql += 'VALUES(' + values.join(',') + ') ';
+    }
+
+    sql += dialect.returning;
+    return { sql, params };
+  };
+
+  // UPDATE SQL
+  updateSQL() {
+    const { query, dialect } = this;
+    const { esc } = dialect;
+
+    // update set
+    let sql = `UPDATE ${esc}${query.table}${esc} SET `;
+
+    // columns
+    const columns = [], params = [];
+    _.forOwn(query.values, (value, key) => {
+      columns.push(`${esc}${key}${esc} = ${dialect.param(this.i++)}`);
+      params.push(value);
+    });
+
+    sql += columns.join(', ') + ' ';
+
+    // where
+    sql += this.whereSQL(params);
+
+    // whereNot
+    sql += this.whereNotSQL(params);
+
+    const ret = {
+      sql: sql,
+      params: params
+    };
+    return ret;
+  };
+
+  // DELETE SQL
+  deleteSQL() {
+    const { query, dialect } = this;
+    const { esc }   = dialect;
+
+    // delete
+    let sql = `DELETE FROM ${esc}${query.table}${esc} `;
+
+    // columns
+    var params = [];
+
+    // where
+    sql += this.whereSQL(params);
+
+    // whereNot
+    sql += this.whereNotSQL(params);
+
+    var ret = {
+      sql: sql,
+      params: params
+    };
+    return ret;
+  };
+
+  // ORDER BY SQL
+  orderSQL() {
+    const { query } = this;
+
+    if (!query.order || !query.order.length) {
+      return '';
+    }
+
+    var sql = 'ORDER BY ' + query.order.join(', ') + ' ';
+
+    return sql;
+  };
+
+
+  // GROUP BY SQL
+  groupSQL() {
+    const { query } = this;
+    
+    if (_.isEmpty(query.group )) {
+      return '';
+    }
+
+    var sql = 'GROUP BY ' + query.group.join(', ') + ' ';
+    return sql;
+  };
+};

package/src/context.js

@@ -0,0 +1,12 @@
+// Dependency injection container for @igojs/db
+// These dependencies are injected by @igojs/server at initialization
+
+const context = {
+  config: null,
+  cache: null,
+  logger: null,
+  utils: null,
+  errorhandler: null,
+};
+
+module.exports = context;

package/src/dbs.js

@@ -0,0 +1,26 @@
+
+
+const _           = require('lodash');
+
+const Db          = require('./Db');
+const context     = require('./context');
+const migrations  = require('./migrations');
+
+// init databases connections
+module.exports.init = async () => {
+  const { config } = context;
+  for (const database of config.databases) {
+    const db = new Db(database);
+    await db.init();
+
+    if (config.databases.length === 1) {
+      db.config.migrations_dir = 'sql';
+    }
+
+    module.exports[database] = db;
+    await migrations.init(db);
+  }
+
+  // main is first database
+  module.exports.main = module.exports[config.databases[0]];
+};

package/src/drivers/mysql.js

@@ -0,0 +1,74 @@
+
+const _       = require('lodash');
+const mysql   = require('mysql2/promise');
+const OPTIONS = [
+  'host', 'port', 'user', 'password', 'database',
+  'charset', 'debug', 'connectionLimit'
+];
+
+
+// create pool
+module.exports.createPool = (dbconfig) => {
+  return mysql.createPool(_.pick(dbconfig, OPTIONS));
+};
+
+// get connection
+module.exports.getConnection = async (pool) => {
+  return await pool.getConnection();
+};
+
+// query
+module.exports.query = async (connection, sql, params) => {
+  return await connection.query(sql, params);
+};
+
+// release
+module.exports.release = (connection) => {
+  connection.release();
+};
+
+// begin transaction
+module.exports.beginTransaction = async (connection) => {
+  return await connection.beginTransaction();
+};
+
+// commit transaction
+module.exports.commit = async (connection) => {
+  return await connection.commit();
+};
+
+// rollback transaction
+module.exports.rollback = async (connection) => {
+  return await connection.rollback();
+};
+
+// dialect
+module.exports.dialect = {
+  createDb: db => `CREATE DATABASE \`${db}\`;`,
+  dropDb:   db => `DROP DATABASE IF EXISTS \`${db}\`;`,
+  createMigrationsTable: `CREATE TABLE IF NOT EXISTS \`__db_migrations\`(
+    \`id\` INTEGER NOT NULL AUTO_INCREMENT,
+    \`file\` VARCHAR(100),
+    \`success\` TINYINT(1),
+    \`err\` VARCHAR(255),
+    \`creation\` DATETIME,
+    PRIMARY KEY (\`id\`)
+   ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`,
+  listMigrations: 'SELECT * FROM `__db_migrations` ORDER BY `id` DESC',
+  findMigration:  'SELECT `id` from  `__db_migrations` WHERE `file`=? AND `success`=1',
+  insertMigration: 'INSERT INTO `__db_migrations` (file, success, err, creation) VALUES(?, ?, ?, ?)',
+  esc: '`',
+  param: () => '?',
+  limit: () => 'LIMIT ?, ? ',
+  returning: '',
+  insertId: result => result && result.insertId,
+  getRows: result => result[0],
+  emptyInsert: null,
+  in: 'IN',
+  notin: 'NOT IN',
+  getLock: lock => `SELECT GET_LOCK('${lock}', 0) AS 'lock'`,
+  gotLock: res => res && res[0] && res[0][0] && res[0][0].lock > 0,
+  releaseLock: lock => `SELECT RELEASE_LOCK('${lock}')`,
+
+  
+};

package/src/drivers/postgresql.js

@@ -0,0 +1,70 @@
+
+const { Pool }  = require('pg');
+
+
+// create pool
+module.exports.createPool = (dbconfig) => {
+  return new Pool(dbconfig);
+};
+
+// get connection
+module.exports.getConnection =  async (pool) => {
+  return await pool.connect();
+};
+
+// query
+module.exports.query = async (connection, sql, params) => {
+  return await connection.query(sql, params);
+};
+
+// release
+module.exports.release = (connection) => {
+  connection.release();
+};
+
+// begin transaction
+module.exports.beginTransaction = async (connection) => {
+  return await connection.query('BEGIN');
+};
+
+// commit transaction
+module.exports.commit = async (connection) => {
+  return await connection.query('COMMIT');
+};
+
+// rollback transaction
+module.exports.rollback = async (connection) => {
+  return await connection.query('ROLLBACK');
+};
+
+// dialect
+module.exports.dialect = {
+  createDb: db => `CREATE DATABASE "${db}";`,
+  dropDb:   db => `DROP DATABASE IF EXISTS "${db}";`,
+  createMigrationsTable: `CREATE TABLE IF NOT EXISTS "__db_migrations"(
+    "id" SERIAL,
+    "file" VARCHAR(100),
+    "success" BOOLEAN,
+    "err" VARCHAR(255),
+    "creation" TIMESTAMP,
+    PRIMARY KEY ("id")
+   );`,
+  listMigrations:   'SELECT * FROM "__db_migrations" ORDER BY "id" DESC',
+  findMigration:    'SELECT "id" from  "__db_migrations" WHERE "file"=$1 AND "success"=TRUE',
+  insertMigration:  'INSERT INTO "__db_migrations" (file, success, err, creation) VALUES($1, $2, $3, $4)',
+  esc: '"',
+  param: i => `$${i}`,
+  limit: (i, j) => `LIMIT $${j} OFFSET $${i} `,
+  returning: 'RETURNING "id"',
+  insertId: result => {
+    return result && result[0] && result[0].id;
+  },
+  getRows: result => result && result.rows,
+  emptyInsert: 'DEFAULT VALUES ',
+  in: '= ANY',
+  notin: '!= ALL',
+  getLock: lock => `SELECT pg_try_advisory_lock(hashtext('${lock}')::bigint);`,
+  gotLock: res => res && res.rows && res.rows[0] && res.rows[0].pg_try_advisory_lock,
+  releaseLock: lock => `SELECT pg_advisory_unlock(hashtext('${lock}')::bigint);`,
+
+};

package/src/migrations.js

@@ -0,0 +1,140 @@
+
+const _       = require('lodash');
+
+const fs      = require('fs/promises');
+const path    = require('path');
+
+const context = require('./context');
+
+//
+module.exports.init = async (db) => {
+  const { config } = context;
+  if (!config.auto_migrate) return;
+
+  try {
+    const { connection } = await db.getConnection();
+    const { dialect } = db.driver;
+    const lock        = db.config.database + '.__db_migrations';
+    const getLock     = dialect.getLock(lock);
+    const res         = await db.driver.query(connection, getLock, []);
+    if (!dialect.gotLock(res)) {
+      // could not get lock, skip migration
+      return db.driver.release(connection);
+    }
+
+    // got lock, migrate!
+    await module.exports.migrate(db);
+
+    // release lock
+    setTimeout(async () => {
+      const releaseLock = dialect.releaseLock(lock);
+      await db.driver.query(connection, releaseLock);
+      db.driver.release(connection);
+    }, 10000);
+
+  } catch (err) {
+    console.error(err);
+  }
+};
+
+
+//
+module.exports.initmigrations = async (db) => {
+  const sql = db.driver.dialect.createMigrationsTable;
+  await db.query(sql);
+};
+
+//
+module.exports.list = async (db) => {
+  await module.exports.initmigrations(db);
+  const sql = db.driver.dialect.listMigrations;
+  return await db.query(sql);
+};
+
+//
+module.exports.migrate = async (db, rootDir = '.') => {
+  const { config, logger } = context;
+
+  if (db.config.noMigrations) {
+    return;
+  }
+
+  const sqldir = `${rootDir}/${db.config.migrations_dir || 'sql'}`;
+  let querybuf  = '';
+
+  // create directory if it does not exist
+  await fs.mkdir(sqldir, { recursive: true });
+
+  // execute SQL line
+  const executeLine = async (line) => {
+    line = line.replace('\r', '').trim();
+    if (line.match('^--')) return;
+    if (line.match('\\;$')) {
+      querybuf += line;
+      if (config.mysql && config.mysql.debugsql) logger.info(querybuf);
+      await db.query(querybuf);
+      querybuf = '';
+    } else if (line.length > 0) {
+      querybuf += line;
+      return;
+    } else {
+      return;
+    }
+  };
+
+  // execute SQL migration file
+  const executeFile = async (file) => {
+    // Skip hidden files silently (like .gitkeep)
+    if (file.filename.startsWith('.')) {
+      return;
+    }
+
+    if (!file.filename.match('[0-9]{8}.*\\.sql$')) {
+      logger.warn('File %s does not match migration pattern, skipping', file.filename);
+      return;
+    }
+
+    // find if file has already been played
+    const result  = await db.query(db.driver.dialect.findMigration, [file.filename]);
+    if (result && result.length > 0) {
+      return 'alreadyplayed';
+    }
+
+    try {
+      const data = await fs.readFile(file.path);
+      const lines = data.toString().split('\n');
+      if (config.mysql && config.mysql.debugsql) {
+        logger.info('Executing ' + file.path + ': ' + lines.length + ' lines to process');
+      }
+      for (const line of lines) {
+        await executeLine(line);
+      }
+
+      // no error
+      await db.query(db.driver.dialect.insertMigration, [file.filename, true, null, new Date()]);
+      logger.info('✅ ' + file.filename);
+
+    } catch (err) {
+      await db.query(db.driver.dialect.insertMigration, [file.filename, false, String(err), new Date()]);
+      logger.info('❌ ' + file.filename);
+      logger.error('SQL error in file %s', file.path);
+      throw err;
+    }
+  }
+
+  let files = [];
+  const filenames = await fs.readdir(sqldir);
+  _.forEach(filenames, (filename) => {
+    files.push({ filename, path: path.join(sqldir, filename) });
+  })
+
+  files = _.sortBy(files, 'filename');
+  await module.exports.initmigrations(db);
+  try {
+    for (const file of files) {
+      await executeFile(file);
+    }
+  } catch (err) {
+    logger.error(err);
+  }
+};