knex 0.14.3 → 0.14.4

package/CHANGELOG.md

@@ -1,6 +1,28 @@
 
 # Master (Unreleased)
 
+# 0.14.4 - 19 Feb, 2018
+
+### Bug fixes:
+
+- containsUndefined only validate plain objects. Fixes #1898 (#2468)
+- Add warning when using .returning() in sqlite3. Fixes #1660 (#2471)
+- Throw an error if .update() results in an empty sql (#2472)
+- Removed unnecessary createTableIfNotExist and replaced with createTable (#2473)
+
+### New Features:
+
+- Allow calling lock procedures (such as forUpdate) outside of transaction. Fixes #2403. (#2475)
+- Added test and documentation for Event 'start' (#2488)
+
+### Test / internal changes
+
+- Added stress test, which uses TCP proxy to simulate flaky connection #2460
+- Removed old docker tests, new stress test setup (#2474)
+- Removed unused property __cid on the base client (#2481)
+- Changed rm to rimraf in 'npm run dev' (#2483)
+- Changed babel preset and use latest node as target when running dev (#2484)
+
 # 0.14.3 - 8 Feb, 2018
 
 ### Bug fixes:

package/lib/client.js

@@ -96,11 +96,6 @@ var debug = require('debug')('knex:client');
 var debugQuery = require('debug')('knex:query');
 var debugBindings = require('debug')('knex:bindings');
 
-var id = 0;
-function clientId() {
-  return 'client' + id++;
-}
-
 // The base client provides the general structure
 // for a dialect specific client object.
 function Client() {
@@ -119,7 +114,6 @@ function Client() {
   if (this.driverName && config.connection) {
     this.initializeDriver();
     if (!config.pool || config.pool && config.pool.max !== 0) {
-      this.__cid = clientId();
       this.initializePool(config);
     }
   }

package/lib/dialects/mysql/index.js

@@ -103,11 +103,15 @@ function Client_MySQL(config) {
 
     return new _bluebird2.default(function (resolver, rejecter) {
       var connection = _this.driver.createConnection(_this.connectionSettings);
+      connection.on('error', function (err) {
+        connection.__knex__disposed = err;
+      });
       connection.connect(function (err) {
-        if (err) return rejecter(err);
-        connection.on('error', function (err) {
-          connection.__knex__disposed = err;
-        });
+        if (err) {
+          // if connection is rejected, remove listener that was registered above...
+          connection.removeAllListeners();
+          return rejecter(err);
+        }
         resolver(connection);
       });
     });
@@ -117,16 +121,16 @@ function Client_MySQL(config) {
   // Used to explicitly close a connection, called internally by the pool
   // when a connection times out or the pool is shutdown.
   destroyRawConnection: function destroyRawConnection(connection) {
-    connection.removeAllListeners();
     return _bluebird2.default.fromCallback(connection.end.bind(connection)).catch(function (err) {
       connection.__knex__disposed = err;
+    }).finally(function () {
+      return connection.removeAllListeners();
     });
   },
   validateConnection: function validateConnection(connection) {
     if (connection.state === 'connected' || connection.state === 'authenticated') {
       return true;
     }
-
     return false;
   },
 

package/lib/dialects/mysql2/index.js

@@ -6,10 +6,6 @@ var _assign2 = require('lodash/assign');
 
 var _assign3 = _interopRequireDefault(_assign2);
 
-var _map2 = require('lodash/map');
-
-var _map3 = _interopRequireDefault(_map2);
-
 var _inherits = require('inherits');
 
 var _inherits2 = _interopRequireDefault(_inherits);
@@ -18,20 +14,10 @@ var _mysql = require('../mysql');
 
 var _mysql2 = _interopRequireDefault(_mysql);
 
-var _bluebird = require('bluebird');
-
-var _bluebird2 = _interopRequireDefault(_bluebird);
-
-var _helpers = require('../../helpers');
-
-var helpers = _interopRequireWildcard(_helpers);
-
 var _transaction = require('./transaction');
 
 var _transaction2 = _interopRequireDefault(_transaction);
 
-function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 // Always initialize with the "QueryBuilder" and "QueryCompiler"
@@ -60,52 +46,7 @@ function Client_MySQL2(config) {
     if (connection._fatalError) {
       return false;
     }
-
     return true;
-  },
-
-
-  // Get a raw connection, called by the `pool` whenever a new
-  // connection needs to be added to the pool.
-  acquireRawConnection: function acquireRawConnection() {
-    var connection = this.driver.createConnection(this.connectionSettings);
-    connection.on('error', function (err) {
-      connection.__knex__disposed = err;
-    });
-    return new _bluebird2.default(function (resolver, rejecter) {
-      connection.connect(function (err) {
-        if (err) {
-          return rejecter(err);
-        }
-        resolver(connection);
-      });
-    });
-  },
-  processResponse: function processResponse(obj, runner) {
-    var response = obj.response;
-    var method = obj.method;
-
-    var rows = response[0];
-    var fields = response[1];
-    if (obj.output) return obj.output.call(runner, rows, fields);
-    switch (method) {
-      case 'select':
-      case 'pluck':
-      case 'first':
-        {
-          var resp = helpers.skim(rows);
-          if (method === 'pluck') return (0, _map3.default)(resp, obj.pluck);
-          return method === 'first' ? resp[0] : resp;
-        }
-      case 'insert':
-        return [rows.insertId];
-      case 'del':
-      case 'update':
-      case 'counter':
-        return rows.affectedRows;
-      default:
-        return response;
-    }
   }
 });
 

package/lib/dialects/sqlite3/query/compiler.js

@@ -42,13 +42,22 @@ var _compiler = require('../../../query/compiler');
 
 var _compiler2 = _interopRequireDefault(_compiler);
 
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+var _helpers = require('../../../helpers');
 
-// SQLite3 Query Builder & Compiler
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 function QueryCompiler_SQLite3(client, builder) {
   _compiler2.default.call(this, client, builder);
+
+  var returning = this.single.returning;
+
+
+  if (returning) {
+    (0, _helpers.warn)('.returning() is not supported by sqlite3 and will not have any effect.');
+  }
 }
+// SQLite3 Query Builder & Compiler
+
 (0, _inherits2.default)(QueryCompiler_SQLite3, _compiler2.default);
 
 (0, _assign3.default)(QueryCompiler_SQLite3.prototype, {

package/lib/helpers.js

@@ -10,9 +10,9 @@ var _isArray2 = require('lodash/isArray');
 
 var _isArray3 = _interopRequireDefault(_isArray2);
 
-var _isObject2 = require('lodash/isObject');
+var _isPlainObject2 = require('lodash/isPlainObject');
 
-var _isObject3 = _interopRequireDefault(_isObject2);
+var _isPlainObject3 = _interopRequireDefault(_isPlainObject2);
 
 var _isUndefined2 = require('lodash/isUndefined');
 
@@ -110,7 +110,7 @@ function containsUndefined(mixed) {
       if (argContainsUndefined) break;
       argContainsUndefined = this.containsUndefined(mixed[i]);
     }
-  } else if ((0, _isObject3.default)(mixed)) {
+  } else if ((0, _isPlainObject3.default)(mixed)) {
     for (var key in mixed) {
       if (mixed.hasOwnProperty(key)) {
         if (argContainsUndefined) break;

package/lib/migrate/index.js

@@ -262,13 +262,10 @@ var Migrator = function () {
     });
   };
 
-  // Create the migration table, if it doesn't already exist.
-
-
   Migrator.prototype._createMigrationTable = function _createMigrationTable(tableName) {
     var trx = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : this.knex;
 
-    return trx.schema.createTableIfNotExists(tableName, function (t) {
+    return trx.schema.createTable(tableName, function (t) {
       t.increments();
       t.string('name');
       t.integer('batch');
@@ -279,7 +276,7 @@ var Migrator = function () {
   Migrator.prototype._createMigrationLockTable = function _createMigrationLockTable(tableName) {
     var trx = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : this.knex;
 
-    return trx.schema.createTableIfNotExists(tableName, function (t) {
+    return trx.schema.createTable(tableName, function (t) {
       t.integer('is_locked');
     });
   };

package/lib/query/compiler.js

@@ -468,11 +468,7 @@ var components = ['columns', 'join', 'where', 'union', 'group', 'having', 'order
   // Compiles the "locks".
   lock: function lock() {
     if (this.single.lock) {
-      if (!this.client.transacting) {
-        helpers.warn('You are attempting to perform a "lock" command outside of a transaction.');
-      } else {
-        return this[this.single.lock]();
-      }
+      return this[this.single.lock]();
     }
   },
 
@@ -644,6 +640,11 @@ var components = ['columns', 'join', 'where', 'union', 'group', 'having', 'order
     while (++i < columns.length) {
       vals.push(this.formatter.wrap(columns[i]) + ' = ' + this.formatter.parameter(data[columns[i]]));
     }
+
+    if ((0, _isEmpty3.default)(vals)) {
+      throw new Error(['Empty .update() call detected!', 'Update data does not contain any values to update.', 'This will result in a faulty query.'].join(' '));
+    }
+
     return vals;
   },
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "knex",
-  "version": "0.14.3",
+  "version": "0.14.4",
   "description": "A batteries-included SQL query & schema builder for Postgres, MySQL and SQLite3 and the Browser",
   "main": "knex.js",
   "dependencies": {
@@ -31,10 +31,9 @@
     "babel-plugin-add-module-exports": "^0.2.1",
     "babel-plugin-lodash": "3.3.2",
     "babel-plugin-transform-runtime": "^6.23.0",
-    "babel-preset-es2015": "^6.24.1",
+    "babel-preset-env": "^1.6.1",
     "chai": "^4.1.2",
     "coveralls": "~3.0.0",
-    "dockerode": "^2.5.3",
     "eslint": "4.16.0",
     "eslint-plugin-import": "^2.8.0",
     "estraverse": "^4.2.0",
@@ -45,7 +44,7 @@
     "mock-fs": "^4.4.2",
     "mssql": "^4.1.0",
     "mysql": "^2.15.0",
-    "mysql2": "^1.5.1",
+    "mysql2": "^1.5.2",
     "pg": "^7.4.1",
     "pg-query-stream": "^1.1.1",
     "rimraf": "^2.6.2",
@@ -55,7 +54,8 @@
     "sqlite3": "^3.1.13",
     "tap-spec": "^4.1.1",
     "tape": "^4.8.0",
-    "through": "^2.3.8"
+    "through": "^2.3.8",
+    "toxiproxy-node-client": "^2.0.6"
   },
   "buildDependencies": [
     "babel-cli",
@@ -71,15 +71,17 @@
     "debug_test": "node-debug _mocha -t 0 test/index.js",
     "babel": "rimraf ./lib && babel src --out-dir lib --copy-files",
     "coveralls": "cat ./test/coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js",
-    "dev": "rm -rf ./lib && babel -w src --out-dir lib --copy-files",
+    "dev": "rimraf ./lib && babel -w src --out-dir lib --copy-files",
     "lint": "eslint src/**",
     "plaintest": "mocha --check-leaks -b -R spec test/index.js && npm run tape",
     "prepublish": "npm run babel",
-    "docker_test": "bash ./scripts/docker-for-test.sh",
-    "pre_test": "npm run lint && npm run docker_test",
+    "pre_test": "npm run lint",
     "tape": "node test/tape/index.js | tap-spec",
     "debug_tape": "node-debug test/tape/index.js",
-    "test": "npm run pre_test && istanbul --config=test/.istanbul.yml cover node_modules/mocha/bin/_mocha -- --check-leaks -t 10000 -b -R spec test/index.js && npm run tape"
+    "test": "npm run pre_test && istanbul --config=test/.istanbul.yml cover node_modules/mocha/bin/_mocha -- --check-leaks -t 10000 -b -R spec test/index.js && npm run tape",
+    "stress:init": "docker-compose -f scripts/stress-test/docker-compose.yml up --no-start && docker-compose -f scripts/stress-test/docker-compose.yml start",
+    "stress:test": "node scripts/stress-test/knex-stress-test.js |  grep -A 3 -- '- STATS '",
+    "stress:destroy": "docker-compose -f scripts/stress-test/docker-compose.yml stop"
   },
   "bin": {
     "knex": "./bin/cli.js"

package/scripts/stress-test/README.txt

@@ -0,0 +1,18 @@
+# Test scripts to evaluate stability of drivers / pool etc.
+
+# To run this test you need to be in this directory + have node >= 8
+# and startup docker containers with proxy and sql servers
+
+docker-compose up --no-start
+docker-compose start
+
+# Select different test script to run:
+
+node mysql2-random-hanging-every-now-and-then.js 2> /dev/null | grep -B500 -A2 -- "- STATS" 
+node mysql2-sudden-exit-without-error
+node knex-stress-test.js | grep -A 3 -- "- STATS "
+node reconnect-test-mysql-based-drivers.js 2> /dev/null | grep -A 3 -- "- STATS "
+
+# Shut down docker instances when done:
+
+docker-compose down

package/scripts/stress-test/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3'
+
+services:
+  toxiproxy:
+    image: shopify/toxiproxy
+    ports:
+      - "8474:8474"
+      - "23306:23306"
+      - "25432:25432"
+      - "21521:21521"
+    links:
+      - "mysql"
+      - "postgresql"
+
+  mysql:
+    image: mysql:5.7
+    ports:
+      - "33306:3306"
+    environment:
+      - TZ=UTC
+      - MYSQL_ROOT_PASSWORD=mysqlrootpassword
+
+  postgresql:
+    image: mdillon/postgis
+    ports:
+      - "35432:5432"
+    environment:
+      - POSTGRES_PASSWORD=postgresrootpassword
+      - POSTGRES_USER=postgres
+
+  oracledbxe:
+    image: wnameless/oracle-xe-11g
+    ports:
+      - "31521:1521"
+    environment:
+      - ORACLE_ALLOW_REMOTE=true
+
+#  mssql:
+#    image: microsoft/mssql-server-linux
+#    ports:
+#      - "31433:1433"
+#    environment:
+#      - ACCEPT_EULA=Y
+#      - SA_PASSWORD=mssqlpassword

package/scripts/stress-test/knex-stress-test.js

@@ -0,0 +1,162 @@
+const Knex = require('../../lib');
+
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+// init instances
+const pg = Knex({
+  client: 'pg',
+  connection: 'postgres://postgres:postgresrootpassword@localhost:25432/postgres',
+  pool: { max: 50 }
+});
+
+const mysql2 = Knex({
+  client: 'mysql2',
+  connection: 'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
+  pool: { max: 50 }
+});
+
+const mysql = Knex({
+  client: 'mysql',
+  connection: 'mysql://root:mysqlrootpassword@localhost:23306/?charset=utf8&connectTimeout=500',
+  pool: { max: 50 }
+});
+
+/* TODO: figure out how to nicely install oracledb node driver on osx
+const mysql = Knex({
+  client: 'oracledb',
+  connection: {
+    user          : "travis",
+    password      : "travis",
+    connectString : "localhost/XE",
+    // https://github.com/oracle/node-oracledb/issues/525
+    stmtCacheSize : 0
+  },
+  pool: { max: 50 }
+});
+*/
+
+const counters = {};
+
+function setQueryCounters(instance, name) {
+  const counts = counters[name] = {queries: 0, results: 0, errors: 0};
+  instance.on('query', () => counts.queries += 1);
+  instance.on('query-response', () => counts.results += 1);
+  instance.on('query-error', () => counts.errors += 1);    
+}
+
+setQueryCounters(pg, 'pg');
+setQueryCounters(mysql, 'mysql');
+setQueryCounters(mysql2, 'mysql2');
+
+const _ = require('lodash');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: (counters[key].queries - lastCounters[key].queries) / 2,
+      results: (counters[key].results - lastCounters[key].results) / 2,
+      errors: (counters[key].errors - lastCounters[key].errors) / 2,
+    }
+  }
+  console.log('------------------------ STATS PER SECOND ------------------------');
+  console.dir(reqsPerSec, {colors: true});
+  console.log('------------------------------- EOS ------------------------------');
+  lastCounters = _.cloneDeep(counters);
+}, 2000);  
+
+
+async function killConnectionsPg() {
+  return pg.raw(`SELECT pg_terminate_backend(pg_stat_activity.pid) 
+    FROM pg_stat_activity
+    WHERE pg_stat_activity.datname = 'postgres' 
+      AND pid <> pg_backend_pid()`);
+} 
+
+async function killConnectionsMyslq(client) {
+  const [rows, colDefs] = await client.raw(`SHOW FULL PROCESSLIST`);
+  await Promise.all(rows.map(row => client.raw(`KILL ${row.Id}`)));
+} 
+
+async function main() {
+  async function loopQueries(prefix, query) {
+    const queries = () => [
+      ...Array(50).fill(query),
+    ];
+
+    while(true) {
+      try {
+        await Promise.all(queries());
+      } catch (err) {
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  async function recreateProxy(serviceName, listenPort, proxyToPort) {
+    try {
+      await rp.delete({
+        url: `${toxicli.host}/proxies/${serviceName}`
+      });  
+    } catch(err) {}
+
+    const proxy = await toxicli.createProxy({
+      name: serviceName,
+      listen: `0.0.0.0:${listenPort}`,
+      upstream: `${serviceName}:${proxyToPort}`
+    });
+
+    // add some latency
+    await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+      type: 'latency',
+      attributes: {latency: 1, jitter: 1}
+    }));
+
+    // cause connections to be closed every 500 bytes
+    await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+      type: 'limit_data',
+      attributes: {bytes: 5000}
+    }));
+  }
+
+  // create TCP proxies for simulating bad connections etc.
+  async function recreateProxies() {
+    await recreateProxy('postgresql', 25432, 5432);
+    await recreateProxy('mysql', 23306, 3306);
+    await recreateProxy('oracledbxe', 21521, 1521);
+  }
+
+  await recreateProxies();
+  
+  loopQueries('PSQL:', pg.raw('select 1'));
+  loopQueries('PSQL TO:', pg.raw('select 1').timeout(20));
+
+  loopQueries('MYSQL:', mysql.raw('select 1'));
+  loopQueries('MYSQL TO:', mysql.raw('select 1').timeout(20));
+
+  loopQueries('MYSQL2:', mysql2.raw('select 1'));
+  loopQueries('MYSQL2 TO:', mysql2.raw('select 1').timeout(20));
+
+
+  while(true) {
+    await Bluebird.delay(20); // kill everything every quite often from server side
+    try {
+      await Promise.all([
+        killConnectionsPg(),
+        killConnectionsMyslq(mysql),
+        killConnectionsMyslq(mysql2),
+      ]);
+    } catch (err) {
+      console.log('KILLER ERROR:', err);
+    }
+  }
+}
+
+process.on('exit', () => console.log('- STATS PRINT NEAR END LOGS ')); // marker for grep...
+main();

package/scripts/stress-test/mysql2-random-hanging-every-now-and-then.js

@@ -0,0 +1,132 @@
+/**
+ * Test case for figuring out robust way to recognize if connection is dead
+ * for mysql based drivers.
+ */
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+const _ = require('lodash');
+
+async function stdMysqlQuery(con, sql) {  
+  return new Promise((resolve, reject) => {
+    try {
+      con.query({
+        sql,
+        timeout: 500
+      }, function (error, results, fields) {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(results);
+        }
+      });    
+    } catch (err) {
+      reject(err); // double sure...
+    }
+  });
+}
+
+const mysql2 = require('mysql2');
+let mysql2Con = {_fatalError: 'initmefirst'};
+async function mysql2Query(sql) {
+  // recreate connection on fatal error
+  if (mysql2Con._fatalError) {
+    console.log('========== Reconnecting mysql2');
+    mysql2Con = mysql2.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500,
+      debug: true
+    });
+    mysql2Con.on('error', err => {
+      console.log('- STATS Mysql2 connection died:', err);
+    });
+  }
+  console.log('================ MYSQL2 Running query ======');
+  const res = await stdMysqlQuery(mysql2Con, sql);
+  console.log('====================== done ================');
+  return res;
+}
+
+const counters = {};
+function setMysqlQueryCounters(name) {
+  const counts = counters[name] = {queries: 0, results: 0, errors: 0};
+}
+setMysqlQueryCounters('mysql2');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: (counters[key].queries - lastCounters[key].queries),
+      results: (counters[key].results - lastCounters[key].results),
+      errors: (counters[key].errors - lastCounters[key].errors),
+    }
+  }
+  console.log('------------------------ STATS PER SECOND ------------------------');
+  console.dir(reqsPerSec, {colors: true});
+  console.log('------------------------------- EOS ------------------------------');
+  lastCounters = _.cloneDeep(counters);
+}, 1000);  
+
+async function recreateProxy(serviceName, listenPort, proxyToPort) {
+  try {
+    await rp.delete({
+      url: `${toxicli.host}/proxies/${serviceName}`
+    });  
+  } catch(err) {}
+
+  const proxy = await toxicli.createProxy({
+    name: serviceName,
+    listen: `0.0.0.0:${listenPort}`,
+    upstream: `${serviceName}:${proxyToPort}`
+  });
+
+  // add some latency
+  await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+    type: 'latency',
+    attributes: {latency: 1, jitter: 1}
+  }));
+
+  // cause connections to be closed every some transferred bytes
+  await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+    type: 'limit_data',
+    attributes: {bytes: 1000}
+  }));
+}
+
+async function main() {
+  await recreateProxy('mysql', 23306, 3306);
+  setInterval(() => recreateProxy('mysql', 23306, 3306), 2000);
+
+  async function loopQueries(prefix, query) {
+    const counts = counters[prefix];
+
+    while(true) {
+      try {
+        counts.queries += 1;
+        // without this delay we might endup to busy failure loop
+        await Bluebird.delay(0);
+        await query();
+        counts.results += 1;
+      } catch (err) {
+        counts.errors += 1;
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  loopQueries('mysql2', () => mysql2Query('select 1'));
+
+  // wait forever
+  while(true) {
+    await Bluebird.delay(1000);
+  }
+}
+
+main().then(() => console.log('DONE')).catch(err => console.log(err));

package/scripts/stress-test/mysql2-sudden-exit-without-error.js

@@ -0,0 +1,98 @@
+/**
+ * Test case when mysql2 driver strangely exits when one tries to send query
+ * to dead connection.
+ */
+
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+// drops old toxicproxy and creates new
+async function recreateProxy(serviceName, listenPort, proxyToPort) {
+  try {
+    await rp.delete({
+      url: `${toxicli.host}/proxies/${serviceName}`
+    });  
+  } catch(err) {
+    // there was no proxy by that name... its ok
+  }
+
+  const proxy = await toxicli.createProxy({
+    name: serviceName,
+    listen: `0.0.0.0:${listenPort}`,
+    upstream: `${serviceName}:${proxyToPort}`
+  });
+}
+
+async function insanelyParanoidQuery(con) {
+  console.log('sending query');
+  const res = await new Promise((resolve,reject) => {
+    try {
+      con.query('select 1', [],  function(err, rows, fields) {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(rows);
+        }
+      });
+    } catch(err) {
+      console.log('Huh synchronous exception?! (shouldnt be possible)');
+      reject(err);
+    }
+  });
+  console.log(res);
+}
+
+
+async function main() {
+  // create proxy from localhost:23306 -> mysqldocker:3306
+  await recreateProxy('mysql', 23306, 3306);
+
+  // ------------- setup mysql2 db driver connection
+  const mysql2 = require('mysql2'); // with mysql this works...
+  const mysql2Con = await mysql2.createConnection({
+    host: 'localhost',
+    user: 'root',
+    password: 'mysqlrootpassword',
+    port: 23306
+  });
+
+  mysql2Con.on('error', function(err) {
+    console.log("I'm dead", err);
+  })
+  
+  console.log('Going to cut connections');
+
+  // cut connection during recreate
+  await recreateProxy('mysql', 23306, 3306);
+
+  console.log('Proxy recreated... start waiting');
+
+  // wait forever
+  while(true) {
+    await Bluebird.delay(1000);
+    try {
+      await insanelyParanoidQuery(mysql2Con);
+    } catch (err) {
+      console.log('query failed:', err);
+    }
+    await recreateProxy('mysql', 23306, 3306);
+  }
+}
+
+main()
+  .then(() => console.log('Process stopped normally'))
+  .catch(err => {
+    console.log('Process stopped to failure', err);
+  })
+
+console.log('Waiting for eventloop to stop...');
+
+process.on('uncaughtException', function (err) {
+  console.log('uncaughtException', err);
+});
+
+process.on('exit', function () {
+  console.log('Did someone call process.exit() or wat? exitCode:', process.exitCode);
+});

package/scripts/stress-test/reconnect-test-mysql-based-drivers.js

@@ -0,0 +1,174 @@
+/**
+ * Test case for figuring out robust way to recognize if connection is dead
+ * for mysql based drivers.
+ */
+const Bluebird = require('bluebird');
+const toxiproxy = require('toxiproxy-node-client');
+const toxicli = new toxiproxy.Toxiproxy('http://localhost:8474');
+const rp = require('request-promise-native');
+
+async function stdMysqlQuery(con, sql) {  
+  return new Promise((resolve, reject) => {
+    try {
+      con.query({
+        sql,
+        timeout: 4000
+      }, function (error, results, fields) {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(results);
+        }
+      });    
+    } catch (err) {
+      reject(err); // double sure...
+    }
+  });
+}
+
+// ALL THE DRIVERS HAS DIFFERENT BAG OF TRICKS TO RECOVER AND
+// RECOGNIZE WHEN CONNECTION HAS BEEN CLOSED
+
+// ------------- setup mysql db driver connection
+const mysql = require('mysql');
+let mysqlCon = {state: 'disconnected'};
+async function mysqlQuery(sql) {
+  // best way to check if connection is still alive
+  if (mysqlCon.state === 'disconnected') {
+    console.log('reconnecting mysql');
+    mysqlCon = mysql.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500
+    });
+    // not always triggered, if this happens during query
+    mysqlCon.on('error', err => {
+      console.log('- STATS Mysql connection died:', err);
+    });
+  }
+  return stdMysqlQuery(mysqlCon, sql);
+}
+
+// ------------- setup mysql2 db driver connection
+const mysql2 = require('mysql2');
+let mysql2Con = {_fatalError: 'initmefirst'};
+async function mysql2Query(sql) {
+  if (mysql2Con._fatalError) {
+    console.log('reconnecting mysql2');
+    mysql2Con = mysql2.createConnection({
+      host: 'localhost',
+      user: 'root',
+      password: 'mysqlrootpassword',
+      port: 23306,
+      connectTimeout: 500
+    });
+    mysql2Con.on('error', err => {
+      console.log('- STATS Mysql2 connection died:', err);
+    });
+  }
+  console.log('================ MYSQL2 Running query....');
+  const res = await stdMysqlQuery(mysql2Con, sql);
+  console.log('=========== done');
+  return res;
+}
+
+const counters = {};
+function setMysqlQueryCounters(name) {
+  const counts = counters[name] = {queries: 0, results: 0, errors: 0};
+}
+setMysqlQueryCounters('mysql');
+setMysqlQueryCounters('mysql2');
+
+const _ = require('lodash');
+
+// start printing out counters
+let lastCounters = _.cloneDeep(counters);
+
+setInterval(() => {
+  const reqsPerSec = {};
+  for (let key of Object.keys(counters)) {
+    reqsPerSec[key] = {
+      queries: (counters[key].queries - lastCounters[key].queries),
+      results: (counters[key].results - lastCounters[key].results),
+      errors: (counters[key].errors - lastCounters[key].errors),
+    }
+  }
+  console.log('------------------------ STATS PER SECOND ------------------------');
+  console.dir(reqsPerSec, {colors: true});
+  console.log('------------------------------- EOS ------------------------------');
+  lastCounters = _.cloneDeep(counters);
+
+  // if hang
+  ///if (reqsPerSec.mysql2.queries === 0) process.exit(0);
+}, 1000);  
+
+
+async function main() {
+
+  async function recreateProxy(serviceName, listenPort, proxyToPort) {
+    try {
+      await rp.delete({
+        url: `${toxicli.host}/proxies/${serviceName}`
+      });  
+    } catch(err) {}
+
+    const proxy = await toxicli.createProxy({
+      name: serviceName,
+      listen: `0.0.0.0:${listenPort}`,
+      upstream: `${serviceName}:${proxyToPort}`
+    });
+
+    // add some latency
+    await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+      type: 'latency',
+      attributes: {latency: 1, jitter: 1}
+    }));
+
+    // cause connections to be closed every some transferred bytes
+    await proxy.addToxic(new toxiproxy.Toxic(proxy, {
+      type: 'limit_data',
+      attributes: {bytes: 1000}
+    }));
+  }
+
+
+  // create TCP proxies for simulating bad connections etc.
+  async function recreateProxies() {
+    console.log('----- Recreating proxies -> cutting connections completely');
+    await recreateProxy('postgresql', 25432, 5432);
+    await recreateProxy('mysql', 23306, 3306);
+    await recreateProxy('oracledbxe', 21521, 1521);
+  }
+  setInterval(() => recreateProxies(), 2000);
+
+  async function loopQueries(prefix, query) {
+    const counts = counters[prefix];
+
+    while(true) {
+      try {
+        counts.queries += 1;
+        // without this delay we endup to busy failure loop
+        await Bluebird.delay(0);
+        await query();
+        counts.results += 1;
+      } catch (err) {
+        counts.errors += 1;
+        console.log(prefix, err);
+      }
+    }
+  }
+
+  await recreateProxies();
+
+  loopQueries('mysql', () => mysqlQuery('select 1'));
+  loopQueries('mysql2', () => mysql2Query('select 1'));
+
+  // wait forever
+  while(true) {
+    await Bluebird.delay(1000);
+  }
+}
+
+main().then(() => console.log('DONE')).catch(err => console.log(err));

package/src/client.js

@@ -27,11 +27,6 @@ const debug = require('debug')('knex:client')
 const debugQuery = require('debug')('knex:query')
 const debugBindings = require('debug')('knex:bindings')
 
-let id = 0
-function clientId() {
-  return `client${id++}`
-}
-
 // The base client provides the general structure
 // for a dialect specific client object.
 function Client(config = {}) {
@@ -48,7 +43,6 @@ function Client(config = {}) {
   if (this.driverName && config.connection) {
     this.initializeDriver()
     if (!config.pool || (config.pool && config.pool.max !== 0)) {
-      this.__cid = clientId()
       this.initializePool(config)
     }
   }

package/src/dialects/mysql/index.js

@@ -64,33 +64,36 @@ assign(Client_MySQL.prototype, {
   // connection needs to be added to the pool.
   acquireRawConnection() {
     return new Promise((resolver, rejecter) => {
-      const connection = this.driver.createConnection(this.connectionSettings)
+      const connection = this.driver.createConnection(this.connectionSettings);
+      connection.on('error', err => {
+        connection.__knex__disposed = err;
+      });
       connection.connect((err) => {
-        if (err) return rejecter(err)
-        connection.on('error', err => {
-          connection.__knex__disposed = err
-        })
-        resolver(connection)
-      })
+        if (err) {
+          // if connection is rejected, remove listener that was registered above...
+          connection.removeAllListeners();
+          return rejecter(err);
+        }
+        resolver(connection);
+      });
     })
   },
 
   // Used to explicitly close a connection, called internally by the pool
   // when a connection times out or the pool is shutdown.
   destroyRawConnection(connection) {
-    connection.removeAllListeners()
     return Promise
       .fromCallback(connection.end.bind(connection))
       .catch(err => {
         connection.__knex__disposed = err
       })
+      .finally(() => connection.removeAllListeners());
   },
 
   validateConnection(connection) {
     if (connection.state === 'connected' || connection.state === 'authenticated') {
       return true
     }
-
     return false
   },
 

package/src/dialects/mysql2/index.js

@@ -3,9 +3,7 @@
 // -------
 import inherits from 'inherits';
 import Client_MySQL from '../mysql';
-import Promise from 'bluebird';
-import * as helpers from '../../helpers';
-import { map, assign } from 'lodash'
+import { assign } from 'lodash'
 import Transaction from './transaction';
 
 // Always initialize with the "QueryBuilder" and "QueryCompiler"
@@ -33,52 +31,8 @@ assign(Client_MySQL2.prototype, {
     if (connection._fatalError) {
       return false
     }
-
     return true
-  },
-
-  // Get a raw connection, called by the `pool` whenever a new
-  // connection needs to be added to the pool.
-  acquireRawConnection() {
-    const connection = this.driver.createConnection(this.connectionSettings)
-    connection.on('error', err => {
-      connection.__knex__disposed = err
-    })
-    return new Promise((resolver, rejecter) => {
-      connection.connect((err) => {
-        if (err) {
-          return rejecter(err)
-        }
-        resolver(connection)
-      })
-    })
-  },
-
-  processResponse(obj, runner) {
-    const { response } = obj
-    const { method } = obj
-    const rows = response[0]
-    const fields = response[1]
-    if (obj.output) return obj.output.call(runner, rows, fields)
-    switch (method) {
-      case 'select':
-      case 'pluck':
-      case 'first': {
-        const resp = helpers.skim(rows)
-        if (method === 'pluck') return map(resp, obj.pluck)
-        return method === 'first' ? resp[0] : resp
-      }
-      case 'insert':
-        return [rows.insertId]
-      case 'del':
-      case 'update':
-      case 'counter':
-        return rows.affectedRows
-      default:
-        return response
-    }
   }
-
 })
 
 export default Client_MySQL2;

package/src/dialects/sqlite3/query/compiler.js

@@ -4,9 +4,16 @@
 import inherits from 'inherits';
 import QueryCompiler from '../../../query/compiler';
 import { assign, each, isEmpty, isString, noop, reduce, identity } from 'lodash'
+import { warn } from '../../../helpers';
 
 function QueryCompiler_SQLite3(client, builder) {
   QueryCompiler.call(this, client, builder)
+
+  const {returning} = this.single;
+
+  if(returning) {
+    warn('.returning() is not supported by sqlite3 and will not have any effect.');
+  }
 }
 inherits(QueryCompiler_SQLite3, QueryCompiler)
 

package/src/helpers.js

@@ -1,6 +1,15 @@
 /* eslint no-console:0 */
 
-import { map, pick, keys, isFunction, isUndefined, isObject, isArray, isTypedArray } from 'lodash'
+import {
+  map,
+  pick,
+  keys,
+  isFunction,
+  isUndefined,
+  isPlainObject,
+  isArray,
+  isTypedArray
+} from 'lodash'
 import chalk from 'chalk';
 
 // Pick off the attributes from only the current layer of the object.
@@ -60,7 +69,7 @@ export function containsUndefined(mixed) {
       if(argContainsUndefined) break;
       argContainsUndefined = this.containsUndefined(mixed[i]);
     }
-  } else if(isObject(mixed)) {
+  } else if(isPlainObject(mixed)) {
     for(const key in mixed) {
       if (mixed.hasOwnProperty(key)) {
         if(argContainsUndefined) break;

package/src/migrate/index.js

@@ -152,9 +152,8 @@ export default class Migrator {
       .then(data => !data.length && trx.into(lockTable).insert({ is_locked: 0 }));
   }
 
-  // Create the migration table, if it doesn't already exist.
   _createMigrationTable(tableName, trx = this.knex) {
-    return trx.schema.createTableIfNotExists(tableName, function(t) {
+    return trx.schema.createTable(tableName, function(t) {
       t.increments();
       t.string('name');
       t.integer('batch');
@@ -163,7 +162,7 @@ export default class Migrator {
   }
 
   _createMigrationLockTable(tableName, trx = this.knex) {
-    return trx.schema.createTableIfNotExists(tableName, function(t) {
+    return trx.schema.createTable(tableName, function(t) {
       t.integer('is_locked');
     });
   }

package/src/query/compiler.js

@@ -425,11 +425,7 @@ assign(QueryCompiler.prototype, {
   // Compiles the "locks".
   lock() {
     if (this.single.lock) {
-      if (!this.client.transacting) {
-        helpers.warn('You are attempting to perform a "lock" command outside of a transaction.')
-      } else {
-        return this[this.single.lock]()
-      }
+      return this[this.single.lock]()
     }
   },
 
@@ -618,6 +614,15 @@ assign(QueryCompiler.prototype, {
         this.formatter.parameter(data[columns[i]])
       );
     }
+
+    if(isEmpty(vals)) {
+      throw new Error([
+        'Empty .update() call detected!',
+        'Update data does not contain any values to update.',
+        'This will result in a faulty query.',
+      ].join(' '));
+    }
+
     return vals;
   },
 

package/scripts/stress-test.js

@@ -1,93 +0,0 @@
-const Bluebird = require('bluebird');
-
-// TODO: setup all DBs with docker
-// TODO: setup proxy for closing connections
-// TODO: use all dialects
-
-const pg = require('../lib')({
-  client: 'pg',
-  connection: 'postgres:///knex_test',
-  pool: { max: 90 }
-});
-
-const mysql = require('../lib')({
-  dialect: 'mysql2',
-  connection: {
-    database: "knex_test",
-    user: "root",
-    charset: 'utf8'
-  },
-  pool: { max: 90 }
-});
-
-async function main() {
-  const pgQueries = [
-    ...Array(40).fill(pg('accounts').timeout(100)), // these might eat memory
-    ...Array(40).fill(pg('accounts'))               // these might hang indefinitely
-  ];
-
-  function killConnectionsPg() {
-    return pg.raw(`SELECT pg_terminate_backend(pg_stat_activity.pid) 
-      FROM pg_stat_activity
-      WHERE pg_stat_activity.datname = 'knex_test' 
-        AND pid <> pg_backend_pid()`);
-  } 
-
-  async function killConnectionsMyslq() {
-    const [rows, colDefs] = await mysql.raw(`SHOW FULL PROCESSLIST`);
-    await Promise.all(rows.map(row => mysql.raw(`KILL ${row.Id}`)));
-  } 
-
-  const mysqlQueries = [
-    ...Array(40).fill(mysql('accounts').timeout(100)),
-    ...Array(40).fill(mysql('accounts'))
-  ];
-
-  const counters = {
-    pgQueries: 0,
-    pgResults: 0,
-    pgErrors: 0,
-    mysqlQueries: 0,
-    mysqlResults: 0,
-    mysqlErrors: 0
-  };
-
-  let lastCounters = {...counters};
-
-  pg.on('query', () => counters.pgQueries += 1);
-  pg.on('query-response', () => counters.pgResults += 1);
-  pg.on('query-error', () => counters.pgErrors += 1);
-  mysql.on('query', () => counters.mysqlQueries += 1);
-  mysql.on('query-response', () => counters.mysqlResults += 1);
-  mysql.on('query-error', () => counters.mysqlErrors += 1);
-
-  setInterval(() => {
-    console.dir({
-      markerPerSecond: '-------------------------------------------------------------------------------------------',
-      pgQueriesPerSecond: (counters.pgQueries - lastCounters.pgQueries)/2,
-      pgResultsPerSecond: (counters.pgResults - lastCounters.pgResults)/2,
-      pgErrorsPerSecond: (counters.pgErrors - lastCounters.pgErrors)/2,
-      mysqlQueriesPerSecond: (counters.mysqlQueries - lastCounters.mysqlQueries)/2,
-      mysqlResultsPerSecond: (counters.mysqlResults - lastCounters.mysqlResults)/2,
-      mysqlErrorsPerSecond: (counters.mysqlErrors - lastCounters.mysqlErrors)/2,
-    }, {colors: true});
-
-    lastCounters = {...counters};
-  }, 2000);
-
-  while(true) {
-    try {
-      await Promise.all([
-        ...pgQueries,
-        killConnectionsPg(),
-//        ...mysqlQueries,
-//        killConnectionsMyslq()
-      ]);
-    } catch (err) {
-      console.log(err);
-      // pass on error and try again...
-    }
-  }
-}
-
-main();