knex 0.14.2 → 0.14.6

package/CHANGELOG.md

@@ -1,6 +1,83 @@
 
 # Master (Unreleased)
 
+# 0.14.6 - 12 Apr, 2018
+
+### Bug fixes:
+
+- Restored functionality of query event #2566 (#2549)
+
+# 0.14.5 - 8 Apr, 2018
+
+### Bug fixes:
+
+- Fix wrapping returning column on oracledb #2554
+
+### New Features:
+
+- Support passing DB schema name for migrations #2499 #2559
+- add clearOrder method #2360 #2553
+- Added knexTxId to query events and debug calls #2476
+- Support multi-column `whereIn` with query #1390
+- Added error if chaining update/insert/etc with first() #2506
+- Checks for an empty, undefined or null object on transacting #2494
+- countDistinct with multiple columns #2449
+
+### Test / internal changes
+
+- Added npm run test:oracledb command that runs oracledb tests in docker #2491
+- Runnin mssql tests in docker #2496
+- Update dependencies #2561
+
+# 0.14.4 - 19 Feb, 2018
+
+### Bug fixes:
+
+- containsUndefined only validate plain objects. Fixes #1898 (#2468)
+- Add warning when using .returning() in sqlite3. Fixes #1660 (#2471)
+- Throw an error if .update() results in an empty sql (#2472)
+- Removed unnecessary createTableIfNotExist and replaced with createTable (#2473)
+
+### New Features:
+
+- Allow calling lock procedures (such as forUpdate) outside of transaction. Fixes #2403. (#2475)
+- Added test and documentation for Event 'start' (#2488)
+
+### Test / internal changes
+
+- Added stress test, which uses TCP proxy to simulate flaky connection #2460
+- Removed old docker tests, new stress test setup (#2474)
+- Removed unused property __cid on the base client (#2481)
+- Changed rm to rimraf in 'npm run dev' (#2483)
+- Changed babel preset and use latest node as target when running dev (#2484)
+
+# 0.14.3 - 8 Feb, 2018
+
+### Bug fixes:
+
+- Use tarn as pool instead of generic-pool which has been given various problems #2450
+- Fixed mysql issue where add columns failed if using both after and collate #2432
+- CLI sets exit-code 1 if the command supplied was not parseable #2358
+- Set toNative() to be not enumerable #2388
+- Use wrapIdentifier in columnInfo. fixes #2402 #2405
+- Fixed a bug when using .returning (OUTPUT) in an update query with joins in MSSQL #2399
+- Better error message when running migrations fail before even starting run migrations #2373
+- Read oracle's UV_THREADPOOL_SIZE env variable correctly #2372
+- Added decimal variable precision / scale support #2353
+
+### New Features:
+
+- Added queryContext to schema and query builders #2314
+- Added redshift dialect #2233
+- Added warning when one uses .createTableIfNotExist and deprecated it from docs #2458
+
+### Test / internal changes
+
+- Update dependencies and fix ESLint warnings accordingly #2433
+- Disable oracledb tests from non LTS nodes #2407
+- Update dependencies #2422
+
+
 # 0.14.2 - 24 Nov, 2017
 
 ### Bug fixes:

package/CONTRIBUTING.md

@@ -22,6 +22,82 @@ Documentation is no longer maintained in knex master repository. All the documen
 
 Documentation pull requests should not be merged before knex version which has the new documented feature is released.
 
+## I would like to add support for new dialect to knex, is it possible?
+
+Currently there are already way too many dialects supported in `knex` and instead of adding new dialect to central codebase, all the dialects should be moved to separate npm packages out from `knex` core library with their respective maintainers and test suites. 
+
+So if you like to write your own dialect, you can just inherit own dialect from knex base classes and use it by passing dilaect to knex in knex configuration (https://runkit.com/embed/90b3cpyr4jh2):
+
+```js
+// simple dialect overriding sqlite3 dialect to use sqlite3-offline driver
+require('sqlite3-offline');
+const Knex = require("knex");
+
+const Dialect = require(`knex/lib/dialects/sqlite3/index.js`);
+Dialect.prototype._driver = () => require('sqlite3-offline');
+
+const knex = Knex({
+  client: Dialect,
+  connection: ':memory:'
+});
+
+console.log(
+  knex.select(knex.raw(1)).toSQL()
+);
+
+await knex.schema.createTable('fooobar', (t) => {
+  t.bigincrements('id');
+  t.string('data');
+});
+await knex('fooobar').insert({ data: 'nomnom' });
+
+console.log("Gimme all the data:", await knex('fooobar'));
+```
+
+## What is minimal code to reproduce bug and why I have to provide that when I can just tell whats the problem is
+
+Writing minimal reproduction code for the problem is timeconsuming and some times it also really hard when for 
+example when the original code where the bug happens is written using express or mocha. So why it is necessary 
+for me to commit so much time to it when the problem is in `knex`? Contributors should be grateful that I reported
+the bug I found. 
+
+The point of runnable code to reproduce the problem is to easily verify that there really is a problem and that the one 
+who did the report did nothing wrong (surprisingly often problem is in the user code). So instead of just description 
+what to do the complete code encourages devs to actually test out that problem exists and start solving it and it 
+saves lots of time.
+
+tl;dr list:
+
+1. Actually in most of the cases developer already figures out what was the problem when writing the minimal test case
+or if there was problem how stuff was initilized or how async code was written it is easy to point out the problem.
+
+2. It motivates developer to actually try out if the bug really exist by not having to figure out from incomplete example
+environment in which and how bug actually manifests.
+
+3. There are curently very few people fixing knex issues and if one has to put easily 15-30 minutes time to issue just 
+to see that I cannot reproduce this issue it just wastes development hours that were available for improving knex.
+
+
+Test case should initialize needed tables, insert needed data and fail...
+
+```
+const knex = require('knex')({
+  client: 'pg',
+  connection: 'postgres:///knex_test'
+});
+
+async function main() {
+  await knex.schema.createTable(...);
+  await knex('table').insert({foo: 'bar}');
+  await knex.destroy();
+}
+
+main(); 
+```
+
+Usually issues without reproduction code available are just closed and if the same issue is reported multiple
+times maybe someone looks into it.
+
 ## Integration Tests
 
 ### The Easy Way
@@ -40,7 +116,7 @@ You can optionally specify which dialects to test using the `DB` environment var
 * postgres
 * sqlite3
 * maria
-* oracle
+* oracledb
 * mssql
 
 ```bash
@@ -69,6 +145,45 @@ Once this is done, check it works by attempting to login:
 psql -h localhost -U postgres -d knex_test
 ```
 
+### Running OracleDB tests in docker
+
+Since node-oracledb driver is so hard to install on every platform, oracle tests
+are actually ran inside docker container. Container has Oracle XE g11,
+node 8 and node-oracledb driver installed and copies local knex directory in 
+to be able to run the tests.
+
+```
+npm run oracledb:test
+```
+
+You can also manually start shell in the docker image and run build commands manually:
+```
+docker run -i -t knex-test-oracledb /bin/bash
+
+root@34f1f1cd20cf:/#
+
+/usr/sbin/startup.sh
+cd knex
+npm install
+npm install oracledb
+npm test
+```
+
+### Runnin MSSQL SQL Server tests
+
+SQL Server needs to be started as docker container before running tests
+
+```
+# start mssql docker container
+npm run mssql:init
+
+# run tests, do changes etc.
+npm run mssql:test
+
+# stop mssql container
+npm run mssql:destroy
+```
+
 ## Want to be Collaborator?
 
 There is always room for more collaborators. Be active on resolving github issues / sending pull requests / reviewing code and we will ask you to join.

package/bin/cli.js

@@ -13,9 +13,9 @@ var cliPkg = require('../package');
 
 function exit(text) {
   if (text instanceof Error) {
-    chalk.red(console.error(text.stack));
+    console.error(chalk.red(text.stack));
   } else {
-    chalk.red(console.error(text));
+    console.error(chalk.red(text));
   }
   process.exit(1);
 }
@@ -186,7 +186,8 @@ function invoke(env) {
   commander.parse(process.argv);
 
   Promise.resolve(pending).then(function() {
-    commander.help();
+    commander.outputHelp();
+    exit('Unknown command-line options, exiting');
   });
 }
 

package/lib/client.js

@@ -2,9 +2,9 @@
 
 exports.__esModule = true;
 
-var _get2 = require('lodash/get');
+var _assign2 = require('babel-runtime/core-js/object/assign');
 
-var _get3 = _interopRequireDefault(_get2);
+var _assign3 = _interopRequireDefault(_assign2);
 
 var _defaults2 = require('lodash/defaults');
 
@@ -18,9 +18,9 @@ var _uniqueId2 = require('lodash/uniqueId');
 
 var _uniqueId3 = _interopRequireDefault(_uniqueId2);
 
-var _assign2 = require('lodash/assign');
+var _assign4 = require('lodash/assign');
 
-var _assign3 = _interopRequireDefault(_assign2);
+var _assign5 = _interopRequireDefault(_assign4);
 
 var _bluebird = require('bluebird');
 
@@ -78,13 +78,7 @@ var _columncompiler = require('./schema/columncompiler');
 
 var _columncompiler2 = _interopRequireDefault(_columncompiler);
 
-var _genericPool = require('generic-pool');
-
-var genericPool = _interopRequireWildcard(_genericPool);
-
-var _errors = require('generic-pool/lib/errors');
-
-var genericPoolErrors = _interopRequireWildcard(_errors);
+var _tarn = require('tarn');
 
 var _inherits = require('inherits');
 
@@ -102,11 +96,6 @@ var debug = require('debug')('knex:client');
 var debugQuery = require('debug')('knex:query');
 var debugBindings = require('debug')('knex:bindings');
 
-var id = 0;
-function clientId() {
-  return 'client' + id++;
-}
-
 // The base client provides the general structure
 // for a dialect specific client object.
 function Client() {
@@ -125,7 +114,6 @@ function Client() {
   if (this.driverName && config.connection) {
     this.initializeDriver();
     if (!config.pool || config.pool && config.pool.max !== 0) {
-      this.__cid = clientId();
       this.initializePool(config);
     }
   }
@@ -136,9 +124,9 @@ function Client() {
 }
 (0, _inherits2.default)(Client, _events.EventEmitter);
 
-(0, _assign3.default)(Client.prototype, {
-  formatter: function formatter() {
-    return new _formatter2.default(this);
+(0, _assign5.default)(Client.prototype, {
+  formatter: function formatter(builder) {
+    return new _formatter2.default(this, builder);
   },
   queryBuilder: function queryBuilder() {
     return new _builder2.default(this);
@@ -203,24 +191,38 @@ function Client() {
     var _this2 = this;
 
     if (typeof obj === 'string') obj = { sql: obj };
-    obj.sql = this.positionBindings(obj.sql);
     obj.bindings = this.prepBindings(obj.bindings);
-    debugQuery(obj.sql);
-    this.emit('query', (0, _assign3.default)({ __knexUid: connection.__knexUid }, obj));
-    debugBindings(obj.bindings);
+
+    var __knexUid = connection.__knexUid,
+        __knexTxId = connection.__knexTxId;
+
+
+    this.emit('query', (0, _assign5.default)({ __knexUid: __knexUid, __knexTxId: __knexTxId }, obj));
+    debugQuery(obj.sql, __knexTxId);
+    debugBindings(obj.bindings, __knexTxId);
+
+    obj.sql = this.positionBindings(obj.sql);
+
     return this._query(connection, obj).catch(function (err) {
       err.message = _this2._formatQuery(obj.sql, obj.bindings) + ' - ' + err.message;
-      _this2.emit('query-error', err, (0, _assign3.default)({ __knexUid: connection.__knexUid }, obj));
+      _this2.emit('query-error', err, (0, _assign5.default)({ __knexUid: __knexUid, __knexTxId: __knexTxId }, obj));
       throw err;
     });
   },
   stream: function stream(connection, obj, _stream, options) {
     if (typeof obj === 'string') obj = { sql: obj };
-    obj.sql = this.positionBindings(obj.sql);
     obj.bindings = this.prepBindings(obj.bindings);
-    this.emit('query', (0, _assign3.default)({ __knexUid: connection.__knexUid }, obj));
-    debugQuery(obj.sql);
-    debugBindings(obj.bindings);
+
+    var __knexUid = connection.__knexUid,
+        __knexTxId = connection.__knexTxId;
+
+
+    this.emit('query', (0, _assign5.default)({ __knexUid: __knexUid, __knexTxId: __knexTxId }, obj));
+    debugQuery(obj.sql, __knexTxId);
+    debugBindings(obj.bindings, __knexTxId);
+
+    obj.sql = this.positionBindings(obj.sql);
+
     return this._stream(connection, obj, _stream, options);
   },
   prepBindings: function prepBindings(bindings) {
@@ -229,17 +231,20 @@ function Client() {
   positionBindings: function positionBindings(sql) {
     return sql;
   },
-  postProcessResponse: function postProcessResponse(resp) {
+  postProcessResponse: function postProcessResponse(resp, queryContext) {
     if (this.config.postProcessResponse) {
-      return this.config.postProcessResponse(resp);
+      return this.config.postProcessResponse(resp, queryContext);
     }
     return resp;
   },
-  wrapIdentifier: function wrapIdentifier(value) {
+  wrapIdentifier: function wrapIdentifier(value, queryContext) {
+    return this.customWrapIdentifier(value, this.wrapIdentifierImpl, queryContext);
+  },
+  customWrapIdentifier: function customWrapIdentifier(value, origImpl, queryContext) {
     if (this.config.wrapIdentifier) {
-      return this.config.wrapIdentifier(value, this.wrapIdentifierImpl);
+      return this.config.wrapIdentifier(value, origImpl, queryContext);
     }
-    return this.wrapIdentifierImpl(value);
+    return origImpl(value);
   },
   wrapIdentifierImpl: function wrapIdentifierImpl(value) {
     return value !== '*' ? '"' + value.replace(/"/g, '""') + '"' : '*';
@@ -252,78 +257,59 @@ function Client() {
     }
   },
   poolDefaults: function poolDefaults() {
-    return { min: 2, max: 10, testOnBorrow: true, Promise: _bluebird2.default };
+    return { min: 2, max: 10, propagateCreateError: true };
   },
   getPoolSettings: function getPoolSettings(poolConfig) {
     var _this3 = this;
 
     poolConfig = (0, _defaults3.default)({}, poolConfig, this.poolDefaults());
-    var timeoutValidator = function timeoutValidator(config, path) {
-      var timeout = (0, _get3.default)(config, path);
-      if (timeout !== undefined) {
-        timeout = parseInt(timeout, 10);
-        if (isNaN(timeout) || timeout <= 0) {
-          throw new Error(path + ' must be a positive int');
-        }
+
+    ['maxWaitingClients', 'testOnBorrow', 'fifo', 'priorityRange', 'autostart', 'evictionRunIntervalMillis', 'numTestsPerRun', 'softIdleTimeoutMillis', 'Promise'].forEach(function (option) {
+      if (option in poolConfig) {
+        helpers.warn(['Pool config option "' + option + '" is no longer supported.', 'See https://github.com/Vincit/tarn.js for possible pool config options.'].join(' '));
       }
-      return timeout;
-    };
+    });
 
-    // acquire connection timeout can be set on config or config.pool
-    // choose the smallest, positive timeout setting and set on poolConfig
-    var timeouts = [timeoutValidator(this.config, 'acquireConnectionTimeout') || 60000, timeoutValidator({ pool: poolConfig }, 'pool.acquireTimeoutMillis')].filter(function (timeout) {
+    var timeouts = [this.config.acquireConnectionTimeout || 60000, poolConfig.acquireTimeoutMillis].filter(function (timeout) {
       return timeout !== undefined;
     });
+
+    // acquire connection timeout can be set on config or config.pool
+    // choose the smallest, positive timeout setting and set on poolConfig
     poolConfig.acquireTimeoutMillis = Math.min.apply(Math, timeouts);
 
-    return {
-      config: poolConfig,
-      factory: {
-        create: function create() {
-          return _this3.acquireRawConnection().tap(function (connection) {
-            connection.__knexUid = (0, _uniqueId3.default)('__knexUid');
-            if (poolConfig.afterCreate) {
-              return _bluebird2.default.promisify(poolConfig.afterCreate)(connection);
-            }
-          }).catch(function (err) {
-            // Acquire connection must never reject, because generic-pool
-            // will retry trying to get connection until acquireConnectionTimeout is
-            // reached. acquireConnectionTimeout should trigger in knex only 
-            // in that case if aquiring connection waits because pool is full
-            // https://github.com/coopernurse/node-pool/pull/184
-            // https://github.com/tgriesser/knex/issues/2325
-            return {
-              genericPoolMissingRetryCountHack: true,
-              __knex__disposed: err,
-              query: function query() {
-                throw err; // pass error to query
-              }
-            };
-          });
-        },
-        destroy: function destroy(connection) {
-          if (connection.genericPoolMissingRetryCountHack) {
-            return;
-          }
-          if (poolConfig.beforeDestroy) {
-            helpers.warn('\n              beforeDestroy is deprecated, please open an issue if you use this\n              to discuss alternative apis\n            ');
-            poolConfig.beforeDestroy(connection, function () {});
-          }
-          if (connection !== void 0) {
-            return _this3.destroyRawConnection(connection);
-          }
+    return (0, _assign3.default)(poolConfig, {
+      create: function create() {
+        return _this3.acquireRawConnection().tap(function (connection) {
+          connection.__knexUid = (0, _uniqueId3.default)('__knexUid');
 
-          return _bluebird2.default.resolve();
-        },
-        validate: function validate(connection) {
-          if (connection.__knex__disposed) {
-            helpers.warn('Connection Error: ' + connection.__knex__disposed);
-            return _bluebird2.default.resolve(false);
+          if (poolConfig.afterCreate) {
+            return _bluebird2.default.promisify(poolConfig.afterCreate)(connection);
           }
-          return _this3.validateConnection(connection);
+        });
+      },
+
+      destroy: function destroy(connection) {
+        if (poolConfig.beforeDestroy) {
+          helpers.warn('\n            beforeDestroy is deprecated, please open an issue if you use this\n            to discuss alternative apis\n          ');
+
+          poolConfig.beforeDestroy(connection, function () {});
+        }
+
+        if (connection !== void 0) {
+          return _this3.destroyRawConnection(connection);
         }
+      },
+
+      validate: function validate(connection) {
+        if (connection.__knex__disposed) {
+          helpers.warn('Connection Error: ' + connection.__knex__disposed);
+          return false;
+        }
+
+        return _this3.validateConnection(connection);
       }
-    };
+    });
   },
   initializePool: function initializePool(config) {
     if (this.pool) {
@@ -331,23 +317,26 @@ function Client() {
       return;
     }
 
-    var poolSettings = this.getPoolSettings(config.pool);
-
-    this.pool = genericPool.createPool(poolSettings.factory, poolSettings.config);
+    this.pool = new _tarn.Pool(this.getPoolSettings(config.pool));
   },
   validateConnection: function validateConnection(connection) {
-    return _bluebird2.default.resolve(true);
+    return true;
   },
 
 
   // Acquire a connection from the pool.
   acquireConnection: function acquireConnection() {
+    var _this4 = this;
+
     if (!this.pool) {
       return _bluebird2.default.reject(new Error('Unable to acquire a connection'));
     }
-    return this.pool.acquire().tap(function (connection) {
+
+    return _bluebird2.default.try(function () {
+      return _this4.pool.acquire().promise;
+    }).tap(function (connection) {
       debug('acquired connection from pool: %s', connection.__knexUid);
-    }).catch(genericPoolErrors.TimeoutError, function () {
+    }).catch(_tarn.TimeoutError, function () {
       throw new _bluebird2.default.TimeoutError('Knex: Timeout acquiring a connection. The pool is probably full. ' + 'Are you missing a .transacting(trx) call?');
     });
   },
@@ -357,24 +346,41 @@ function Client() {
   // returning a promise resolved when the connection is released.
   releaseConnection: function releaseConnection(connection) {
     debug('releasing connection to pool: %s', connection.__knexUid);
-    return this.pool.release(connection).catch(function () {
+    var didRelease = this.pool.release(connection);
+
+    if (!didRelease) {
       debug('pool refused connection: %s', connection.__knexUid);
-    });
+    }
+
+    return _bluebird2.default.resolve();
   },
 
 
   // Destroy the current connection pool for the client.
   destroy: function destroy(callback) {
-    var _this4 = this;
+    var _this5 = this;
+
+    var promise = null;
+
+    if (this.pool) {
+      promise = this.pool.destroy();
+    } else {
+      promise = _bluebird2.default.resolve();
+    }
+
+    return promise.then(function () {
+      _this5.pool = void 0;
 
-    return _bluebird2.default.resolve(this.pool && this.pool.drain().then(function () {
-      return _this4.pool.clear();
-    }).then(function () {
-      _this4.pool = void 0;
       if (typeof callback === 'function') {
         callback();
       }
-    }));
+    }).catch(function (err) {
+      if (typeof callback === 'function') {
+        callback(err);
+      }
+
+      return _bluebird2.default.reject(err);
+    });
   },
 
 

package/lib/dialects/maria/index.js

@@ -69,16 +69,19 @@ function Client_MariaSQL(config) {
       var connection = new _this.driver();
       connection.connect((0, _assign3.default)({ metadata: true }, _this.connectionSettings));
       connection.on('ready', function () {
-        connection.removeAllListeners('error');
         resolver(connection);
-      }).on('error', rejecter);
+      }).on('error', function (err) {
+        connection.__knex__disposed = err;
+        rejecter(err);
+      });
     });
   },
   validateConnection: function validateConnection(connection) {
     if (connection.connected === true) {
-      return _bluebird2.default.resolve(true);
+      return true;
     }
-    return _bluebird2.default.resolve(false);
+
+    return false;
   },
 
 

package/lib/dialects/mssql/index.js

@@ -104,7 +104,7 @@ var SQL_BIGINT_SAFE = { MIN: -9007199254740991, MAX: 9007199254740991
     return require('mssql');
   },
   formatter: function formatter() {
-    return new MSSQL_Formatter(this);
+    return new (Function.prototype.bind.apply(MSSQL_Formatter, [null].concat([this], Array.prototype.slice.call(arguments))))();
   },
   transaction: function transaction() {
     return new (Function.prototype.bind.apply(_transaction2.default, [null].concat([this], Array.prototype.slice.call(arguments))))();
@@ -122,7 +122,7 @@ var SQL_BIGINT_SAFE = { MIN: -9007199254740991, MAX: 9007199254740991
     return new (Function.prototype.bind.apply(_columncompiler2.default, [null].concat([this], Array.prototype.slice.call(arguments))))();
   },
   wrapIdentifierImpl: function wrapIdentifierImpl(value) {
-    return value !== '*' ? '[' + value.replace(/\[/g, '\[') + ']' : '*';
+    return value !== '*' ? '[' + value.replace(/\[/g, '[') + ']' : '*';
   },
 
 
@@ -146,9 +146,10 @@ var SQL_BIGINT_SAFE = { MIN: -9007199254740991, MAX: 9007199254740991
   },
   validateConnection: function validateConnection(connection) {
     if (connection.connected === true) {
-      return _bluebird2.default.resolve(true);
+      return true;
     }
-    return _bluebird2.default.resolve(false);
+
+    return false;
   },