knex 2.0.0 → 2.3.0

package/CHANGELOG.md

@@ -1,5 +1,58 @@
 # Master (Unreleased)
 
+# 2.3.0 - 31 August, 2022
+
+### New features:
+
+- PostgreSQL: Explicit jsonb support for custom pg clients #5201
+- SQLite: Support returning with sqlite3 and better-sqlite3 #5285
+- MSSQL: Implement mapBinding mssql dialect option #5292
+
+### Typings:
+
+- Update types for TS 4.8 #5279
+- Fix typo #5267
+- Fix WhereJsonObject withCompositeTableType #5306
+- Fix AnalyticFunction type #5304
+- Infer specific column value type in aggregations #5297
+
+# 2.2.0 - 19 July, 2022
+
+### New features:
+
+- Inline primary key creation for postgres flavours #5233
+- SQLite: Add warning for undefined connection file #5223
+- MSSQL: Add JSON parameter support for connection #5200
+
+### Bug fixes:
+
+- PostgreSQL: add primaryKey option for uuid #5212
+
+### Typings:
+
+- Add promisable and better types #5222
+- Update raw query bind parameter type #5208
+
+# 2.1.0 - 26 May, 2022
+
+### New features:
+
+- Improve bundling experience to safely import dialects while using static paths #5142
+- Implement extendable builders #5041
+- PostgreSQL: Refresh materialized view concurrently #5166
+
+### Bug fixes:
+
+- Use correct paths in package.json browser field #5174
+- MariaDB: Fix 'NULL' returned instead of NULL on MariaDB 10.2.6+ #5181
+- MySQL: fix hasColumn Error (hasColumn ('a_id') is true, but hasColumn('a_Id') is false) #5148
+- MSSQL: Fix .hasTable result when using .withSchema #5176
+- Oracle: correctly INSERTS Buffer #4869
+
+### Typings:
+
+- Update type definitions for pg connection #5139
+
 # 2.0.0 - 21 April, 2022
 
 ### Breaking changes

package/CONTRIBUTING.md

@@ -169,6 +169,16 @@ Once this is done, check it works by attempting to login:
 psql -h localhost -U postgres -d knex_test
 ```
 
+## Typescript source files
+
+> TL;DR: Starting with release 2.0.0 Knex is adding support for Typescript source files. Thus to develop in this repo you will need to run `npm run build` each time you edit `.ts` files to generate the resulting `.js` files. This is automatically run whenever you run `npm install` or checkout a new Git branch so when developing in Javascript you don't have to worry about it. It is encouraged that new functionality and sources be written in Typescript but this is not required.
+
+Starting with release 2.0.0, Knex is support source additions in Typescript! This allows for better safety in the code added. However, pre-2.0.0 Knex was always written in pure Javascript and thus a "hybrid" approach is being used for 2.0.0 to allow for the new `.ts` files to exist along `.js` files that make up the majority of this repository.
+
+To develop in this repository use the `npm run build` and `npm run clean` commands to compile and delete the `.js` and related files from `.ts` files. If you wish to have the `tsc` compiled watch and recompile on changes then run `npm run build:ts -- --watch`. Note that for easy integration with Javascript the outputted files are done in a "side-by-side" manner meaning that `lib/foo/bar.ts` will result in `lib/foo/bar.js`. This is done automatically via the npm script command `"prepare"` whenever you run `npm install` and Git hook for `post-checkout` (added by Husky) which executes when you run commands like `git checkout` , thus making it easier to not have to worry about this if you're working in pure Javascript.
+
+The script file `./scripts/update_gitignore_for_tsc_output.js` file is called as part of the `npm run build` command which will update the `lib/.gitignore` file which is used to ensure generated `.js` and related files from `tsc` compilation are not checked into the git repo.
+
 ## Want to be Collaborator?
 
 There is always room for more collaborators. Be active on resolving github issues / sending pull requests / reviewing code and we will ask you to join.

package/lib/dialects/cockroachdb/crdb-columncompiler.js

@@ -0,0 +1,14 @@
+const ColumnCompiler_PG = require('../postgres/schema/pg-columncompiler.js');
+
+class ColumnCompiler_CRDB extends ColumnCompiler_PG {
+  uuid(options = { primaryKey: false }) {
+    return (
+      'uuid' +
+      (this.tableCompiler._canBeAddPrimaryKey(options)
+        ? ' primary key default gen_random_uuid()'
+        : '')
+    );
+  }
+}
+
+module.exports = ColumnCompiler_CRDB;

package/lib/dialects/cockroachdb/index.js

@@ -3,6 +3,7 @@
 const Client_PostgreSQL = require('../postgres');
 const Transaction = require('../postgres/execution/pg-transaction');
 const QueryCompiler = require('./crdb-querycompiler');
+const ColumnCompiler = require('./crdb-columncompiler');
 const TableCompiler = require('./crdb-tablecompiler');
 const ViewCompiler = require('./crdb-viewcompiler');
 const QueryBuilder = require('./crdb-querybuilder');
@@ -19,6 +20,10 @@ class Client_CockroachDB extends Client_PostgreSQL {
     return new QueryCompiler(this, builder, formatter);
   }
 
+  columnCompiler() {
+    return new ColumnCompiler(this, ...arguments);
+  }
+
   tableCompiler() {
     return new TableCompiler(this, ...arguments);
   }

package/lib/dialects/index.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDialectByNameOrAlias = void 0;
+const { resolveClientNameWithAliases } = require('../util/helpers');
+const dbNameToDialectLoader = Object.freeze({
+    'better-sqlite3': () => require('./better-sqlite3'),
+    cockroachdb: () => require('./cockroachdb'),
+    mssql: () => require('./mssql'),
+    mysql: () => require('./mysql'),
+    mysql2: () => require('./mysql2'),
+    oracle: () => require('./oracle'),
+    oracledb: () => require('./oracledb'),
+    pgnative: () => require('./pgnative'),
+    postgres: () => require('./postgres'),
+    redshift: () => require('./redshift'),
+    sqlite3: () => require('./sqlite3'),
+});
+/**
+ * Gets the Dialect object with the given client name or throw an
+ * error if not found.
+ *
+ * NOTE: This is a replacement for prior practice of doing dynamic
+ * string construction for imports of Dialect objects.
+ */
+function getDialectByNameOrAlias(clientName) {
+    const resolvedClientName = resolveClientNameWithAliases(clientName);
+    const dialectLoader = dbNameToDialectLoader[resolvedClientName];
+    if (!dialectLoader) {
+        throw new Error(`Invalid clientName given: ${clientName}`);
+    }
+    return dialectLoader();
+}
+exports.getDialectByNameOrAlias = getDialectByNameOrAlias;
+//# sourceMappingURL=index.js.map

package/lib/dialects/mssql/index.js

@@ -333,14 +333,24 @@ class Client_MSSQL extends Client {
   _typeForBinding(binding) {
     const Driver = this._driver();
 
+    if (
+      this.connectionSettings.options &&
+      this.connectionSettings.options.mapBinding
+    ) {
+      const result = this.connectionSettings.options.mapBinding(binding);
+      if (result) {
+        return [result.value, result.type];
+      }
+    }
+
     switch (typeof binding) {
       case 'string':
-        return Driver.TYPES.NVarChar;
+        return [binding, Driver.TYPES.NVarChar];
       case 'boolean':
-        return Driver.TYPES.Bit;
+        return [binding, Driver.TYPES.Bit];
       case 'number': {
         if (binding % 1 !== 0) {
-          return Driver.TYPES.Float;
+          return [binding, Driver.TYPES.Float];
         }
 
         if (binding < SQL_INT4.MIN || binding > SQL_INT4.MAX) {
@@ -350,25 +360,21 @@ class Client_MSSQL extends Client {
             );
           }
 
-          return Driver.TYPES.BigInt;
+          return [binding, Driver.TYPES.BigInt];
         }
 
-        return Driver.TYPES.Int;
+        return [binding, Driver.TYPES.Int];
       }
       default: {
-        // if (binding === null || typeof binding === 'undefined') {
-        //   return tedious.TYPES.Null;
-        // }
-
         if (binding instanceof Date) {
-          return Driver.TYPES.DateTime;
+          return [binding, Driver.TYPES.DateTime];
         }
 
         if (binding instanceof Buffer) {
-          return Driver.TYPES.VarBinary;
+          return [binding, Driver.TYPES.VarBinary];
         }
 
-        return Driver.TYPES.NVarChar;
+        return [binding, Driver.TYPES.NVarChar];
       }
     }
   }
@@ -401,8 +407,8 @@ class Client_MSSQL extends Client {
   }
 
   // sets a request input parameter. Detects bigints and decimals and sets type appropriately.
-  _setReqInput(req, i, binding) {
-    const tediousType = this._typeForBinding(binding);
+  _setReqInput(req, i, inputBinding) {
+    const [binding, tediousType] = this._typeForBinding(inputBinding);
     const bindingName = 'p'.concat(i);
     let options;
 

package/lib/dialects/mssql/schema/mssql-compiler.js

@@ -49,10 +49,17 @@ class SchemaCompiler_MSSQL extends SchemaCompiler {
       this.builder,
       this.bindingsHolder
     );
-    const sql =
+    const bindings = [tableName];
+    let sql =
       `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES ` +
       `WHERE TABLE_NAME = ${formattedTable}`;
-    this.pushQuery({ sql, output: (resp) => resp.length > 0 });
+
+    if (this.schema) {
+      sql += ' AND TABLE_SCHEMA = ?';
+      bindings.push(this.schema);
+    }
+
+    this.pushQuery({ sql, bindings, output: (resp) => resp.length > 0 });
   }
 
   // Check whether a column exists on the schema.

package/lib/dialects/mysql/query/mysql-querycompiler.js

@@ -131,7 +131,7 @@ class QueryCompiler_MySQL extends QueryCompiler {
       output(resp) {
         const out = resp.reduce(function (columns, val) {
           columns[val.COLUMN_NAME] = {
-            defaultValue: val.COLUMN_DEFAULT,
+            defaultValue: val.COLUMN_DEFAULT === 'NULL' ? null : val.COLUMN_DEFAULT,
             type: val.DATA_TYPE,
             maxLength: val.CHARACTER_MAXIMUM_LENGTH,
             nullable: val.IS_NULLABLE === 'YES',

package/lib/dialects/mysql/schema/mysql-compiler.js

@@ -48,8 +48,8 @@ class SchemaCompiler_MySQL extends SchemaCompiler {
       output(resp) {
         return resp.some((row) => {
           return (
-            this.client.wrapIdentifier(row.Field) ===
-            this.client.wrapIdentifier(column)
+            this.client.wrapIdentifier(row.Field.toLowerCase()) ===
+            this.client.wrapIdentifier(column.toLowerCase())
           );
         });
       },

package/lib/dialects/oracledb/index.js

@@ -111,7 +111,8 @@ class Client_Oracledb extends Client_Oracle {
         this
       );
     } else if (value instanceof BlobHelper) {
-      return 'EMPTY_BLOB()';
+      formatter.bindings.push(value.value);
+      return '?';
     }
     return unwrapRaw(value, true, builder, this, formatter) || '?';
   }

package/lib/dialects/postgres/schema/pg-columncompiler.js

@@ -124,6 +124,13 @@ class ColumnCompiler_PG extends ColumnCompiler {
       (this.tableCompiler._canBeAddPrimaryKey(options) ? ' primary key' : '')
     );
   }
+
+  uuid(options = { primaryKey: false }) {
+    return (
+      'uuid' +
+      (this.tableCompiler._canBeAddPrimaryKey(options) ? ' primary key' : '')
+    );
+  }
 }
 
 ColumnCompiler_PG.prototype.bigint = 'bigint';
@@ -133,12 +140,12 @@ ColumnCompiler_PG.prototype.double = 'double precision';
 ColumnCompiler_PG.prototype.floating = 'real';
 ColumnCompiler_PG.prototype.smallint = 'smallint';
 ColumnCompiler_PG.prototype.tinyint = 'smallint';
-ColumnCompiler_PG.prototype.uuid = 'uuid';
 
 function jsonColumn(client, jsonb) {
   if (
     !client.version ||
     client.config.client === 'cockroachdb' ||
+    client.config.jsonbSupport === true ||
     parseFloat(client.version) >= 9.2
   ) {
     return jsonb ? 'jsonb' : 'json';

package/lib/dialects/postgres/schema/pg-compiler.js

@@ -118,9 +118,9 @@ class SchemaCompiler_PG extends SchemaCompiler {
     );
   }
 
-  refreshMaterializedView(viewName) {
+  refreshMaterializedView(viewName, concurrently = false) {
     this.pushQuery({
-      sql: `refresh materialized view ${this.formatter.wrap(viewName)}`,
+      sql: `refresh materialized view${concurrently ? " concurrently" : ""} ${this.formatter.wrap(viewName)}`,
     });
   }
 

package/lib/dialects/postgres/schema/pg-tablecompiler.js

@@ -44,7 +44,10 @@ class TableCompiler_PG extends TableCompiler {
     const createStatement = ifNot
       ? 'create table if not exists '
       : 'create table ';
-    const columnsSql = ' (' + columns.sql.join(', ') + this._addChecks() + ')';
+    const columnsSql = ` (${columns.sql.join(', ')}${
+      this.primaryKeys() || ''
+    }${this._addChecks()})`;
+
     let sql =
       createStatement +
       this.tableName() +
@@ -65,6 +68,28 @@ class TableCompiler_PG extends TableCompiler {
     if (hasComment) this.comment(this.single.comment);
   }
 
+  primaryKeys() {
+    const pks = (this.grouped.alterTable || []).filter(
+      (k) => k.method === 'primary'
+    );
+    if (pks.length > 0 && pks[0].args.length > 0) {
+      const columns = pks[0].args[0];
+      let constraintName = pks[0].args[1] || '';
+      let deferrable;
+      if (isObject(constraintName)) {
+        ({ constraintName, deferrable } = constraintName);
+      }
+      deferrable = deferrable ? ` deferrable initially ${deferrable}` : '';
+      constraintName = constraintName
+        ? this.formatter.wrap(constraintName)
+        : this.formatter.wrap(`${this.tableNameRaw}_pkey`);
+
+      return `, constraint ${constraintName} primary key (${this.formatter.columnize(
+        columns
+      )})${deferrable}`;
+    }
+  }
+
   addColumns(columns, prefix, colCompilers) {
     if (prefix === this.alterColumnsPrefix) {
       // alter columns
@@ -153,11 +178,13 @@ class TableCompiler_PG extends TableCompiler {
     constraintName = constraintName
       ? this.formatter.wrap(constraintName)
       : this.formatter.wrap(`${this.tableNameRaw}_pkey`);
-    this.pushQuery(
-      `alter table ${this.tableName()} add constraint ${constraintName} primary key (${this.formatter.columnize(
-        columns
-      )})${deferrable}`
-    );
+    if (this.method !== 'create' && this.method !== 'createIfNot') {
+      this.pushQuery(
+        `alter table ${this.tableName()} add constraint ${constraintName} primary key (${this.formatter.columnize(
+          columns
+        )})${deferrable}`
+      );
+    }
   }
 
   unique(columns, indexName) {

package/lib/dialects/sqlite3/index.js

@@ -20,11 +20,20 @@ const QueryBuilder = require('./query/sqlite-querybuilder');
 class Client_SQLite3 extends Client {
   constructor(config) {
     super(config);
+
+    if (config.connection && config.connection.filename === undefined) {
+      this.logger.warn(
+        'Could not find `connection.filename` in config. Please specify ' +
+          'the database path and name to avoid errors. ' +
+          '(see docs https://knexjs.org/guide/#configuration-options)'
+      );
+    }
+
     if (config.useNullAsDefault === undefined) {
       this.logger.warn(
         'sqlite does not support inserting default values. Set the ' +
           '`useNullAsDefault` flag to hide this warning. ' +
-          '(see docs http://knexjs.org/#Builder-insert).'
+          '(see docs https://knexjs.org/guide/query-builder.html#insert).'
       );
     }
   }
@@ -117,6 +126,8 @@ class Client_SQLite3 extends Client {
     switch (method) {
       case 'insert':
       case 'update':
+        callMethod = obj.returning ? 'all' : 'run';
+        break;
       case 'counter':
       case 'del':
         callMethod = 'run';
@@ -181,16 +192,18 @@ class Client_SQLite3 extends Client {
           if (response) {
             return response;
           }
-
-          // ToDo Implement after https://github.com/microsoft/vscode-node-sqlite3/issues/15 is resolved
-          this.logger.warn(
-            'node-sqlite3 does not currently support RETURNING clause'
-          );
         }
         return [ctx.lastID];
       }
+      case 'update': {
+        if (returning) {
+          if (response) {
+            return response;
+          }
+        }
+        return ctx.changes;
+      }
       case 'del':
-      case 'update':
       case 'counter':
         return ctx.changes;
       default: {

package/lib/dialects/sqlite3/query/sqlite-querycompiler.js

@@ -76,7 +76,7 @@ class QueryCompiler_SQLite3 extends QueryCompiler {
             throw new TypeError(
               '`sqlite` does not support inserting default values. Specify ' +
                 'values explicitly or use the `useNullAsDefault` config flag. ' +
-                '(see docs http://knexjs.org/#Builder-insert).'
+                '(see docs https://knexjs.org/guide/query-builder.html#insert).'
             );
         });
       });
@@ -149,6 +149,23 @@ class QueryCompiler_SQLite3 extends QueryCompiler {
     };
   }
 
+  // Compiles an `update` query, allowing for a return value.
+  update() {
+    const withSQL = this.with();
+    const updateData = this._prepUpdate(this.single.update);
+    const wheres = this.where();
+    const { returning } = this.single;
+    return {
+      sql:
+        withSQL +
+        `update ${this.single.only ? 'only ' : ''}${this.tableName} ` +
+        `set ${updateData.join(', ')}` +
+        (wheres ? ` ${wheres}` : '') +
+        this._returning(returning),
+      returning,
+    };
+  }
+
   _ignore(columns) {
     if (columns === true) {
       return ' on conflict do nothing';

package/lib/execution/runner.js

@@ -61,22 +61,15 @@ class Runner {
     Transform = Transform || require('stream').Transform;
 
     const queryContext = this.builder.queryContext();
-    let queryStream;
 
     const stream = new Transform({
       objectMode: true,
       transform: (chunk, _, callback) => {
         callback(null, this.client.postProcessResponse(chunk, queryContext));
       },
-      destroy() {
-        // For some reason destroy is not available for mssql on Node 14. Might be a problem with tedious: https://github.com/tediousjs/tedious/issues/1139
-        if (queryStream && queryStream.destroy) {
-          queryStream.destroy(new Error('stream destroyed'));
-        }
-      },
     });
-    stream.on('pipe', (qs) => {
-      queryStream = qs;
+    stream.on('close', () => {
+      this.client.releaseConnection(this.connection);
     });
 
     const connectionAcquirePromise = this.ensureConnection(

package/lib/formatter/wrappingFormatter.js

@@ -225,8 +225,7 @@ function rawOrFn(value, method, builder, client, bindingHolder) {
       compileCallback(value, method, client, bindingHolder),
       undefined,
       builder,
-      client,
-      bindingHolder
+      client
     );
   }
   return unwrapRaw(value, undefined, builder, client, bindingHolder) || '';

package/lib/knex-builder/Knex.js

@@ -5,6 +5,10 @@ const QueryInterface = require('../query/method-constants');
 const makeKnex = require('./make-knex');
 const { KnexTimeoutError } = require('../util/timeout');
 const { resolveConfig } = require('./internal/config-resolver');
+const SchemaBuilder = require('../schema/builder');
+const ViewBuilder = require('../schema/viewbuilder');
+const ColumnBuilder = require('../schema/columnbuilder');
+const TableBuilder = require('../schema/tablebuilder');
 
 function knex(config) {
   const { resolvedConfig, Dialect } = resolveConfig(...arguments);
@@ -28,4 +32,28 @@ knex.QueryBuilder = {
   },
 };
 
+knex.SchemaBuilder = {
+  extend: function (methodName, fn) {
+    SchemaBuilder.extend(methodName, fn);
+  },
+};
+
+knex.ViewBuilder = {
+  extend: function (methodName, fn) {
+    ViewBuilder.extend(methodName, fn);
+  },
+};
+
+knex.ColumnBuilder = {
+  extend: function (methodName, fn) {
+    ColumnBuilder.extend(methodName, fn);
+  },
+};
+
+knex.TableBuilder  = {
+  extend: function (methodName, fn) {
+    TableBuilder.extend(methodName, fn);
+  },
+};
+
 module.exports = knex;

package/lib/knex-builder/internal/config-resolver.js

@@ -2,7 +2,7 @@ const Client = require('../../client');
 const { SUPPORTED_CLIENTS } = require('../../constants');
 
 const parseConnection = require('./parse-connection');
-const { resolveClientNameWithAliases } = require('../../util/helpers');
+const { getDialectByNameOrAlias } = require('../../dialects');
 
 function resolveConfig(config) {
   let Dialect;
@@ -34,8 +34,7 @@ function resolveConfig(config) {
       );
     }
 
-    const resolvedClientName = resolveClientNameWithAliases(clientName);
-    Dialect = require(`../../dialects/${resolvedClientName}/index.js`);
+    Dialect = getDialectByNameOrAlias(clientName);
   }
 
   // If config connection parameter is passed as string, try to parse it

package/lib/knex-builder/internal/parse-connection.js

@@ -69,8 +69,10 @@ function connectionObject(parsed) {
   }
   if (parsed.searchParams) {
     for (const [key, value] of parsed.searchParams.entries()) {
-      const isMySQLLike = ['mysql:', 'mariadb:'].includes(parsed.protocol);
-      if (isMySQLLike) {
+      const isNestedConfigSupported = ['mysql:', 'mariadb:', 'mssql:'].includes(
+        parsed.protocol
+      );
+      if (isNestedConfigSupported) {
         try {
           connection[key] = JSON.parse(value);
         } catch (err) {

package/lib/migrations/migrate/MigrationGenerator.js

@@ -41,7 +41,7 @@ class MigrationGenerator {
 
   _getNewMigrationName(name) {
     if (name[0] === '-') name = name.slice(1);
-    return yyyymmddhhmmss() + '_' + name + '.' + this.config.extension;
+    return yyyymmddhhmmss() + '_' + name + '.' + this.config.extension.split('-')[0];
   }
 
   _getNewMigrationPath(name) {

package/lib/migrations/migrate/stub/js-schema.stub

@@ -0,0 +1,22 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function({schema}) {
+  <% if (d.tableName) { %>
+  return schema.createTable("<%= d.tableName %>", function(t) {
+    t.increments();
+    t.timestamp();
+  });
+  <% } %>
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function({schema}) {
+  <% if (d.tableName) { %>
+  return schema.dropTable("<%= d.tableName %>");
+  <% } %>
+};

package/lib/migrations/migrate/stub/ts-schema.stub

@@ -0,0 +1,21 @@
+import { Knex } from "knex";
+
+<% if (d.tableName) { %>
+export async function up({schema}: Knex): Promise<Knex.SchemaBuilder> {
+    return schema.createTable("<%= d.tableName %>", (t) => {
+        t.increments();
+        t.timestamps();
+    });
+}
+<% } else { %>
+export async function up({schema}: Knex): Promise<void> {
+}
+<% } %>
+<% if (d.tableName) { %>
+export async function down({schema}: Knex): Promise<Knex.SchemaBuilder> {
+    return schema.dropTable("<%= d.tableName %>");
+}
+<% } else { %>
+export async function down({schema}: Knex): Promise<void> {
+}
+<% } %>

package/lib/schema/builder.js

@@ -1,5 +1,6 @@
 const { EventEmitter } = require('events');
 const toArray = require('lodash/toArray');
+const assign = require('lodash/assign');
 const { addQueryContext } = require('../util/helpers');
 const saveAsyncStack = require('../util/save-async-stack');
 const {
@@ -96,6 +97,17 @@ class SchemaBuilder extends EventEmitter {
   };
 });
 
+
+SchemaBuilder.extend = (methodName, fn) => {
+  if (Object.prototype.hasOwnProperty.call(SchemaBuilder.prototype, methodName)) {
+    throw new Error(
+      `Can't extend SchemaBuilder with existing method ('${methodName}').`
+    );
+  }
+
+  assign(SchemaBuilder.prototype, { [methodName]: fn });
+};
+
 augmentWithBuilderInterface(SchemaBuilder);
 addQueryContext(SchemaBuilder);