@cubejs-backend/mssql-driver 0.32.26 → 0.32.29

package/CHANGELOG.md

@@ -3,6 +3,25 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.32.29](https://github.com/cube-js/cube/compare/v0.32.28...v0.32.29) (2023-04-25)
+
+**Note:** Version bump only for package @cubejs-backend/mssql-driver
+
+
+
+
+
+## [0.32.28](https://github.com/cube-js/cube/compare/v0.32.27...v0.32.28) (2023-04-19)
+
+
+### Bug Fixes
+
+* **mssql-driver:** release stream connection ([#6453](https://github.com/cube-js/cube/issues/6453)) ([7113fa1](https://github.com/cube-js/cube/commit/7113fa1247f3761c9fde5f8d1af8de6036350e72))
+
+
+
+
+
 ## [0.32.26](https://github.com/cube-js/cube/compare/v0.32.25...v0.32.26) (2023-04-13)
 
 **Note:** Version bump only for package @cubejs-backend/mssql-driver

package/driver/MSSqlDriver.js

@@ -10,6 +10,7 @@ const {
 } = require('@cubejs-backend/shared');
 const sql = require('mssql');
 const { BaseDriver } = require('@cubejs-backend/base-driver');
+const QueryStream = require('./QueryStream');
 
 
 const GenericTypeToMSSql = {
@@ -93,6 +94,129 @@ class MSSqlDriver extends BaseDriver {
     return this.initialConnectPromise.then((pool) => pool.request().query('SELECT 1 as number'));
   }
 
+  /**
+   * Executes query in streaming mode.
+   *
+   * @param {string} query 
+   * @param {Array} values 
+   * @param {{ highWaterMark: number? }} options
+   * @return {Promise<StreamTableDataWithTypes>}
+   */
+  async stream(
+    query,
+    values,
+    options,
+  ) {
+    const pool = await this.initialConnectPromise;
+    const request = pool.request();
+
+    request.stream = true;
+    (values || []).forEach((v, i) => {
+      request.input(`_${i + 1}`, v);
+    });
+    request.query(query);
+
+    const stream = new QueryStream(request, options?.highWaterMark);
+    const fields = await new Promise((resolve, reject) => {
+      request.on('recordset', (columns) => {
+        resolve(this.mapFields(columns));
+      });
+      request.on('error', (err) => {
+        reject(err);
+      });
+    });
+    return {
+      rowStream: stream,
+      types: fields,
+      release: async () => {
+        request.cancel();
+      },
+    };
+  }
+
+  /**
+   * @param {{
+   *   [name: string]: {
+   *     index: number,
+   *     name: string,
+   *     type: *,
+   *     nullable: boolean,
+   *     caseSensitive: boolean,
+   *     identity: boolean,
+   *     readOnly: boolean,
+   *     length: number?,
+   *     scale: number?,
+   *     precision: number?
+   *   }
+   * }} fields 
+   */
+  mapFields(fields) {
+    return Object.keys(fields).map((field) => {
+      let type;
+      switch (fields[field].type) {
+        case sql.Bit:
+          type = 'boolean';
+          break;
+        // integers
+        case sql.Int:
+        case sql.SmallInt:
+        case sql.TinyInt:
+          type = 'int';
+          break;
+        // float
+        case sql.Real:
+        case sql.Money:
+        case sql.SmallMoney:
+        case sql.Numeric:
+          type = 'float';
+          break;
+        // double
+        case sql.Float:
+        case sql.Decimal:
+          type = 'double';
+          break;
+        // strings
+        case sql.Char:
+        case sql.NChar:
+        case sql.Text:
+        case sql.VarChar:
+        case sql.NVarChar:
+        case sql.Xml:
+          type = 'text';
+          break;
+        // date and time
+        case sql.Time:
+          type = 'date';
+          break;
+        case sql.Date:
+          type = 'date';
+          break;
+        case sql.DateTime:
+        case sql.DateTime2:
+        case sql.SmallDateTime:
+        case sql.DateTimeOffset:
+          type = 'date';
+          break;
+        // others
+        case sql.UniqueIdentifier:
+        case sql.Variant:
+        case sql.Binary:
+        case sql.VarBinary:
+        case sql.Image:
+        case sql.UDT:
+        case sql.Geography:
+        case sql.Geometry:
+          type = 'string';
+          break;
+        // unknown
+        default:
+          type = 'string';
+          break;
+      }
+      return { name: fields[field].name, type };
+    });
+  }
+
   query(query, values) {
     let cancelFn = null;
     const promise = this.initialConnectPromise.then((pool) => {
@@ -159,7 +283,11 @@ class MSSqlDriver extends BaseDriver {
     `;
   }
 
-  async downloadQueryResults(query, values) {
+  async downloadQueryResults(query, values, options) {
+    if ((options || {}).streamImport) {
+      return this.stream(query, values, options);
+    }
+
     const result = await this.query(query, values);
     const types = Object.keys(result.columns).map((key) => ({
       name: result.columns[key].name,

package/driver/QueryStream.js

@@ -0,0 +1,61 @@
+const { Readable } = require('stream');
+const { getEnv } = require('@cubejs-backend/shared');
+
+/**
+ * MS-SQL query stream class.
+ */
+class QueryStream extends Readable {
+  request = null;
+  recordset = [];
+  done = false;
+
+  /**
+   * @constructor
+   */
+  constructor(request, highWaterMark) {
+    super({
+      objectMode: true,
+      highWaterMark:
+        highWaterMark || getEnv('dbQueryStreamHighWaterMark'),
+    });
+    this.request = request;
+    this.request.on('row', (row) => {
+      this.recordset?.push(row);
+      if (this.recordset?.length === getEnv('dbQueryStreamHighWaterMark')) {
+        this.request.pause();
+      }
+    });
+    this.request.on('error', (err) => {
+      this.destroy(err);
+    });
+    this.request.on('done', () => {
+      this.done = true;
+    });
+  }
+
+  /**
+   * @override
+   */
+  _read(highWaterMark) {
+    const chunk = this.recordset?.splice(0, highWaterMark);
+    chunk?.forEach((row) => {
+      this.push(row);
+    });
+    if (this.recordset?.length === 0 && this.done) {
+      this.push(null);
+    } else {
+      this.request.resume();
+    }
+  }
+
+  /**
+   * @override
+   */
+  _destroy(error, callback) {
+    this.request = null;
+    this.recordset = null;
+    callback(error);
+  }
+}
+
+module.exports = QueryStream;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@cubejs-backend/mssql-driver",
   "description": "Cube.js MS SQL database driver",
   "author": "Cube Dev, Inc.",
-  "version": "0.32.26",
+  "version": "0.32.29",
   "repository": {
     "type": "git",
     "url": "https://github.com/cube-js/cube.git",
@@ -13,12 +13,12 @@
   },
   "main": "driver/MSSqlDriver.js",
   "dependencies": {
-    "@cubejs-backend/base-driver": "^0.32.26",
+    "@cubejs-backend/base-driver": "^0.32.29",
     "mssql": "^6.1.0"
   },
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "5e552433ab582d8f41f272bbee0fd439cc52f8cc"
+  "gitHead": "a41aa533c103c818511845b5df727e0502c03114"
 }