dbgate-api 6.1.6 → 6.2.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "6.1.6",
+  "version": "6.2.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -29,10 +29,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^6.1.6",
+    "dbgate-datalib": "^6.2.1",
     "dbgate-query-splitter": "^4.11.3",
-    "dbgate-sqltree": "^6.1.6",
-    "dbgate-tools": "^6.1.6",
+    "dbgate-sqltree": "^6.2.1",
+    "dbgate-tools": "^6.2.1",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -74,6 +74,8 @@
     "start:storage": "env-cmd -f env/storage/.env node src/index.js --listen-api",
     "start:storage:built": "env-cmd -f env/storage/.env cross-env DEVMODE= BUILTWEBMODE=1 node dist/bundle.js --listen-api",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
+    "start:azure": "env-cmd -f env/azure/.env node src/index.js --listen-api",
+    "start:e2e:team": "cross-env DEVWEB=1 DEVMODE=1 env-cmd -f ../../e2e-tests/env/team/.env node src/index.js --listen-api",
     "ts": "tsc",
     "build": "webpack",
     "build:doc": "jsdoc2md --template doctpl.hbs  ./src/shell/* > ../../../dbgate.github.io/_docs/apidoc.md"
@@ -81,7 +83,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^6.1.6",
+    "dbgate-types": "^6.2.1",
     "env-cmd": "^10.1.0",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",

package/src/controllers/config.js

@@ -136,7 +136,7 @@ module.exports = {
 
   deleteSettings_meta: true,
   async deleteSettings() {
-    await fs.unlink(path.join(datadir(), 'settings.json'));
+    await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'));
     return true;
   },
 
@@ -161,7 +161,10 @@ module.exports = {
 
   async loadSettings() {
     try {
-      const settingsText = await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' });
+      const settingsText = await fs.readFile(
+        path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
+        { encoding: 'utf-8' }
+      );
       return {
         ...this.fillMissingSettings(JSON.parse(settingsText)),
         'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
@@ -247,7 +250,10 @@ module.exports = {
           ...currentValue,
           ..._.omit(values, ['other.licenseKey']),
         };
-        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        await fs.writeFile(
+          path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
+          JSON.stringify(updated, undefined, 2)
+        );
         // this.settingsValue = updated;
 
         if (currentValue['other.licenseKey'] != values['other.licenseKey']) {

package/src/controllers/databaseConnections.js

@@ -34,6 +34,8 @@ const pipeForkLogs = require('../utility/pipeForkLogs');
 const crypto = require('crypto');
 const loadModelTransform = require('../utility/loadModelTransform');
 const exportDbModelSql = require('../utility/exportDbModelSql');
+const axios = require('axios');
+const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy');
 
 const logger = getLogger('databaseConnections');
 
@@ -69,6 +71,11 @@ module.exports = {
   handle_error(conid, database, props) {
     const { error } = props;
     logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
+    if (props?.msgid) {
+      const [resolve, reject] = this.requests[props?.msgid];
+      reject(error);
+      delete this.requests[props?.msgid];
+    }
   },
   handle_response(conid, database, { msgid, ...response }) {
     const [resolve, reject] = this.requests[msgid];
@@ -253,9 +260,9 @@ module.exports = {
   },
 
   loadFieldValues_meta: true,
-  async loadFieldValues({ conid, database, schemaName, pureName, field, search }, req) {
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search, dataType }, req) {
     testConnectionPermission(conid, req);
-    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search, dataType });
   },
 
   callMethod_meta: true,
@@ -401,6 +408,10 @@ module.exports = {
 
   structure_meta: true,
   async structure({ conid, database, modelTransFile = null }, req) {
+    if (!conid || !database) {
+      return {};
+    }
+
     testConnectionPermission(conid, req);
     if (conid == '__model') {
       const model = await importDbModel(database);
@@ -558,4 +569,47 @@ module.exports = {
 
     return true;
   },
+
+  textToSql_meta: true,
+  async textToSql({ conid, database, text, dialect }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    const { structure } = existing || {};
+    if (!structure) return { errorMessage: 'No database structure' };
+
+    const res = await callTextToSqlApi(text, structure, dialect);
+
+    if (!res?.sql) {
+      return { errorMessage: 'No SQL generated' };
+    }
+
+    return res;
+  },
+
+  completeOnCursor_meta: true,
+  async completeOnCursor({ conid, database, text, dialect, line }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    const { structure } = existing || {};
+    if (!structure) return { errorMessage: 'No database structure' };
+    const res = await callCompleteOnCursorApi(text, structure, dialect, line);
+
+    if (!res?.variants) {
+      return { errorMessage: 'No SQL generated' };
+    }
+
+    return res;
+  },
+
+  refactorSqlQuery_meta: true,
+  async refactorSqlQuery({ conid, database, query, task, dialect }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    const { structure } = existing || {};
+    if (!structure) return { errorMessage: 'No database structure' };
+    const res = await callRefactorSqlQueryApi(query, task, structure, dialect);
+
+    if (!res?.sql) {
+      return { errorMessage: 'No SQL generated' };
+    }
+
+    return res;
+  },
 };

package/src/controllers/sessions.js

@@ -56,12 +56,19 @@ module.exports = {
   handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
     if (!props.skipFinishedMessage) {
-      this.dispatchMessage(sesid, 'Query execution finished');
+      if (props.controlCommand) {
+        this.dispatchMessage(sesid, `${_.startCase(props.controlCommand)} finished`);
+      } else {
+        this.dispatchMessage(sesid, 'Query execution finished');
+      }
     }
     const session = this.opened.find(x => x.sesid == sesid);
     if (session.loadingReader_jslid) {
       socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
     }
+    if (props.autoCommit) {
+      this.executeControlCommand({ sesid, command: 'commitTransaction' });
+    }
     if (session.killOnDone) {
       this.kill({ sesid });
     }
@@ -131,7 +138,7 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql }) {
+  async executeQuery({ sesid, sql, autoCommit }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
@@ -139,7 +146,21 @@ module.exports = {
 
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
-    session.subprocess.send({ msgtype: 'executeQuery', sql });
+    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit });
+
+    return { state: 'ok' };
+  },
+
+  executeControlCommand_meta: true,
+  async executeControlCommand({ sesid, command }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    logger.info({ sesid, command }, 'Processing control command');
+    this.dispatchMessage(sesid, `${_.startCase(command)} started`);
+    session.subprocess.send({ msgtype: 'executeControlCommand', command });
 
     return { state: 'ok' };
   },

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '6.1.6',
-  buildTime: '2025-02-04T16:05:53.985Z'
+  version: '6.2.1',
+  buildTime: '2025-02-28T09:57:00.403Z'
 };

package/src/main.js

@@ -78,6 +78,8 @@ function start() {
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
   } else if (platformInfo.isAwsUbuntuLayout) {
     app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
+  } else if (platformInfo.isAzureUbuntuLayout) {
+    app.use(getExpressPath('/'), express.static('/home/azureuser/build/public'));
   } else if (processArgs.runE2eTests) {
     app.use(getExpressPath('/'), express.static(path.resolve('packer/build/public')));
   } else if (platformInfo.isNpmDist) {
@@ -140,6 +142,10 @@ function start() {
     const port = process.env.PORT || 3000;
     logger.info(`DbGate API listening on port ${port} (AWS AMI build)`);
     server.listen(port);
+  } else if (platformInfo.isAzureUbuntuLayout) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (Azure VM build)`);
+    server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
       port: parseInt(

package/src/proc/connectProcess.js

@@ -1,6 +1,6 @@
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
 const _ = require('lodash');

package/src/proc/databaseConnectionProcess.js

@@ -11,7 +11,7 @@ const {
   extractErrorLogData,
 } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
@@ -213,13 +213,12 @@ async function handleRunOperation({ msgid, operation, useTransaction }, skipRead
   }
 }
 
-async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
+async function handleQueryData({ msgid, sql, range }, skipReadonlyCheck = false) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
     if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
-    // console.log(sql);
-    const res = await driver.query(dbhan, sql);
+    const res = await driver.query(dbhan, sql, { range });
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
     process.send({
@@ -234,7 +233,7 @@ async function handleSqlSelect({ msgid, select }) {
   const driver = requireEngineDriver(storedConnection);
   const dmp = driver.createDumper();
   dumpSqlSelect(dmp, select);
-  return handleQueryData({ msgid, sql: dmp.s }, true);
+  return handleQueryData({ msgid, sql: dmp.s, range: select.range }, true);
 }
 
 async function handleDriverDataCore(msgid, callMethod, { logName }) {
@@ -291,10 +290,14 @@ async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
   });
 }
 
-async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
-  return handleDriverDataCore(msgid, driver => driver.loadFieldValues(dbhan, { schemaName, pureName }, field, search), {
-    logName: 'loadFieldValues',
-  });
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search, dataType }) {
+  return handleDriverDataCore(
+    msgid,
+    driver => driver.loadFieldValues(dbhan, { schemaName, pureName }, field, search, dataType),
+    {
+      logName: 'loadFieldValues',
+    }
+  );
 }
 
 function ensureExecuteCustomScript(driver) {
@@ -336,6 +339,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
       }, 500);
     }
   } catch (err) {
+    console.error(err);
     process.send({
       msgtype: 'response',
       msgid,
@@ -423,7 +427,11 @@ function start() {
       await handleMessage(message);
     } catch (err) {
       logger.error(extractErrorLogData(err), 'Error in DB connection');
-      process.send({ msgtype: 'error', error: extractErrorMessage(err, 'Error processing message') });
+      process.send({
+        msgtype: 'error',
+        error: extractErrorMessage(err, 'Error processing message'),
+        msgid: message?.msgid,
+      });
     }
   });
 }

package/src/proc/serverConnectionProcess.js

@@ -2,7 +2,7 @@ const stableStringify = require('json-stable-stringify');
 const { extractBoolSettingsValue, extractIntSettingsValue, getLogger, extractErrorLogData } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const logger = getLogger('srvconnProcess');
 

package/src/proc/sessionProcess.js

@@ -8,7 +8,7 @@ const { splitQuery } = require('dbgate-query-splitter');
 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
 
@@ -245,7 +245,47 @@ async function handleStopProfiler({ jslid }) {
   currentProfiler = null;
 }
 
-async function handleExecuteQuery({ sql }) {
+async function handleExecuteControlCommand({ command }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (command == 'commitTransaction' && !allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
+  executingScripts++;
+  try {
+    const dmp = driver.createDumper();
+    switch (command) {
+      case 'commitTransaction':
+        await dmp.commitTransaction();
+        break;
+      case 'rollbackTransaction':
+        await dmp.rollbackTransaction();
+        break;
+      case 'beginTransaction':
+        await dmp.beginTransaction();
+        break;
+    }
+    await driver.query(dbhan, dmp.s, { discardResult: true });
+    process.send({ msgtype: 'done', controlCommand: command });
+  } finally {
+    executingScripts--;
+  }
+}
+
+async function handleExecuteQuery({ sql, autoCommit }) {
   lastActivity = new Date().getTime();
 
   await waitConnected();
@@ -279,7 +319,7 @@ async function handleExecuteQuery({ sql }) {
       // handler.stream = stream;
       // resultIndex = handler.resultIndex;
     }
-    process.send({ msgtype: 'done' });
+    process.send({ msgtype: 'done', autoCommit });
   } finally {
     executingScripts--;
   }
@@ -323,6 +363,7 @@ function handlePing() {
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeControlCommand: handleExecuteControlCommand,
   executeReader: handleExecuteReader,
   startProfiler: handleStartProfiler,
   stopProfiler: handleStopProfiler,

package/src/shell/dataDuplicator.js

@@ -2,7 +2,7 @@ const stream = require('stream');
 const path = require('path');
 const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const logger = getLogger('dataDuplicator');
 const { DataDuplicator } = require('dbgate-datalib');
 const copyStream = require('./copyStream');

package/src/shell/deployDb.js

@@ -1,7 +1,7 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
 const { ScriptDrivedDeployer } = require('dbgate-datalib');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const loadModelFolder = require('../utility/loadModelFolder');
 const crypto = require('crypto');

package/src/shell/dropAllDbObjects.js

@@ -1,6 +1,6 @@
 const executeQuery = require('./executeQuery');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
 
 const logger = getLogger('dropAllDbObjects');

package/src/shell/dumpDatabase.js

@@ -1,5 +1,5 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { getLogger } = require('dbgate-tools');
 
 const logger = getLogger('dumpDb');

package/src/shell/executeQuery.js

@@ -1,6 +1,6 @@
 const fs = require('fs-extra');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');

package/src/shell/generateDeploySql.js

@@ -13,7 +13,7 @@ const {
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 
 /**
  * Generates query for deploying model into database

package/src/shell/importDatabase.js

@@ -1,6 +1,6 @@
 const fs = require('fs');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
 const download = require('./download');
 const stream = require('stream');

package/src/shell/importDbFromFolder.js

@@ -0,0 +1,110 @@
+const path = require('path');
+const fs = require('fs-extra');
+const executeQuery = require('./executeQuery');
+const { connectUtility } = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const { getAlterDatabaseScript, DatabaseAnalyser, runCommandOnDriver } = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const jsonLinesReader = require('./jsonLinesReader');
+const tableWriter = require('./tableWriter');
+const copyStream = require('./copyStream');
+
+/**
+ * Deploys database model stored in modelFolder (table as yamls) to database
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {string} options.folder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
+ * @param {function[]} options.modelTransforms - array of functions for transforming model
+ */
+async function importDbFromFolder({ connection, systemConnection, driver, folder, modelTransforms }) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    const model = await importDbModel(folder);
+
+    let modelAdapted = {
+      ...model,
+      tables: model.tables.map(table => driver.adaptTableInfo(table)),
+    };
+    for (const transform of modelTransforms || []) {
+      modelAdapted = transform(modelAdapted);
+    }
+
+    const modelNoFk = {
+      ...modelAdapted,
+      tables: modelAdapted.tables.map(table => ({
+        ...table,
+        foreignKeys: [],
+      })),
+    };
+
+    // const plan = createAlterDatabasePlan(
+    //   DatabaseAnalyser.createEmptyStructure(),
+    //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+    //   {},
+    //   DatabaseAnalyser.createEmptyStructure(),
+    //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+    //   driver
+    // );
+    // const dmp1 = driver.createDumper({ useHardSeparator: true });
+    // if (driver.dialect.enableAllForeignKeys) {
+    //   dmp1.enableAllForeignKeys(false);
+    // }
+    // plan.run(dmp1);
+    // if (driver.dialect.enableAllForeignKeys) {
+    //   dmp1.enableAllForeignKeys(true);
+    // }
+
+    const { sql } = getAlterDatabaseScript(
+      DatabaseAnalyser.createEmptyStructure(),
+      driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      {},
+      DatabaseAnalyser.createEmptyStructure(),
+      driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      driver
+    );
+    // console.log('CREATING STRUCTURE:', sql);
+    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+    if (driver.dialect.enableAllForeignKeys) {
+      await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(false));
+    }
+
+    for (const table of modelAdapted.tables) {
+      const fileName = path.join(folder, `${table.pureName}.jsonl`);
+      if (await fs.exists(fileName)) {
+        const src = await jsonLinesReader({ fileName });
+        const dst = await tableWriter({
+          systemConnection: dbhan,
+          pureName: table.pureName,
+          driver,
+          targetTableStructure: table,
+        });
+        await copyStream(src, dst);
+      }
+    }
+
+    if (driver.dialect.enableAllForeignKeys) {
+      await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(true));
+    } else if (driver.dialect.createForeignKey) {
+      const dmp = driver.createDumper();
+      for (const table of modelAdapted.tables) {
+        for (const fk of table.foreignKeys) {
+          dmp.createForeignKey(fk);
+        }
+      }
+
+      // create foreign keys
+      await executeQuery({ connection, systemConnection: dbhan, driver, sql: dmp.s, logScriptItems: true });
+    }
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = importDbFromFolder;

package/src/shell/index.js

@@ -35,6 +35,7 @@ const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
 const generateDeploySql = require('./generateDeploySql');
 const dropAllDbObjects = require('./dropAllDbObjects');
+const importDbFromFolder = require('./importDbFromFolder');
 
 const dbgateApi = {
   queryReader,
@@ -73,6 +74,7 @@ const dbgateApi = {
   autoIndexForeignKeysTransform,
   generateDeploySql,
   dropAllDbObjects,
+  importDbFromFolder,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/loadDatabase.js

@@ -1,5 +1,5 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { getLogger } = require('dbgate-tools');
 const exportDbModel = require('../utility/exportDbModel');
 

package/src/shell/queryReader.js

@@ -1,5 +1,5 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const { getLogger } = require('dbgate-tools');
 const logger = getLogger('queryReader');
 

package/src/shell/tableReader.js

@@ -1,6 +1,6 @@
 const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const logger = getLogger('tableReader');
 
 /**

package/src/shell/tableWriter.js

@@ -1,6 +1,6 @@
 const { fullNameToString, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const connectUtility = require('../utility/connectUtility');
+const { connectUtility } = require('../utility/connectUtility');
 const logger = getLogger('tableWriter');
 
 /**
@@ -15,6 +15,7 @@ const logger = getLogger('tableWriter');
  * @param {boolean} options.truncate - truncate table before insert
  * @param {boolean} options.createIfNotExists - create table if not exists
  * @param {boolean} options.commitAfterInsert - commit transaction after insert
+ * @param {any} options.targetTableStructure - target table structure (don't analyse if given)
  * @returns {Promise<writerType>} - writer object
  */
 async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {

package/src/utility/authProxy.js

@@ -24,6 +24,18 @@ async function getAwsIamToken(params) {
   return null;
 }
 
+async function callTextToSqlApi(text, structure, dialect) {
+  return null;
+}
+
+async function callCompleteOnCursorApi(cursorId, query, position, dialect) {
+  return null;
+}
+
+async function callRefactorSqlQueryApi(query, task, structure, dialect) {
+  return null;
+}
+
 module.exports = {
   isAuthProxySupported,
   authProxyGetRedirectUrl,
@@ -32,4 +44,7 @@ module.exports = {
   getAuthProxyUrl,
   supportsAwsIam,
   getAwsIamToken,
+  callTextToSqlApi,
+  callCompleteOnCursorApi,
+  callRefactorSqlQueryApi,
 };

package/src/utility/connectUtility.js

@@ -47,50 +47,32 @@ async function loadConnection(driver, storedConnection, connectionMode) {
   return storedConnection;
 }
 
-async function connectUtility(driver, storedConnection, connectionMode, additionalOptions = null) {
-  const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
-
-  const connection = {
-    database: connectionLoaded.defaultDatabase,
-    ...decryptConnection(connectionLoaded),
-  };
-
-  if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
-
-  if (connection.useSshTunnel) {
-    const tunnel = await getSshTunnelProxy(connection);
-    if (tunnel.state == 'error') {
-      throw new Error(tunnel.message);
-    }
-
-    connection.server = tunnel.localHost;
-    connection.port = tunnel.localPort;
-  }
-
-  // SSL functionality - copied from https://github.com/beekeeper-studio/beekeeper-studio
+async function extractConnectionSslParams(connection) {
+  /** @type {any} */
+  let ssl = undefined;
   if (connection.useSsl) {
-    connection.ssl = {};
+    ssl = {};
 
     if (connection.sslCaFile) {
-      connection.ssl.ca = await fs.readFile(connection.sslCaFile);
-      connection.ssl.sslCaFile = connection.sslCaFile;
+      ssl.ca = await fs.readFile(connection.sslCaFile);
+      ssl.sslCaFile = connection.sslCaFile;
     }
 
     if (connection.sslCertFile) {
-      connection.ssl.cert = await fs.readFile(connection.sslCertFile);
-      connection.ssl.sslCertFile = connection.sslCertFile;
+      ssl.cert = await fs.readFile(connection.sslCertFile);
+      ssl.sslCertFile = connection.sslCertFile;
     }
 
     if (connection.sslKeyFile) {
-      connection.ssl.key = await fs.readFile(connection.sslKeyFile);
-      connection.ssl.sslKeyFile = connection.sslKeyFile;
+      ssl.key = await fs.readFile(connection.sslKeyFile);
+      ssl.sslKeyFile = connection.sslKeyFile;
     }
 
     if (connection.sslCertFilePassword) {
-      connection.ssl.password = connection.sslCertFilePassword;
+      ssl.password = connection.sslCertFilePassword;
     }
 
-    if (!connection.ssl.key && !connection.ssl.ca && !connection.ssl.cert) {
+    if (!ssl.key && !ssl.ca && !ssl.cert) {
       // TODO: provide this as an option in settings
       // or per-connection as 'reject self-signed certs'
       // How it works:
@@ -98,14 +80,41 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
       // if true, has to be from a public CA
       // Heroku certs are self-signed.
       // if you provide ca/cert/key files, it overrides this
-      connection.ssl.rejectUnauthorized = false;
+      ssl.rejectUnauthorized = false;
     } else {
-      connection.ssl.rejectUnauthorized = connection.sslRejectUnauthorized;
+      ssl.rejectUnauthorized = connection.sslRejectUnauthorized;
+    }
+  }
+  return ssl;
+}
+
+async function connectUtility(driver, storedConnection, connectionMode, additionalOptions = null) {
+  const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
+
+  const connection = {
+    database: connectionLoaded.defaultDatabase,
+    ...decryptConnection(connectionLoaded),
+  };
+
+  if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
+
+  if (connection.useSshTunnel) {
+    const tunnel = await getSshTunnelProxy(connection);
+    if (tunnel.state == 'error') {
+      throw new Error(tunnel.message);
     }
+
+    connection.server = tunnel.localHost;
+    connection.port = tunnel.localPort;
   }
 
+  connection.ssl = await extractConnectionSslParams(connection);
+
   const conn = await driver.connect({ ...connection, ...additionalOptions });
   return conn;
 }
 
-module.exports = connectUtility;
+module.exports = {
+  extractConnectionSslParams,
+  connectUtility,
+};

package/src/utility/directories.js

@@ -58,9 +58,11 @@ const jsldir = dirFunc('jsl', true);
 const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
-const archivedir = dirFunc('archive', false, ['default']);
+const archivedir = processArgs.runE2eTests
+  ? dirFunc('archive-e2etests', false, ['default'])
+  : dirFunc('archive', false, ['default']);
 const appdir = dirFunc('apps');
-const filesdir = dirFunc('files');
+const filesdir = processArgs.runE2eTests ? dirFunc('files-e2etests') : dirFunc('files');
 const logsdir = dirFunc('logs', 3600 * 24 * 7);
 
 function packagedPluginsDir() {
@@ -80,6 +82,9 @@ function packagedPluginsDir() {
   if (platformInfo.isAwsUbuntuLayout) {
     return '/home/ubuntu/build/plugins';
   }
+  if (platformInfo.isAzureUbuntuLayout) {
+    return '/home/azureuser/build/plugins';
+  }
   if (platformInfo.isNpmDist) {
     // node_modules
     return global['PLUGINS_DIR'];

package/src/utility/hardwareFingerprint.js

@@ -73,6 +73,7 @@ async function getPublicHardwareFingerprint() {
       region: fingerprint.region,
       isDocker: platformInfo.isDocker,
       isAwsUbuntuLayout: platformInfo.isAwsUbuntuLayout,
+      isAzureUbuntuLayout: platformInfo.isAzureUbuntuLayout,
       isElectron: platformInfo.isElectron,
     },
   };

package/src/utility/platformInfo.js

@@ -15,6 +15,7 @@ const isNpmDist = !!global['IS_NPM_DIST'];
 const isDbModel = !!global['IS_DB_MODEL'];
 const isForkedApi = processArgs.isForkedApi;
 const isAwsUbuntuLayout = fs.existsSync('/home/ubuntu/build/public');
+const isAzureUbuntuLayout = fs.existsSync('/home/azureuser/build/public');
 
 // function moduleAvailable(name) {
 //   try {
@@ -57,6 +58,7 @@ const platformInfo = {
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
   isAwsUbuntuLayout,
+  isAzureUbuntuLayout,
 };
 
 module.exports = platformInfo;

package/src/utility/processArgs.js

@@ -28,9 +28,9 @@ function getPassArgs() {
   if (listenApiChild) {
     res.push('listen-api-child');
   }
-  // if (runE2eTests) {
-  //   res.push('--run-e2e-tests');
-  // }
+  if (runE2eTests) {
+    res.push('--run-e2e-tests');
+  }
   return res;
 }
 

package/src/utility/useController.js

@@ -86,7 +86,7 @@ module.exports = function useController(app, electron, route, controller) {
               detail: err.detail,
             });
           } else {
-            res.status(500).json({ apiErrorMessage: err.message });
+            res.status(500).json({ apiErrorMessage: (_.isString(err) ? err : err.message) ?? 'Unknown error' });
           }
         }
       });