dbgate-api-premium 6.2.0 → 6.3.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api-premium",
   "main": "src/index.js",
-  "version": "6.2.0",
+  "version": "6.3.0",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -29,10 +29,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^6.2.0",
+    "dbgate-datalib": "^6.3.0",
     "dbgate-query-splitter": "^4.11.3",
-    "dbgate-sqltree": "^6.2.0",
-    "dbgate-tools": "^6.2.0",
+    "dbgate-sqltree": "^6.3.0",
+    "dbgate-tools": "^6.3.0",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -83,7 +83,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^6.2.0",
+    "dbgate-types": "^6.3.0",
     "env-cmd": "^10.1.0",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",

package/src/controllers/auth.js

@@ -43,6 +43,7 @@ function authMiddleware(req, res, next) {
     '/connections/dblogin-app',
     '/connections/dblogin-auth',
     '/connections/dblogin-auth-token',
+    '/health',
   ];
 
   // console.log('********************* getAuthProvider()', getAuthProvider());

package/src/controllers/connections.js

@@ -62,7 +62,10 @@ function getPortalCollections() {
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
       useDatabaseUrl: !!process.env[`URL_${id}`],
-      databaseFile: process.env[`FILE_${id}`],
+      databaseFile: process.env[`FILE_${id}`]?.replace(
+        '%%E2E_TEST_DATA_DIRECTORY%%',
+        path.join(path.dirname(path.dirname(__dirname)), 'e2e-tests', 'tmpdata')
+      ),
       socketPath: process.env[`SOCKET_PATH_${id}`],
       serviceName: process.env[`SERVICE_NAME_${id}`],
       authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
@@ -77,6 +80,7 @@ function getPortalCollections() {
       allowedDatabasesRegex: process.env[`ALLOWED_DATABASES_REGEX_${id}`],
       parent: process.env[`PARENT_${id}`] || undefined,
       useSeparateSchemas: !!process.env[`USE_SEPARATE_SCHEMAS_${id}`],
+      localDataCenter: process.env[`LOCAL_DATA_CENTER_${id}`],
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],

package/src/controllers/databaseConnections.js

@@ -1,4 +1,5 @@
 const connections = require('./connections');
+const runners = require('./runners');
 const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
@@ -233,6 +234,7 @@ module.exports = {
   dispatchDatabaseChangedEvent_meta: true,
   dispatchDatabaseChangedEvent({ event, conid, database }) {
     socket.emitChanged(event, { conid, database });
+    return null;
   },
 
   loadKeys_meta: true,
@@ -612,4 +614,129 @@ module.exports = {
 
     return res;
   },
+
+  async getNativeOpCommandArgs(
+    command,
+    { conid, database, outputFile, inputFile, options, selectedTables, skippedTables, argsFormat }
+  ) {
+    const connection = await connections.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+
+    const settingsValue = await config.getSettings();
+
+    const externalTools = {};
+    for (const pair of Object.entries(settingsValue || {})) {
+      const [name, value] = pair;
+      if (name.startsWith('externalTools.')) {
+        externalTools[name.substring('externalTools.'.length)] = value;
+      }
+    }
+
+    return {
+      ...(command == 'backup'
+        ? driver.backupDatabaseCommand(
+            connection,
+            { outputFile, database, options, selectedTables, skippedTables, argsFormat },
+            // @ts-ignore
+            externalTools
+          )
+        : driver.restoreDatabaseCommand(
+            connection,
+            { inputFile, database, options, argsFormat },
+            // @ts-ignore
+            externalTools
+          )),
+      transformMessage: driver.transformNativeCommandMessage
+        ? message => driver.transformNativeCommandMessage(message, command)
+        : null,
+    };
+  },
+
+  commandArgsToCommandLine(commandArgs) {
+    const { command, args, stdinFilePath } = commandArgs;
+    let res = `${command} ${args.join(' ')}`;
+    if (stdinFilePath) {
+      res += ` < ${stdinFilePath}`;
+    }
+    return res;
+  },
+
+  nativeBackup_meta: true,
+  async nativeBackup({ conid, database, outputFile, runid, options, selectedTables, skippedTables }) {
+    const commandArgs = await this.getNativeOpCommandArgs('backup', {
+      conid,
+      database,
+      inputFile: undefined,
+      outputFile,
+      options,
+      selectedTables,
+      skippedTables,
+      argsFormat: 'spawn',
+    });
+
+    return runners.nativeRunCore(runid, {
+      ...commandArgs,
+      onFinished: () => {
+        socket.emitChanged(`files-changed`, { folder: 'sql' });
+        socket.emitChanged(`all-files-changed`);
+      },
+    });
+  },
+
+  nativeBackupCommand_meta: true,
+  async nativeBackupCommand({ conid, database, outputFile, options, selectedTables, skippedTables }) {
+    const commandArgs = await this.getNativeOpCommandArgs('backup', {
+      conid,
+      database,
+      outputFile,
+      inputFile: undefined,
+      options,
+      selectedTables,
+      skippedTables,
+      argsFormat: 'shell',
+    });
+
+    return {
+      ...commandArgs,
+      transformMessage: null,
+      commandLine: this.commandArgsToCommandLine(commandArgs),
+    };
+  },
+
+  nativeRestore_meta: true,
+  async nativeRestore({ conid, database, inputFile, runid }) {
+    const commandArgs = await this.getNativeOpCommandArgs('restore', {
+      conid,
+      database,
+      inputFile,
+      outputFile: undefined,
+      options: undefined,
+      argsFormat: 'spawn',
+    });
+
+    return runners.nativeRunCore(runid, {
+      ...commandArgs,
+      onFinished: () => {
+        this.syncModel({ conid, database, isFullRefresh: true });
+      },
+    });
+  },
+
+  nativeRestoreCommand_meta: true,
+  async nativeRestoreCommand({ conid, database, inputFile }) {
+    const commandArgs = await this.getNativeOpCommandArgs('restore', {
+      conid,
+      database,
+      inputFile,
+      outputFile: undefined,
+      options: undefined,
+      argsFormat: 'shell',
+    });
+
+    return {
+      ...commandArgs,
+      transformMessage: null,
+      commandLine: this.commandArgsToCommandLine(commandArgs),
+    };
+  },
 };

package/src/controllers/runners.js

@@ -4,7 +4,7 @@ const path = require('path');
 const fs = require('fs-extra');
 const byline = require('byline');
 const socket = require('../utility/socket');
-const { fork } = require('child_process');
+const { fork, spawn } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const {
   extractShellApiPlugins,
@@ -13,6 +13,8 @@ const {
   getLogger,
   safeJsonParse,
   pinoLogRecordToMessageRecord,
+  extractErrorMessage,
+  extractErrorLogData,
 } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
@@ -80,6 +82,7 @@ module.exports = {
           }
         : {
             message,
+            severity: 'info',
             time: new Date(),
           };
 
@@ -94,14 +97,26 @@ module.exports = {
   handle_ping() {},
 
   handle_freeData(runid, { freeData }) {
-    const [resolve, reject] = this.requests[runid];
+    const { resolve } = this.requests[runid];
     resolve(freeData);
     delete this.requests[runid];
   },
 
+  handle_copyStreamError(runid, { copyStreamError }) {
+    const { reject, exitOnStreamError } = this.requests[runid] || {};
+    if (exitOnStreamError) {
+      reject(copyStreamError);
+      delete this.requests[runid];
+    }
+  },
+
+  handle_progress(runid, progressData) {
+    socket.emit(`runner-progress-${runid}`, progressData);
+  },
+
   rejectRequest(runid, error) {
     if (this.requests[runid]) {
-      const [resolve, reject] = this.requests[runid];
+      const { reject } = this.requests[runid];
       reject(error);
       delete this.requests[runid];
     }
@@ -113,6 +128,8 @@ module.exports = {
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
     const pluginNames = extractPlugins(scriptText);
+    // console.log('********************** SCRIPT TEXT **********************');
+    // console.log(scriptText);
     logger.info({ scriptFile }, 'Running script');
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
     const subprocess = fork(
@@ -150,17 +167,21 @@ module.exports = {
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
+      // console.log('... EXITED', code);
       this.rejectRequest(runid, { message: 'No data returned, maybe input data source is too big' });
       logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
     subprocess.on('error', error => {
+      // console.log('... ERROR subprocess', error);
       this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
       console.error('... ERROR subprocess', error);
-      this.dispatchMessage({
+      this.dispatchMessage(runid, {
         severity: 'error',
         message: error.toString(),
       });
+      this.opened = this.opened.filter(x => x.runid != runid);
     });
     const newOpened = {
       runid,
@@ -176,6 +197,77 @@ module.exports = {
     return _.pick(newOpened, ['runid']);
   },
 
+  nativeRunCore(runid, commandArgs) {
+    const { command, args, env, transformMessage, stdinFilePath, onFinished } = commandArgs;
+    const pipeDispatcher = severity => data => {
+      let messageObject = {
+        message: data.toString().trim(),
+        severity,
+      };
+      if (transformMessage) {
+        messageObject = transformMessage(messageObject);
+      }
+
+      if (messageObject) {
+        return this.dispatchMessage(runid, messageObject);
+      }
+    };
+
+    const subprocess = spawn(command, args, { env: { ...process.env, ...env } });
+
+    byline(subprocess.stdout).on('data', pipeDispatcher('info'));
+    byline(subprocess.stderr).on('data', pipeDispatcher('error'));
+
+    subprocess.on('exit', code => {
+      console.log('... EXITED', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
+      this.dispatchMessage(runid, `Finished external process with code ${code}`);
+      socket.emit(`runner-done-${runid}`, code);
+      if (onFinished) {
+        onFinished();
+      }
+      this.opened = this.opened.filter(x => x.runid != runid);
+    });
+    subprocess.on('spawn', () => {
+      this.dispatchMessage(runid, `Started external process ${command}`);
+    });
+    subprocess.on('error', error => {
+      console.log('... ERROR subprocess', error);
+      this.dispatchMessage(runid, {
+        severity: 'error',
+        message: error.toString(),
+      });
+      if (error['code'] == 'ENOENT') {
+        this.dispatchMessage(runid, {
+          severity: 'error',
+          message: `Command ${command} not found, please install it and configure its location in DbGate settings, Settings/External tools, if ${command} is not in system PATH`,
+        });
+      }
+      socket.emit(`runner-done-${runid}`);
+      this.opened = this.opened.filter(x => x.runid != runid);
+    });
+
+    if (stdinFilePath) {
+      const inputStream = fs.createReadStream(stdinFilePath);
+      inputStream.pipe(subprocess.stdin);
+
+      subprocess.stdin.on('error', err => {
+        this.dispatchMessage(runid, {
+          severity: 'error',
+          message: extractErrorMessage(err),
+        });
+        logger.error(extractErrorLogData(err), 'Caught error on stdin');
+      });
+    }
+
+    const newOpened = {
+      runid,
+      subprocess,
+    };
+    this.opened.push(newOpened);
+    return _.pick(newOpened, ['runid']);
+  },
+
   start_meta: true,
   async start({ script }) {
     const runid = crypto.randomUUID();
@@ -231,7 +323,7 @@ module.exports = {
 
     const promise = new Promise((resolve, reject) => {
       const runid = crypto.randomUUID();
-      this.requests[runid] = [resolve, reject];
+      this.requests[runid] = { resolve, reject, exitOnStreamError: true };
       this.startCore(runid, loaderScriptTemplate(prefix, functionName, props, runid));
     });
     return promise;

package/src/controllers/sessions.js

@@ -56,12 +56,19 @@ module.exports = {
   handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
     if (!props.skipFinishedMessage) {
-      this.dispatchMessage(sesid, 'Query execution finished');
+      if (props.controlCommand) {
+        this.dispatchMessage(sesid, `${_.startCase(props.controlCommand)} finished`);
+      } else {
+        this.dispatchMessage(sesid, 'Query execution finished');
+      }
     }
     const session = this.opened.find(x => x.sesid == sesid);
     if (session.loadingReader_jslid) {
       socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
     }
+    if (props.autoCommit) {
+      this.executeControlCommand({ sesid, command: 'commitTransaction' });
+    }
     if (session.killOnDone) {
       this.kill({ sesid });
     }
@@ -120,6 +127,9 @@ module.exports = {
       this.dispatchMessage(sesid, 'Query session closed');
       socket.emit(`session-closed-${sesid}`);
     });
+    subprocess.on('error', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+    });
 
     subprocess.send({
       msgtype: 'connect',
@@ -131,7 +141,7 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql }) {
+  async executeQuery({ sesid, sql, autoCommit }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
@@ -139,7 +149,21 @@ module.exports = {
 
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
-    session.subprocess.send({ msgtype: 'executeQuery', sql });
+    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit });
+
+    return { state: 'ok' };
+  },
+
+  executeControlCommand_meta: true,
+  async executeControlCommand({ sesid, command }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    logger.info({ sesid, command }, 'Processing control command');
+    this.dispatchMessage(sesid, `${_.startCase(command)} started`);
+    session.subprocess.send({ msgtype: 'executeControlCommand', command });
 
     return { state: 'ok' };
   },

package/src/controllers/uploads.js

@@ -1,6 +1,6 @@
 const crypto = require('crypto');
 const path = require('path');
-const { uploadsdir, getLogsFilePath } = require('../utility/directories');
+const { uploadsdir, getLogsFilePath, filesdir } = require('../utility/directories');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const logger = getLogger('uploads');
 const axios = require('axios');
@@ -13,6 +13,7 @@ const serverConnections = require('./serverConnections');
 const config = require('./config');
 const gistSecret = require('../gistSecret');
 const currentVersion = require('../currentVersion');
+const socket = require('../utility/socket');
 
 module.exports = {
   upload_meta: {
@@ -38,6 +39,52 @@ module.exports = {
     });
   },
 
+  uploadDataFile_meta: {
+    method: 'post',
+    raw: true,
+  },
+  uploadDataFile(req, res) {
+    const { data } = req.files || {};
+
+    if (!data) {
+      res.json(null);
+      return;
+    }
+
+    if (data.name.toLowerCase().endsWith('.sql')) {
+      logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
+      data.mv(path.join(filesdir(), 'sql', data.name), () => {
+        res.json({
+          name: data.name,
+          folder: 'sql',
+        });
+
+        socket.emitChanged(`files-changed`, { folder: 'sql' });
+        socket.emitChanged(`all-files-changed`);
+      });
+      return;
+    }
+
+    res.json(null);
+  },
+
+  saveDataFile_meta: true,
+  async saveDataFile({ filePath }) {
+    if (filePath.toLowerCase().endsWith('.sql')) {
+      logger.info(`Saving SQL file ${filePath}`);
+      await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
+
+      socket.emitChanged(`files-changed`, { folder: 'sql' });
+      socket.emitChanged(`all-files-changed`);
+      return {
+        name: path.basename(filePath),
+        folder: 'sql',
+      };
+    }
+
+    return null;
+  },
+
   get_meta: {
     method: 'get',
     raw: true,

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '6.2.0',
-  buildTime: '2025-02-14T13:58:57.650Z'
+  version: '6.3.0',
+  buildTime: '2025-03-18T11:15:02.094Z'
 };

package/src/main.js

@@ -30,7 +30,7 @@ const queryHistory = require('./controllers/queryHistory');
 const onFinished = require('on-finished');
 const processArgs = require('./utility/processArgs');
 
-const { rundir } = require('./utility/directories');
+const { rundir, filesdir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
 const _ = require('lodash');
@@ -38,6 +38,7 @@ const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
 const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 const { isProApp } = require('./utility/checkLicense');
+const getHealthStatus = require('./utility/healthStatus');
 
 const logger = getLogger('main');
 
@@ -117,6 +118,12 @@ function start() {
     });
   });
 
+  app.get(getExpressPath('/health'), async function (req, res) {
+    res.setHeader('Content-Type', 'application/json');
+    const health = await getHealthStatus();
+    res.end(JSON.stringify(health, null, 2));
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
@@ -133,6 +140,7 @@ function start() {
   // }
 
   app.use(getExpressPath('/runners/data'), express.static(rundir()));
+  app.use(getExpressPath('/files/data'), express.static(filesdir()));
 
   if (platformInfo.isDocker) {
     const port = process.env.PORT || 3000;

package/src/proc/sessionProcess.js

@@ -245,7 +245,47 @@ async function handleStopProfiler({ jslid }) {
   currentProfiler = null;
 }
 
-async function handleExecuteQuery({ sql }) {
+async function handleExecuteControlCommand({ command }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (command == 'commitTransaction' && !allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
+  executingScripts++;
+  try {
+    const dmp = driver.createDumper();
+    switch (command) {
+      case 'commitTransaction':
+        await dmp.commitTransaction();
+        break;
+      case 'rollbackTransaction':
+        await dmp.rollbackTransaction();
+        break;
+      case 'beginTransaction':
+        await dmp.beginTransaction();
+        break;
+    }
+    await driver.query(dbhan, dmp.s, { discardResult: true });
+    process.send({ msgtype: 'done', controlCommand: command });
+  } finally {
+    executingScripts--;
+  }
+}
+
+async function handleExecuteQuery({ sql, autoCommit }) {
   lastActivity = new Date().getTime();
 
   await waitConnected();
@@ -279,7 +319,7 @@ async function handleExecuteQuery({ sql }) {
       // handler.stream = stream;
       // resultIndex = handler.resultIndex;
     }
-    process.send({ msgtype: 'done' });
+    process.send({ msgtype: 'done', autoCommit });
   } finally {
     executingScripts--;
   }
@@ -323,6 +363,7 @@ function handlePing() {
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeControlCommand: handleExecuteControlCommand,
   executeReader: handleExecuteReader,
   startProfiler: handleStartProfiler,
   stopProfiler: handleStopProfiler,

package/src/shell/copyStream.js

@@ -1,6 +1,27 @@
 const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
-const Stream = require('stream');
 const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+const streamPipeline = require('../utility/streamPipeline');
+const { getLogger, extractErrorLogData, RowProgressReporter, extractErrorMessage } = require('dbgate-tools');
+const logger = getLogger('copyStream');
+const stream = require('stream');
+
+class ReportingTransform extends stream.Transform {
+  constructor(reporter, options = {}) {
+    super({ ...options, objectMode: true });
+    this.reporter = reporter;
+  }
+  _transform(chunk, encoding, callback) {
+    if (!chunk?.__isStreamHeader) {
+      this.reporter.add(1);
+    }
+    this.push(chunk);
+    callback();
+  }
+  _flush(callback) {
+    this.reporter.finish();
+    callback();
+  }
+}
 
 /**
  * Copies reader to writer. Used for import, export tables and transfer data between tables
@@ -9,10 +30,23 @@ const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
  * @param {object} options - options
  * @returns {Promise}
  */
-function copyStream(input, output, options) {
-  const { columns } = options || {};
+async function copyStream(input, output, options) {
+  const { columns, progressName } = options || {};
+
+  if (progressName) {
+    process.send({
+      msgtype: 'progress',
+      progressName,
+      status: 'running',
+    });
+  }
 
   const transforms = [];
+
+  if (progressName) {
+    const reporter = new RowProgressReporter(progressName, 'readRowCount');
+    transforms.push(new ReportingTransform(reporter));
+  }
   if (columns) {
     transforms.push(new ColumnMapTransformStream(columns));
   }
@@ -20,36 +54,37 @@ function copyStream(input, output, options) {
     transforms.push(new EnsureStreamHeaderStream());
   }
 
-  // return new Promise((resolve, reject) => {
-  //   Stream.pipeline(input, ...transforms, output, err => {
-  //     if (err) {
-  //       reject(err);
-  //     } else {
-  //       resolve();
-  //     }
-  //   });
-  // });
-
-  return new Promise((resolve, reject) => {
-    const finisher = output['finisher'] || output;
-    finisher.on('finish', resolve);
-    finisher.on('error', reject);
-
-    let lastStream = input;
-    for (const tran of transforms) {
-      lastStream.pipe(tran);
-      lastStream = tran;
+  try {
+    await streamPipeline(input, transforms, output);
+
+    if (progressName) {
+      process.send({
+        msgtype: 'progress',
+        progressName,
+        status: 'done',
+      });
     }
-    lastStream.pipe(output);
-
-    //  if (output.requireFixedStructure) {
-    //   const ensureHeader = new EnsureStreamHeaderStream();
-    //   input.pipe(ensureHeader);
-    //   ensureHeader.pipe(output);
-    // } else {
-    //   input.pipe(output);
-    // }
-  });
+  } catch (err) {
+    process.send({
+      msgtype: 'copyStreamError',
+      copyStreamError: {
+        message: extractErrorMessage(err),
+        ...err,
+      },
+    });
+
+    if (progressName) {
+      process.send({
+        msgtype: 'progress',
+        progressName,
+        status: 'error',
+        errorMessage: extractErrorMessage(err),
+      });
+    }
+
+    logger.error(extractErrorLogData(err, { progressName }), 'Import/export job failed');
+    // throw err;
+  }
 }
 
 module.exports = copyStream;

package/src/shell/dataDuplicator.js

@@ -24,8 +24,6 @@ async function dataDuplicator({
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
 
   try {
-    logger.info(`Connected.`);
-
     if (!analysedStructure) {
       analysedStructure = await driver.analyseFull(dbhan);
     }

package/src/shell/dropAllDbObjects.js

@@ -19,8 +19,6 @@ async function dropAllDbObjects({ connection, systemConnection, driver, analysed
 
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
 
-  logger.info(`Connected.`);
-
   if (!analysedStructure) {
     analysedStructure = await driver.analyseFull(dbhan);
   }

package/src/shell/executeQuery.js

@@ -36,7 +36,7 @@ async function executeQuery({
   }
 
   try {
-    logger.info(`Connected.`);
+    logger.debug(`Running SQL query, length: ${sql.length}`);
 
     await driver.script(dbhan, sql, { logScriptItems });
   } finally {

package/src/shell/importDatabase.js

@@ -5,6 +5,7 @@ const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream
 const download = require('./download');
 const stream = require('stream');
 const { getLogger } = require('dbgate-tools');
+const streamPipeline = require('../utility/streamPipeline');
 
 const logger = getLogger('importDb');
 
@@ -43,25 +44,12 @@ class ImportStream extends stream.Transform {
   }
 }
 
-function awaitStreamEnd(stream) {
-  return new Promise((resolve, reject) => {
-    stream.once('end', () => {
-      resolve(true);
-    });
-    stream.once('error', err => {
-      reject(err);
-    });
-  });
-}
-
 async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
   logger.info(`Importing database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
   try {
-    logger.info(`Connected.`);
-
     logger.info(`Input file: ${inputFile}`);
     const downloadedFile = await download(inputFile);
     logger.info(`Downloaded file: ${downloadedFile}`);
@@ -72,9 +60,8 @@ async function importDatabase({ connection = undefined, systemConnection = undef
       returnRichInfo: true,
     });
     const importStream = new ImportStream(dbhan, driver);
-    // @ts-ignore
-    splittedStream.pipe(importStream);
-    await awaitStreamEnd(importStream);
+
+    await streamPipeline(splittedStream, importStream);
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

package/src/shell/importDbFromFolder.js

@@ -0,0 +1,125 @@
+const path = require('path');
+const fs = require('fs-extra');
+const executeQuery = require('./executeQuery');
+const { connectUtility } = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const { getAlterDatabaseScript, DatabaseAnalyser, runCommandOnDriver } = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const jsonLinesReader = require('./jsonLinesReader');
+const tableWriter = require('./tableWriter');
+const copyStream = require('./copyStream');
+
+/**
+ * Deploys database model stored in modelFolder (table as yamls) to database
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {string} options.folder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
+ * @param {function[]} options.modelTransforms - array of functions for transforming model
+ */
+async function importDbFromFolder({ connection, systemConnection, driver, folder, modelTransforms }) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    if (driver?.databaseEngineTypes?.includes('sql')) {
+      const model = await importDbModel(folder);
+
+      let modelAdapted = {
+        ...model,
+        tables: model.tables.map(table => driver.adaptTableInfo(table)),
+      };
+      for (const transform of modelTransforms || []) {
+        modelAdapted = transform(modelAdapted);
+      }
+
+      const modelNoFk = {
+        ...modelAdapted,
+        tables: modelAdapted.tables.map(table => ({
+          ...table,
+          foreignKeys: [],
+        })),
+      };
+
+      // const plan = createAlterDatabasePlan(
+      //   DatabaseAnalyser.createEmptyStructure(),
+      //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      //   {},
+      //   DatabaseAnalyser.createEmptyStructure(),
+      //   driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+      //   driver
+      // );
+      // const dmp1 = driver.createDumper({ useHardSeparator: true });
+      // if (driver.dialect.enableAllForeignKeys) {
+      //   dmp1.enableAllForeignKeys(false);
+      // }
+      // plan.run(dmp1);
+      // if (driver.dialect.enableAllForeignKeys) {
+      //   dmp1.enableAllForeignKeys(true);
+      // }
+
+      const { sql } = getAlterDatabaseScript(
+        DatabaseAnalyser.createEmptyStructure(),
+        driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+        {},
+        DatabaseAnalyser.createEmptyStructure(),
+        driver.dialect.enableAllForeignKeys ? modelAdapted : modelNoFk,
+        driver
+      );
+      // console.log('CREATING STRUCTURE:', sql);
+      await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+      if (driver.dialect.enableAllForeignKeys) {
+        await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(false));
+      }
+
+      for (const table of modelAdapted.tables) {
+        const fileName = path.join(folder, `${table.pureName}.jsonl`);
+        if (await fs.exists(fileName)) {
+          const src = await jsonLinesReader({ fileName });
+          const dst = await tableWriter({
+            systemConnection: dbhan,
+            pureName: table.pureName,
+            driver,
+            targetTableStructure: table,
+          });
+          await copyStream(src, dst);
+        }
+      }
+
+      if (driver.dialect.enableAllForeignKeys) {
+        await runCommandOnDriver(dbhan, driver, dmp => dmp.enableAllForeignKeys(true));
+      } else if (driver.dialect.createForeignKey) {
+        const dmp = driver.createDumper();
+        for (const table of modelAdapted.tables) {
+          for (const fk of table.foreignKeys) {
+            dmp.createForeignKey(fk);
+          }
+        }
+
+        // create foreign keys
+        await executeQuery({ connection, systemConnection: dbhan, driver, sql: dmp.s, logScriptItems: true });
+      }
+    } else if (driver?.databaseEngineTypes?.includes('document')) {
+      for (const file of fs.readdirSync(folder)) {
+        if (!file.endsWith('.jsonl')) continue;
+        const pureName = path.parse(file).name;
+        const src = await jsonLinesReader({ fileName: path.join(folder, file) });
+        const dst = await tableWriter({
+          systemConnection: dbhan,
+          pureName,
+          driver,
+          createIfNotExists: true,
+        });
+        await copyStream(src, dst);
+      }
+    }
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = importDbFromFolder;

package/src/shell/index.js

@@ -21,7 +21,6 @@ const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
 const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
-const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
 const loadDatabase = require('./loadDatabase');
 const generateModelSql = require('./generateModelSql');
@@ -35,6 +34,7 @@ const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
 const generateDeploySql = require('./generateDeploySql');
 const dropAllDbObjects = require('./dropAllDbObjects');
+const importDbFromFolder = require('./importDbFromFolder');
 
 const dbgateApi = {
   queryReader,
@@ -60,7 +60,6 @@ const dbgateApi = {
   loadFile,
   deployDb,
   initializeApiEnvironment,
-  dumpDatabase,
   importDatabase,
   loadDatabase,
   generateModelSql,
@@ -73,6 +72,7 @@ const dbgateApi = {
   autoIndexForeignKeysTransform,
   generateDeploySql,
   dropAllDbObjects,
+  importDbFromFolder,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/jsonLinesReader.js

@@ -53,8 +53,7 @@ async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undef
   );
   const liner = byline(fileStream);
   const parser = new ParseStream({ limitRows });
-  liner.pipe(parser);
-  return parser;
+  return [liner, parser];
 }
 
 module.exports = jsonLinesReader;

package/src/shell/jsonReader.js

@@ -10,7 +10,6 @@ const download = require('./download');
 
 const logger = getLogger('jsonReader');
 
-
 class ParseStream extends stream.Transform {
   constructor({ limitRows, jsonStyle, keyField }) {
     super({ objectMode: true });
@@ -72,8 +71,12 @@ async function jsonReader({
     // @ts-ignore
     encoding
   );
+
   const parseJsonStream = parser();
-  fileStream.pipe(parseJsonStream);
+
+  const resultPipe = [fileStream, parseJsonStream];
+
+  // fileStream.pipe(parseJsonStream);
 
   const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
 
@@ -81,15 +84,20 @@ async function jsonReader({
 
   if (rootField) {
     const filterStream = pick({ filter: rootField });
-    parseJsonStream.pipe(filterStream);
-    filterStream.pipe(tramsformer);
-  } else {
-    parseJsonStream.pipe(tramsformer);
+    resultPipe.push(filterStream);
+    // parseJsonStream.pipe(filterStream);
+    // filterStream.pipe(tramsformer);
   }
+  // else {
+  //   parseJsonStream.pipe(tramsformer);
+  // }
+
+  resultPipe.push(tramsformer);
+  resultPipe.push(parseStream);
 
-  tramsformer.pipe(parseStream);
+  // tramsformer.pipe(parseStream);
 
-  return parseStream;
+  return resultPipe;
 }
 
 module.exports = jsonReader;

package/src/shell/jsonWriter.js

@@ -99,9 +99,10 @@ async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, e
   logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
   const fileStream = fs.createWriteStream(fileName, encoding);
-  stringify.pipe(fileStream);
-  stringify['finisher'] = fileStream;
-  return stringify;
+  return [stringify, fileStream];
+  // stringify.pipe(fileStream);
+  // stringify['finisher'] = fileStream;
+  // return stringify;
 }
 
 module.exports = jsonWriter;

package/src/shell/loadDatabase.js

@@ -6,15 +6,13 @@ const exportDbModel = require('../utility/exportDbModel');
 const logger = getLogger('analyseDb');
 
 async function loadDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, outputDir }) {
-  logger.info(`Analysing database`);
+  logger.debug(`Analysing database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
   try {
-    logger.info(`Connected.`);
-
     const dbInfo = await driver.analyseFull(dbhan);
-    logger.info(`Analyse finished`);
+    logger.debug(`Analyse finished`);
 
     await exportDbModel(dbInfo, outputDir);
   } finally {

package/src/shell/modifyJsonLinesReader.js

@@ -141,8 +141,9 @@ async function modifyJsonLinesReader({
   );
   const liner = byline(fileStream);
   const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode });
-  liner.pipe(parser);
-  return parser;
+  return [liner, parser];
+  // liner.pipe(parser);
+  // return parser;
 }
 
 module.exports = modifyJsonLinesReader;

package/src/shell/queryReader.js

@@ -30,7 +30,6 @@ async function queryReader({
 
   const driver = requireEngineDriver(connection);
   const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
-  logger.info(`Connected.`);
   const reader =
     queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
   return reader;

package/src/shell/sqlDataWriter.js

@@ -44,9 +44,10 @@ async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' })
   logger.info(`Writing file ${fileName}`);
   const stringify = new SqlizeStream({ fileName, dataName });
   const fileStream = fs.createWriteStream(fileName, encoding);
-  stringify.pipe(fileStream);
-  stringify['finisher'] = fileStream;
-  return stringify;
+  return [stringify, fileStream];
+  // stringify.pipe(fileStream);
+  // stringify['finisher'] = fileStream;
+  // return stringify;
 }
 
 module.exports = sqlDataWriter;

package/src/shell/tableReader.js

@@ -18,7 +18,6 @@ async function tableReader({ connection, systemConnection, pureName, schemaName,
     driver = requireEngineDriver(connection);
   }
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
-  logger.info(`Connected.`);
 
   const fullName = { pureName, schemaName };
 

package/src/shell/tableWriter.js

@@ -15,6 +15,8 @@ const logger = getLogger('tableWriter');
  * @param {boolean} options.truncate - truncate table before insert
  * @param {boolean} options.createIfNotExists - create table if not exists
  * @param {boolean} options.commitAfterInsert - commit transaction after insert
+ * @param {string} options.progressName - name for reporting progress
+ * @param {any} options.targetTableStructure - target table structure (don't analyse if given)
  * @returns {Promise<writerType>} - writer object
  */
 async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
@@ -25,8 +27,20 @@ async function tableWriter({ connection, schemaName, pureName, driver, systemCon
   }
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
 
-  logger.info(`Connected.`);
-  return await driver.writeTable(dbhan, { schemaName, pureName }, options);
+  try {
+    return await driver.writeTable(dbhan, { schemaName, pureName }, options);
+  } catch (err) {
+    if (options.progressName) {
+      process.send({
+        msgtype: 'progress',
+        progressName: options.progressName,
+        status: 'error',
+        errorMessage: err.message,
+      });
+    }
+
+    throw err;
+  }
 }
 
 module.exports = tableWriter;

package/src/utility/crypting.js

@@ -88,6 +88,9 @@ function decryptConnection(connection) {
 }
 
 function pickSafeConnectionInfo(connection) {
+  if (process.env.LOG_CONNECTION_SENSITIVE_VALUES) {
+    return connection;
+  }
   return _.mapValues(connection, (v, k) => {
     if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
     if (v === null || v === true || v === false) return v;

package/src/utility/healthStatus.js

@@ -0,0 +1,27 @@
+const os = require('os');
+
+const databaseConnections = require('../controllers/databaseConnections');
+const serverConnections = require('../controllers/serverConnections');
+const sessions = require('../controllers/sessions');
+const runners = require('../controllers/runners');
+
+async function getHealthStatus() {
+  const memory = process.memoryUsage();
+  const cpuUsage = process.cpuUsage();
+
+  return {
+    status: 'ok',
+    databaseConnectionCount: databaseConnections.opened.length,
+    serverConnectionCount: serverConnections.opened.length,
+    sessionCount: sessions.opened.length,
+    runProcessCount: runners.opened.length,
+    memory,
+    cpuUsage,
+    systemMemory: {
+      total: os.totalmem(),
+      free: os.freemem(),
+    },
+  };
+}
+
+module.exports = getHealthStatus;

package/src/utility/streamPipeline.js

@@ -0,0 +1,18 @@
+const stream = require('stream');
+const _ = require('lodash');
+
+function streamPipeline(...processedStreams) {
+  const streams = _.flattenDeep(processedStreams);
+  return new Promise((resolve, reject) => {
+    // @ts-ignore
+    stream.pipeline(...streams, err => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+module.exports = streamPipeline;

package/src/nativeModulesContent.js

@@ -1,9 +0,0 @@
-
-// this file is generated automatically by script fillNativeModules.js, do not edit it manually
-const content = {};
-
-content['better-sqlite3'] = () => require('better-sqlite3');
-content['oracledb'] = () => require('oracledb');
-
-
-module.exports = content;

package/src/shell/dumpDatabase.js

@@ -1,49 +0,0 @@
-const requireEngineDriver = require('../utility/requireEngineDriver');
-const { connectUtility } = require('../utility/connectUtility');
-const { getLogger } = require('dbgate-tools');
-
-const logger = getLogger('dumpDb');
-
-function doDump(dumper) {
-  return new Promise((resolve, reject) => {
-    dumper.once('end', () => {
-      resolve(true);
-    });
-    dumper.once('error', err => {
-      reject(err);
-    });
-    dumper.run();
-  });
-}
-
-async function dumpDatabase({
-  connection = undefined,
-  systemConnection = undefined,
-  driver = undefined,
-  outputFile,
-  databaseName,
-  schemaName,
-}) {
-  logger.info(`Dumping database`);
-
-  if (!driver) driver = requireEngineDriver(connection);
-
-  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
-
-  try {
-    logger.info(`Connected.`);
-
-    const dumper = await driver.createBackupDumper(dbhan, {
-      outputFile,
-      databaseName,
-      schemaName,
-    });
-    await doDump(dumper);
-  } finally {
-    if (!systemConnection) {
-      await driver.close(dbhan);
-    }
-  }
-}
-
-module.exports = dumpDatabase;