dbgate-api-premium 7.0.4 → 7.1.0

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "dbgate-api-premium",
   "main": "src/index.js",
-  "version": "7.0.4",
-  "homepage": "https://dbgate.org/",
+  "version": "7.1.0",
+  "homepage": "https://www.dbgate.io/",
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
@@ -30,10 +30,11 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^7.0.4",
+    "dbgate-datalib": "^7.1.0",
     "dbgate-query-splitter": "^4.11.9",
-    "dbgate-sqltree": "^7.0.4",
-    "dbgate-tools": "^7.0.4",
+    "dbgate-rest": "^7.1.0",
+    "dbgate-sqltree": "^7.1.0",
+    "dbgate-tools": "^7.1.0",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -87,7 +88,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^7.0.4",
+    "dbgate-types": "^7.1.0",
     "env-cmd": "^10.1.0",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",

package/src/controllers/auth.js

@@ -55,6 +55,8 @@ function authMiddleware(req, res, next) {
     '/stream',
     '/storage/get-connections-for-login-page',
     '/storage/set-admin-password',
+    '/storage/request-password-reset',
+    '/storage/reset-password',
     '/auth/get-providers',
     '/connections/dblogin-web',
     '/connections/dblogin-app',

package/src/controllers/connections.js

@@ -202,10 +202,10 @@ module.exports = {
 
     const storageConnections = await storage.connections(req);
     if (storageConnections) {
-      return storageConnections;
+      return storageConnections.map(maskConnection);
     }
     if (portalConnections) {
-      if (platformInfo.allowShellConnection) return portalConnections;
+      if (platformInfo.allowShellConnection) return portalConnections.map(x => encryptConnection(x));
       return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, loadedPermissions));
     }
     return (await this.datastore.find()).filter(x => connectionHasPermission(x, loadedPermissions));
@@ -243,13 +243,13 @@ module.exports = {
     subprocess.send({ ...connection, requestDbList });
     return new Promise((resolve, reject) => {
       let isWaitingForVolatile = false;
-      
+
       const cleanup = () => {
         if (connection._id && pendingTestSubprocesses[connection._id]) {
           delete pendingTestSubprocesses[connection._id];
         }
       };
-      
+
       subprocess.on('message', resp => {
         if (handleProcessCommunication(resp, subprocess)) return;
         // @ts-ignore
@@ -279,8 +279,8 @@ module.exports = {
           reject(new MissingCredentialsError(missingCredentialsDetail));
         }
       });
-      
-      subprocess.on('exit', (code) => {
+
+      subprocess.on('exit', code => {
         // If exit happens while waiting for volatile, that's expected
         if (isWaitingForVolatile && code === 0) {
           cleanup();
@@ -291,8 +291,8 @@ module.exports = {
           reject(new Error(`Test subprocess exited with code ${code}`));
         }
       });
-      
-      subprocess.on('error', (err) => {
+
+      subprocess.on('error', err => {
         cleanup();
         reject(err);
       });
@@ -327,7 +327,7 @@ module.exports = {
       return testRes;
     } else {
       volatileConnections[res._id] = res;
-      
+
       // Check if there's a pending test subprocess waiting for this volatile connection
       const pendingTest = pendingTestSubprocesses[conid];
       if (pendingTest) {
@@ -335,7 +335,7 @@ module.exports = {
         try {
           // Send the volatile connection to the waiting subprocess
           subprocess.send({ ...res, requestDbList, isVolatileResolved: true });
-          
+
           // Wait for the test result and emit it as an event
           subprocess.once('message', resp => {
             if (handleProcessCommunication(resp, subprocess)) return;
@@ -358,7 +358,7 @@ module.exports = {
           delete pendingTestSubprocesses[conid];
         }
       }
-      
+
       return res;
     }
   },
@@ -484,12 +484,12 @@ module.exports = {
 
     const storageConnection = await storage.getConnection({ conid });
     if (storageConnection) {
-      return storageConnection;
+      return mask ? maskConnection(storageConnection) : storageConnection;
     }
 
     if (portalConnections) {
       const res = portalConnections.find(x => x._id == conid) || null;
-      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : encryptConnection(res);
     }
     const res = await this.datastore.get(conid);
     return res || null;

package/src/controllers/databaseConnections.js

@@ -165,6 +165,11 @@ module.exports = {
     if (!connection) {
       throw new Error(`databaseConnections: Connection with conid="${conid}" not found`);
     }
+
+    if (connection.engine?.endsWith('@rest')) {
+      return { isApiConnection: true };
+    }
+
     if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
       throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
     }

package/src/controllers/restConnections.js

@@ -0,0 +1,316 @@
+// *** This file is part of DbGate Premium ***
+
+const crypto = require('crypto');
+const connections = require('./connections');
+const socket = require('../utility/socket');
+const { fork } = require('child_process');
+const _ = require('lodash');
+const AsyncLock = require('async-lock');
+const { handleProcessCommunication } = require('../utility/processComm');
+const lock = new AsyncLock();
+const config = require('./config');
+const processArgs = require('../utility/processArgs');
+const { testConnectionPermission, loadPermissionsFromRequest, hasPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { sendToAuditLog } = require('../utility/auditlog');
+const { decryptPasswordString } = require('../utility/crypting');
+const { getRestAuthFromConnection } = require('../utility/connectUtility');
+
+const logger = getLogger('restConnection');
+
+module.exports = {
+  opened: [],
+  closed: {},
+  requests: {},
+
+  handle_apiInfo(conid, { apiInfo }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.apiInfo = apiInfo;
+    socket.emitChanged(`rest-api-info-changed`, { conid });
+  },
+
+  handle_status(conid, { status }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
+    existing.status = status;
+    socket.emitChanged(`rest-status-changed`, { conid });
+  },
+
+  handle_ping() {},
+
+  handle_response(conid, { msgid, ...response }) {
+    const pending = this.requests[msgid];
+    if (!pending) {
+      logger.warn(
+        `DBGM-00275 restConnections: Received response for unknown or already handled msgid="${msgid}" (conid="${conid}")`
+      );
+      return;
+    }
+    const [resolve, reject] = pending;
+    resolve(response);
+    delete this.requests[msgid];
+  },
+
+  async ensureOpened(conid) {
+    const res = await lock.acquire(conid, async () => {
+      const existing = this.opened.find(x => x.conid == conid);
+      if (existing) return existing;
+
+      const connection = await connections.getCore({ conid });
+      if (!connection) {
+        throw new Error(`restConnections: Connection with conid="${conid}" not found`);
+      }
+
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
+
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        ['--is-forked-api', '--start-process', 'restConnectionProcess', ...processArgs.getPassArgs()],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+
+      pipeForkLogs(subprocess);
+
+      const newOpened = {
+        conid,
+        subprocess,
+        apiInfo: null,
+        connection,
+        status: {
+          name: 'pending',
+        },
+        disconnected: false,
+      };
+
+      this.opened.push(newOpened);
+      delete this.closed[conid];
+      socket.emitChanged(`rest-status-changed`, { conid });
+
+      subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, subprocess)) return;
+        if (newOpened.disconnected) return;
+        const funcName = `handle_${msgtype}`;
+        if (!this[funcName]) {
+          logger.error({ msgtype, conid }, 'DBGM-00276 Unknown message type from subprocess restConnectionProcess');
+          return;
+        }
+        this[funcName](conid, message);
+      });
+
+      subprocess.on('exit', () => {
+        if (newOpened.disconnected) return;
+        this.close(conid, false);
+      });
+
+      subprocess.on('error', err => {
+        logger.error(extractErrorLogData(err), 'DBGM-00277 Error in REST connection subprocess');
+        if (newOpened.disconnected) return;
+        this.close(conid, false);
+      });
+
+      subprocess.send({
+        msgtype: 'connect',
+        connection: { ...connection, restAuth: getRestAuthFromConnection(connection) },
+        globalSettings: await config.getSettings(),
+      });
+      return newOpened;
+    });
+    return res;
+  },
+
+  close(conid, kill = true) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (existing) {
+      existing.disconnected = true;
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error(extractErrorLogData(err), 'DBGM-00278 Error killing REST subprocess');
+        }
+      }
+      this.opened = this.opened.filter(x => x.conid != conid);
+      this.closed[conid] = {
+        ...existing.status,
+        name: 'error',
+      };
+      socket.emitChanged(`rest-status-changed`, { conid });
+    }
+  },
+
+  disconnect_meta: true,
+  async disconnect({ conid }, req) {
+    await testConnectionPermission(conid, req);
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
+  getApiInfo_meta: true,
+  async getApiInfo({ conid }, req) {
+    if (!conid) return null;
+
+    await testConnectionPermission(conid, req);
+
+    const opened = await this.ensureOpened(conid);
+    sendToAuditLog(req, {
+      category: 'restop',
+      component: 'RestConnectionsController',
+      action: 'getApiInfo',
+      event: 'rest.getApiInfo',
+      severity: 'info',
+      conid,
+      sessionParam: `${conid}`,
+      sessionGroup: 'getApiInfo',
+      message: `Loaded API info for REST connection`,
+    });
+
+    return opened?.apiInfo ?? null;
+  },
+
+  restStatus_meta: true,
+  async restStatus() {
+    return {
+      ...this.closed,
+      ..._.mapValues(_.keyBy(this.opened, 'conid'), 'status'),
+    };
+  },
+
+  ping_meta: true,
+  async ping({ conidArray, strmid }) {
+    await Promise.all(
+      _.uniq(conidArray).map(async conid => {
+        try {
+          const opened = await this.ensureOpened(conid);
+          if (!opened) {
+            return Promise.resolve();
+          }
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error(extractErrorLogData(err), 'DBGM-00279 Error pinging REST connection');
+          this.close(conid);
+        }
+      })
+    );
+    socket.setStreamIdFilter(strmid, { conid: conidArray ?? [] });
+    return { status: 'ok' };
+  },
+
+  refresh_meta: true,
+  async refresh({ conid, keepOpen }, req) {
+    await testConnectionPermission(conid, req);
+    if (!keepOpen) this.close(conid);
+
+    const opened = await this.ensureOpened(conid);
+    return this.sendRequest(opened, { msgtype: 'refresh' });
+  },
+
+  testConnection_meta: true,
+  async testConnection({ conid }, req) {
+    await testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return { errorMessage: 'Could not open REST connection' };
+    }
+    const res = await this.sendRequest(opened, { msgtype: 'testConnection' });
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return { status: 'ok' };
+  },
+
+  async executeCore(msgtype, { conid, method, endpoint, parameters, server }, req) {
+    await testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return { errorMessage: 'Could not open REST connection' };
+    }
+    const auth = getRestAuthFromConnection(opened.connection);
+    return this.sendRequest(opened, {
+      msgtype,
+      method,
+      endpoint,
+      parameters,
+      server,
+      auth,
+    });
+  },
+
+  executeOpenapi_meta: true,
+  async executeOpenapi({ conid, method, endpoint, parameters, server }, req) {
+    return this.executeCore('executeOpenapi', { conid, method, endpoint, parameters, server }, req);
+  },
+
+  executeOdata_meta: true,
+  async executeOdata({ conid, method, endpoint, parameters, server }, req) {
+    return this.executeCore('executeOdata', { conid, method, endpoint, parameters, server }, req);
+  },
+
+  apiQuery_meta: true,
+  async apiQuery({ conid, server, query, variables }, req) {
+    await testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return { errorMessage: 'Could not open REST connection' };
+    }
+    const auth = getRestAuthFromConnection(opened.connection);
+    const res = await this.sendRequest(opened, {
+      msgtype: 'apiQuery',
+      server,
+      query,
+      variables,
+      auth,
+    });
+    return res;
+  },
+
+  loadGraphqlConnectionData_meta: true,
+  async loadGraphqlConnectionData(
+    { conid, server, operationName, projection, queryField, selectedFieldSelection, pageSize, filterParameterName, filterValue },
+    req
+  ) {
+    await testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return { errorMessage: 'Could not open REST connection' };
+    }
+    const auth = getRestAuthFromConnection(opened.connection);
+    return this.sendRequest(opened, {
+      msgtype: 'loadGraphqlConnectionData',
+      server,
+      operationName,
+      projection,
+      queryField,
+      selectedFieldSelection,
+      pageSize,
+      filterParameterName,
+      filterValue,
+      auth,
+    });
+  },
+
+  sendRequest(conn, message) {
+    const msgid = crypto.randomUUID();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'DBGM-00280 Error sending request to REST connection');
+        this.close(conn.conid);
+      }
+    });
+    return promise;
+  },
+};

package/src/controllers/runners.js

@@ -172,7 +172,7 @@ module.exports = {
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
       // console.log('... EXITED', code);
-      this.rejectRequest(runid, { message: 'No data returned, maybe input data source is too big' });
+      this.rejectRequest(runid, { message: 'DBGM-00281 No data returned, maybe input data source is too big' });
       logger.info({ code, pid: subprocess.pid }, 'DBGM-00016 Exited process');
       socket.emit(`runner-done-${runid}`, code);
       this.opened = this.opened.filter(x => x.runid != runid);
@@ -225,7 +225,7 @@ module.exports = {
     subprocess.on('exit', code => {
       console.log('... EXITED', code);
       logger.info({ code, pid: subprocess.pid }, 'DBGM-00017 Exited process');
-      this.dispatchMessage(runid, `Finished external process with code ${code}`);
+      this.dispatchMessage(runid, `DBGM-00282 Finished external process with code ${code}`);
       socket.emit(`runner-done-${runid}`, code);
       if (onFinished) {
         onFinished();
@@ -233,7 +233,7 @@ module.exports = {
       this.opened = this.opened.filter(x => x.runid != runid);
     });
     subprocess.on('spawn', () => {
-      this.dispatchMessage(runid, `Started external process ${command}`);
+      this.dispatchMessage(runid, `DBGM-00283 Started external process ${command}`);
     });
     subprocess.on('error', error => {
       console.log('... ERROR subprocess', error);
@@ -279,7 +279,7 @@ module.exports = {
     if (script.type == 'json') {
       if (!platformInfo.isElectron) {
         if (!checkSecureDirectoriesInScript(script)) {
-          return { errorMessage: 'Unallowed directories in script' };
+          return { errorMessage: 'DBGM-00284 Unallowed directories in script' };
         }
       }
 
@@ -299,10 +299,10 @@ module.exports = {
         action: 'script',
         severity: 'warn',
         detail: script,
-        message: 'Scripts are not allowed',
+        message: 'DBGM-00285 Scripts are not allowed',
       });
 
-      return { errorMessage: 'Shell scripting is not allowed' };
+      return { errorMessage: 'DBGM-00286 Shell scripting is not allowed' };
     }
 
     sendToAuditLog(req, {
@@ -312,7 +312,7 @@ module.exports = {
       action: 'script',
       severity: 'info',
       detail: script,
-      message: 'Running JS script',
+      message: 'DBGM-00287 Running JS script',
     });
 
     return this.startCore(runid, scriptTemplate(script, false));
@@ -327,7 +327,7 @@ module.exports = {
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
-      throw new Error('Invalid runner');
+      throw new Error('DBGM-00288 Invalid runner');
     }
     runner.subprocess.kill();
     return { state: 'ok' };
@@ -353,7 +353,7 @@ module.exports = {
   async loadReader({ functionName, props }) {
     if (!platformInfo.isElectron) {
       if (props?.fileName && !checkSecureDirectories(props.fileName)) {
-        return { errorMessage: 'Unallowed file' };
+        return { errorMessage: 'DBGM-00289 Unallowed file' };
       }
     }
     const prefix = extractShellApiPlugins(functionName)
@@ -371,7 +371,7 @@ module.exports = {
   scriptResult_meta: true,
   async scriptResult({ script }) {
     if (script.type != 'json') {
-      return { errorMessage: 'Only JSON scripts are allowed' };
+      return { errorMessage: 'DBGM-00290 Only JSON scripts are allowed' };
     }
 
     const promise = new Promise(async (resolve, reject) => {