dbgate-api-premium 6.5.4 → 6.5.6

package/src/controllers/config.js

@@ -16,7 +16,7 @@ const connections = require('../controllers/connections');
 const { getAuthProviderFromReq } = require('../auth/authProvider');
 const { checkLicense, checkLicenseKey } = require('../utility/checkLicense');
 const storage = require('./storage');
-const { getAuthProxyUrl } = require('../utility/authProxy');
+const { getAuthProxyUrl, tryToGetRefreshedLicense } = require('../utility/authProxy');
 const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
 const { extractErrorMessage } = require('dbgate-tools');
 const {
@@ -29,6 +29,7 @@ const {
 } = require('../utility/crypting');
 
 const lock = new AsyncLock();
+let cachedSettingsValue = null;
 
 module.exports = {
   // settingsValue: {},
@@ -108,6 +109,7 @@ module.exports = {
       ),
       isAdminPasswordMissing,
       isInvalidToken: req?.isInvalidToken,
+      skipAllAuth: !!process.env.SKIP_ALL_AUTH,
       adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
@@ -118,6 +120,7 @@ module.exports = {
       supportCloudAutoUpgrade: !!process.env.CLOUD_UPGRADE_FILE,
       allowPrivateCloud: platformInfo.isElectron || !!process.env.ALLOW_DBGATE_PRIVATE_CLOUD,
       ...currentVersion,
+      redirectToDbGateCloudLogin: !!process.env.REDIRECT_TO_DBGATE_CLOUD_LOGIN,
     };
 
     return configResult;
@@ -144,6 +147,13 @@ module.exports = {
     return res;
   },
 
+  async getCachedSettings() {
+    if (!cachedSettingsValue) {
+      cachedSettingsValue = await this.loadSettings();
+    }
+    return cachedSettingsValue;
+  },
+
   deleteSettings_meta: true,
   async deleteSettings() {
     await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'));
@@ -182,6 +192,7 @@ module.exports = {
         return {
           ...this.fillMissingSettings(JSON.parse(settingsText)),
           'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
+          // 'other.licenseKey': await this.loadLicenseKey(),
         };
       }
     } catch (err) {
@@ -199,21 +210,34 @@ module.exports = {
   },
 
   saveLicenseKey_meta: true,
-  async saveLicenseKey({ licenseKey }) {
-    const decoded = jwt.decode(licenseKey?.trim());
-    if (!decoded) {
-      return {
-        status: 'error',
-        errorMessage: 'Invalid license key',
-      };
-    }
+  async saveLicenseKey({ licenseKey, forceSave = false, tryToRenew = false }) {
+    if (!forceSave) {
+      const decoded = jwt.decode(licenseKey?.trim());
+      if (!decoded) {
+        return {
+          status: 'error',
+          errorMessage: 'Invalid license key',
+        };
+      }
 
-    const { exp } = decoded;
-    if (exp * 1000 < Date.now()) {
-      return {
-        status: 'error',
-        errorMessage: 'License key is expired',
-      };
+      const { exp } = decoded;
+      if (exp * 1000 < Date.now()) {
+        let renewed = false;
+        if (tryToRenew) {
+          const newLicenseKey = await tryToGetRefreshedLicense(licenseKey);
+          if (newLicenseKey.status == 'ok') {
+            licenseKey = newLicenseKey.token;
+            renewed = true;
+          }
+        }
+
+        if (!renewed) {
+          return {
+            status: 'error',
+            errorMessage: 'License key is expired',
+          };
+        }
+      }
     }
 
     try {
@@ -257,6 +281,7 @@ module.exports = {
   updateSettings_meta: true,
   async updateSettings(values, req) {
     if (!hasPermission(`settings/change`, req)) return false;
+    cachedSettingsValue = null;
 
     const res = await lock.acquire('settings', async () => {
       const currentValue = await this.loadSettings();
@@ -265,7 +290,11 @@ module.exports = {
         if (process.env.STORAGE_DATABASE) {
           updated = {
             ...currentValue,
-            ...values,
+            ..._.mapValues(values, v => {
+              if (v === true) return 'true';
+              if (v === false) return 'false';
+              return v;
+            }),
           };
           await storage.writeConfig({
             group: 'settings',
@@ -283,7 +312,7 @@ module.exports = {
           // this.settingsValue = updated;
 
           if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
-            await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
+            await this.saveLicenseKey({ licenseKey: values['other.licenseKey'], forceSave: true });
             socket.emitChanged(`config-changed`);
           }
         }
@@ -303,7 +332,7 @@ module.exports = {
       const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
       return resp.data;
     } catch (err) {
-      return ''
+      return '';
     }
   },
 
@@ -313,6 +342,16 @@ module.exports = {
     return resp;
   },
 
+  getNewLicense_meta: true,
+  async getNewLicense({ oldLicenseKey }) {
+    const newLicenseKey = await tryToGetRefreshedLicense(oldLicenseKey);
+    const res = await checkLicenseKey(newLicenseKey.token);
+    if (res.status == 'ok') {
+      res.licenseKey = newLicenseKey.token;
+    }
+    return res;
+  },
+
   recryptDatabaseForExport(db) {
     const encryptionKey = generateTransportEncryptionKey();
     const transportEncryptor = createTransportEncryptor(encryptionKey);

package/src/controllers/connections.js

@@ -536,14 +536,14 @@ module.exports = {
   },
 
   dbloginAuthToken_meta: true,
-  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) {
+  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }, req) {
     try {
       const connection = await this.getCore({ conid });
       const driver = requireEngineDriver(connection);
       const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid });
       const volatile = await this.saveVolatile({ conid, accessToken });
       const authProvider = getAuthProviderById(amoid);
-      const resp = await authProvider.login(null, null, { conid: volatile._id });
+      const resp = await authProvider.login(null, null, { conid: volatile._id }, req);
       return resp;
     } catch (err) {
       logger.error(extractErrorLogData(err), 'Error getting DB token');
@@ -552,18 +552,18 @@ module.exports = {
   },
 
   dbloginAuth_meta: true,
-  async dbloginAuth({ amoid, conid, user, password }) {
+  async dbloginAuth({ amoid, conid, user, password }, req) {
     if (user || password) {
       const saveResp = await this.saveVolatile({ conid, user, password, test: true });
       if (saveResp.msgtype == 'connected') {
-        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id });
+        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }, req);
         return loginResp;
       }
       return saveResp;
     }
 
     // user and password is stored in connection, volatile connection is not needed
-    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid });
+    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }, req);
     return loginResp;
   },
 

package/src/controllers/databaseConnections.js

@@ -41,6 +41,7 @@ const { decryptConnection } = require('../utility/crypting');
 const { getSshTunnel } = require('../utility/sshTunnel');
 const sessions = require('./sessions');
 const jsldata = require('./jsldata');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('databaseConnections');
 
@@ -83,8 +84,11 @@ module.exports = {
     }
   },
   handle_response(conid, database, { msgid, ...response }) {
-    const [resolve, reject] = this.requests[msgid];
+    const [resolve, reject, additionalData] = this.requests[msgid];
     resolve(response);
+    if (additionalData?.auditLogger) {
+      additionalData?.auditLogger(response);
+    }
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
@@ -215,10 +219,10 @@ module.exports = {
   },
 
   /** @param {import('dbgate-types').OpenedDatabaseConnection} conn */
-  sendRequest(conn, message) {
+  sendRequest(conn, message, additionalData = {}) {
     const msgid = crypto.randomUUID();
     const promise = new Promise((resolve, reject) => {
-      this.requests[msgid] = [resolve, reject];
+      this.requests[msgid] = [resolve, reject, additionalData];
       try {
         conn.subprocess.send({ msgid, ...message });
       } catch (err) {
@@ -242,18 +246,57 @@ module.exports = {
   },
 
   sqlSelect_meta: true,
-  async sqlSelect({ conid, database, select }, req) {
+  async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    const res = await this.sendRequest(
+      opened,
+      { msgtype: 'sqlSelect', select },
+      {
+        auditLogger:
+          auditLogSessionGroup && select?.from?.name?.pureName
+            ? response => {
+                sendToAuditLog(req, {
+                  category: 'dbop',
+                  component: 'DatabaseConnectionsController',
+                  event: 'sql.select',
+                  action: 'select',
+                  severity: 'info',
+                  conid,
+                  database,
+                  schemaName: select?.from?.name?.schemaName,
+                  pureName: select?.from?.name?.pureName,
+                  sumint1: response?.rows?.length,
+                  sessionParam: `${conid}::${database}::${select?.from?.name?.schemaName || '0'}::${
+                    select?.from?.name?.pureName
+                  }`,
+                  sessionGroup: auditLogSessionGroup,
+                  message: `Loaded table data from ${select?.from?.name?.pureName}`,
+                });
+              }
+            : null,
+      }
+    );
     return res;
   },
 
   runScript_meta: true,
-  async runScript({ conid, database, sql, useTransaction }, req) {
+  async runScript({ conid, database, sql, useTransaction, logMessage }, req) {
     testConnectionPermission(conid, req);
     logger.info({ conid, database, sql }, 'Processing script');
     const opened = await this.ensureOpened(conid, database);
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      event: 'sql.runscript',
+      action: 'runscript',
+      severity: 'info',
+      conid,
+      database,
+      detail: sql,
+      message: logMessage || `Running SQL script`,
+    });
+
     const res = await this.sendRequest(opened, { msgtype: 'runScript', sql, useTransaction });
     return res;
   },
@@ -262,16 +305,53 @@ module.exports = {
   async runOperation({ conid, database, operation, useTransaction }, req) {
     testConnectionPermission(conid, req);
     logger.info({ conid, database, operation }, 'Processing operation');
+
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      event: 'sql.runoperation',
+      action: operation.type,
+      severity: 'info',
+      conid,
+      database,
+      detail: operation,
+      message: `Running DB operation: ${operation.type}`,
+    });
+
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'runOperation', operation, useTransaction });
     return res;
   },
 
   collectionData_meta: true,
-  async collectionData({ conid, database, options }, req) {
+  async collectionData({ conid, database, options, auditLogSessionGroup }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    const res = await this.sendRequest(
+      opened,
+      { msgtype: 'collectionData', options },
+      {
+        auditLogger:
+          auditLogSessionGroup && options?.pureName
+            ? response => {
+                sendToAuditLog(req, {
+                  category: 'dbop',
+                  component: 'DatabaseConnectionsController',
+                  event: 'nosql.collectionData',
+                  action: 'select',
+                  severity: 'info',
+                  conid,
+                  database,
+                  pureName: options?.pureName,
+                  sumint1: response?.result?.rows?.length,
+                  sessionParam: `${conid}::${database}::${options?.pureName}`,
+                  sessionGroup: auditLogSessionGroup,
+                  message: `Loaded collection data ${options?.pureName}`,
+                });
+              }
+            : null,
+      }
+    );
     return res.result || null;
   },
 
@@ -492,6 +572,20 @@ module.exports = {
     }
 
     const opened = await this.ensureOpened(conid, database);
+
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      action: 'structure',
+      event: 'dbStructure.get',
+      severity: 'info',
+      conid,
+      database,
+      sessionParam: `${conid}::${database}`,
+      sessionGroup: 'getStructure',
+      message: `Loaded database structure for ${database}`,
+    });
+
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
     // if (existing) return existing.status;

package/src/controllers/files.js

@@ -203,10 +203,10 @@ module.exports = {
   },
 
   exportChart_meta: true,
-  async exportChart({ filePath, title, config, image }) {
+  async exportChart({ filePath, title, config, image, plugins }) {
     const fileName = path.parse(filePath).base;
     const imageFile = fileName.replace('.html', '-preview.png');
-    const html = getChartExport(title, config, imageFile);
+    const html = getChartExport(title, config, imageFile, plugins);
     await fs.writeFile(filePath, html);
     if (image) {
       const index = image.indexOf('base64,');

package/src/controllers/runners.js

@@ -20,6 +20,7 @@ const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
 const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security');
+const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog');
 const logger = getLogger('runners');
 
 function extractPlugins(script) {
@@ -270,7 +271,7 @@ module.exports = {
   },
 
   start_meta: true,
-  async start({ script }) {
+  async start({ script }, req) {
     const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
@@ -280,14 +281,36 @@ module.exports = {
         }
       }
 
+      logJsonRunnerScript(req, script);
+
       const js = await jsonScriptToJavascript(script);
       return this.startCore(runid, scriptTemplate(js, false));
     }
 
     if (!platformInfo.allowShellScripting) {
+      sendToAuditLog(req, {
+        category: 'shell',
+        component: 'RunnersController',
+        event: 'script.runFailed',
+        action: 'script',
+        severity: 'warn',
+        detail: script,
+        message: 'Scripts are not allowed',
+      });
+
       return { errorMessage: 'Shell scripting is not allowed' };
     }
 
+    sendToAuditLog(req, {
+      category: 'shell',
+      component: 'RunnersController',
+      event: 'script.run.shell',
+      action: 'script',
+      severity: 'info',
+      detail: script,
+      message: 'Running JS script',
+    });
+
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

package/src/controllers/serverConnections.js

@@ -12,6 +12,7 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('serverConnection');
 
@@ -145,6 +146,17 @@ module.exports = {
     if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    sendToAuditLog(req, {
+      category: 'serverop',
+      component: 'ServerConnectionsController',
+      action: 'listDatabases',
+      event: 'databases.list',
+      severity: 'info',
+      conid,
+      sessionParam: `${conid}`,
+      sessionGroup: 'listDatabases',
+      message: `Loaded databases for connection`,
+    });
     return opened?.databases ?? [];
   },
 

package/src/controllers/sessions.js

@@ -11,6 +11,7 @@ const { appdir } = require('../utility/directories');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const config = require('./config');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('sessions');
 
@@ -146,12 +147,24 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }) {
+  async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }
 
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'SessionController',
+      action: 'executeQuery',
+      event: 'query.execute',
+      severity: 'info',
+      detail: sql,
+      conid: session.conid,
+      database: session.database,
+      message: 'Executing query',
+    });
+
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({

package/src/controllers/storage.js

@@ -18,13 +18,14 @@ const {
 const { hasPermission } = require('../utility/hasPermission');
 const { changeSetToSql, removeSchemaFromChangeSet } = require('dbgate-datalib');
 const storageModel = require('../storageModel');
-const { dumpSqlCommand } = require('dbgate-sqltree');
+const { dumpSqlCommand, dumpSqlSelect } = require('dbgate-sqltree');
 const {
   runCommandOnDriver,
   getLogger,
   runQueryFmt,
   getPredefinedPermissions,
   runQueryOnDriver,
+  safeJsonParse,
 } = require('dbgate-tools');
 const socket = require('../utility/socket');
 const { obtainRefreshedLicense } = require('../utility/authProxy');
@@ -38,6 +39,7 @@ const {
 const crypto = require('crypto');
 const dataReplicator = require('../shell/dataReplicator');
 const storageReplicatorItems = require('../utility/storageReplicatorItems');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('storage');
 
@@ -226,9 +228,9 @@ module.exports = {
           conn,
           'update ~auth_methods set ~name=%v, ~is_disabled=%v, ~is_default = %v, ~is_collapsed = %v where ~id = %v',
           method.name,
-          method.isDisabled,
-          method.isDefault,
-          method.isCollapsed,
+          method.isDisabled ? 1 : 0,
+          method.isDefault ? 1 : 0,
+          method.isCollapsed ? 1 : 0,
           method.id
         );
       } else {
@@ -618,6 +620,30 @@ module.exports = {
     return true;
   },
 
+  copyConnection_meta: true,
+  async copyConnection({ id, name }) {
+    const [conn, driver] = await getStorageConnection();
+    if (!conn) {
+      return null;
+    }
+
+    const oldConnection = await storageSelectFmt(`select * from ~connections where ~id = %v`, id);
+    const oldConnectionPairs = Object.entries(oldConnection[0] || {})
+      .filter(x => x[0] != 'id')
+      .map(x => (x[0] == 'displayName' ? [x[0], name] : [x[0], x[1]]))
+      .map(x => (x[0] == 'conid' ? [x[0], crypto.randomUUID()] : [x[0], x[1]]));
+    await runQueryFmt(
+      driver,
+      conn,
+      'insert into ~connections (%,i) values (%,v)',
+      oldConnectionPairs.map(x => x[0]),
+      oldConnectionPairs.map(x => x[1])
+    );
+    socket.emitChanged('connection-list-changed');
+
+    return true;
+  },
+
   getRoleList_meta: true,
   async getRoleList() {
     const resp = await storageSelectFmt(`select ~roles.~id,~roles.~name from ~roles`);
@@ -699,6 +725,103 @@ module.exports = {
     return true;
   },
 
+  getAuditLog_meta: true,
+  async getAuditLog({ offset = 0, limit = 100, dateFrom = 0, dateTo = new Date().getTime(), filters = {} }) {
+    const [conn, driver] = await getStorageConnection();
+    const conditions = [
+      {
+        conditionType: 'binary',
+        operator: '>=',
+        left: { exprType: 'column', columnName: 'created' },
+        right: { exprType: 'value', value: dateFrom },
+      },
+      {
+        conditionType: 'binary',
+        operator: '<=',
+        left: { exprType: 'column', columnName: 'created' },
+        right: { exprType: 'value', value: dateTo },
+      },
+    ];
+    for (const [key, values] of Object.entries(filters)) {
+      if (values.length == 1 && values[0] == null) {
+        // @ts-ignore
+        conditions.push({
+          conditionType: 'isNull',
+          expr: { exprType: 'column', columnName: key },
+        });
+        continue;
+      }
+      // @ts-ignore
+      conditions.push({
+        conditionType: 'in',
+        expr: { exprType: 'column', columnName: key },
+        values,
+      });
+    }
+    const COLUMNS = [
+      'id',
+      'created',
+      'user_login',
+      'event',
+      'conid',
+      'database',
+      'connection_data',
+      'pure_name',
+      'message',
+    ];
+    const select = {
+      commandType: 'select',
+      from: {
+        name: {
+          pureName: 'audit_log',
+        },
+      },
+      columns: COLUMNS.map(columnName => ({
+        exprType: 'column',
+        columnName,
+      })),
+      where: {
+        conditionType: 'and',
+        conditions,
+      },
+      range: {
+        limit: limit,
+        offset: offset,
+      },
+      orderBy: [
+        {
+          exprType: 'column',
+          columnName: 'id',
+          direction: 'desc',
+        },
+      ],
+    };
+    const dmp = driver.createDumper();
+    // @ts-ignore
+    dumpSqlSelect(dmp, select);
+    const resp = await driver.query(conn, dmp.s);
+    return resp.rows.map(x => ({
+      ..._.mapKeys(x, (_v, k) => _.camelCase(k)),
+      connectionData: x.connection_data ? safeJsonParse(x.connection_data, null) : null,
+    }));
+  },
+
+  getAuditLogDetail_meta: true,
+  async getAuditLogDetail({ id }) {
+    const [_conn, driver] = await getStorageConnection();
+    const res = await storageSelectFmt(
+      `select *
+      from ~audit_log
+      where ~audit_log.~id = %v`,
+      id
+    );
+    if (!res[0]) return null;
+    return {
+      ..._.mapKeys(res[0], (_v, k) => _.camelCase(k)),
+      connectionData: res[0].connection_data ? safeJsonParse(res[0].connection_data, null) : null,
+    };
+  },
+
   async getExportedDatabase() {
     const connections = await storageSelectFmt(`select * from ~connections`);
     const user_permissions = await storageSelectFmt(`select * from ~user_permissions`);
@@ -751,4 +874,10 @@ module.exports = {
     const resp = await storageSelectFmt(`select distinct ~engine from ~connections`);
     return resp.map(x => x.engine);
   },
+
+  sendAuditLog_meta: true,
+  async sendAuditLog(props, req) {
+    sendToAuditLog(req, props);
+    return null;
+  },
 };

package/src/controllers/storageDb.js

@@ -7,6 +7,7 @@ const {
   extractErrorLogData,
   runQueryFmt,
   runQueryOnDriver,
+  adaptDatabaseInfo,
 } = require('dbgate-tools');
 const _ = require('lodash');
 const logger = getLogger('storageDb');
@@ -122,7 +123,8 @@ async function getStorageConnectionCore() {
   await dbgateApi.deployDb({
     systemConnection: newConnection,
     driver: storageDriver,
-    loadedDbModel: storageModel,
+    // @ts-ignore
+    loadedDbModel: adaptDatabaseInfo(storageModel, storageDriver),
     targetSchema: process.env.STORAGE_SCHEMA,
   });