dbgate-api-premium 6.5.4 → 6.5.6

package/src/utility/auditlog.js

@@ -0,0 +1,288 @@
+// *** This file is part of DbGate Premium ***
+
+const _ = require('lodash');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { storageSqlCommandFmt, storageSelectFmt } = require('../controllers/storageDb');
+const logger = getLogger('auditLog');
+
+let auditLogQueue = [];
+let isProcessing = false;
+let isPlanned = false;
+
+function nullableSum(a, b) {
+  const res = (a || 0) + (b || 0);
+  return res == 0 ? null : res;
+}
+
+async function processAuditLogQueue() {
+  do {
+    isProcessing = true;
+    const elements = [...auditLogQueue];
+    auditLogQueue = [];
+
+    while (elements.length > 0) {
+      const element = elements.shift();
+      if (!element) continue;
+      if (element.sessionId && element.sessionGroup && element.sessionParam) {
+        const existingRows = await storageSelectFmt(
+          '^select ~id, ~sumint_1, ~sumint_2  from ~audit_log where ~session_id = %v and ~session_group = %v and ~session_param = %v',
+          element.sessionId,
+          element.sessionGroup,
+          element.sessionParam
+        );
+        if (existingRows && existingRows.length > 0) {
+          const existing = existingRows[0];
+          await storageSqlCommandFmt(
+            '^update ~audit_log set ~sumint_1 = %v, ~sumint_2 = %v, ~modified = %v where ~id = %v',
+            nullableSum(element.sumint1, existing.sumint_1),
+            nullableSum(element.sumint2, existing.sumint_2),
+            element.created,
+            existing.id
+          );
+          // only update existing session
+          continue;
+        }
+      }
+      try {
+        let connectionData = null;
+        if (element.conid) {
+          const connections = await storageSelectFmt('^select * from ~connections where ~conid = %v', element.conid);
+          if (connections[0])
+            connectionData = _.pick(connections[0], [
+              'displayName',
+              'engine',
+              'displayName',
+              'databaseUrl',
+              'singleDatabase',
+              'server',
+              'databaseFile',
+              'useSshTunnel',
+              'sshHost',
+              'defaultDatabase',
+            ]);
+        }
+
+        const detailText = _.isPlainObject(element.detail) ? JSON.stringify(element.detail) : element.detail || null;
+        const connectionDataText = connectionData ? JSON.stringify(connectionData) : null;
+        await storageSqlCommandFmt(
+          `^insert ^into ~audit_log (
+            ~user_id, ~user_login, ~created, ~category, ~component, ~event, ~detail, ~detail_full_length, ~action, ~severity, 
+            ~conid, ~database, ~schema_name, ~pure_name, ~sumint_1, ~sumint_2, ~session_id, ~session_group, ~session_param, ~connection_data, ~message) 
+            values (%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v)`,
+          element.userId || null,
+          element.login || null,
+          element.created,
+          element.category || null,
+          element.component || null,
+          element.event || null,
+          detailText?.slice(0, 1000) || null,
+          detailText?.length || null,
+          element.action || null,
+          element.severity || 'info',
+          element.conid || null,
+          element.database || null,
+          element.schemaName || null,
+          element.pureName || null,
+          element.sumint1 || null,
+          element.sumint2 || null,
+          element.sessionId || null,
+          element.sessionGroup || null,
+          element.sessionParam || null,
+          connectionDataText?.slice(0, 1000) || null,
+          element.message || null
+        );
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'Error processing audit log entry');
+      }
+    }
+
+    isProcessing = false;
+  } while (auditLogQueue.length > 0);
+  isPlanned = false;
+}
+
+async function sendToAuditLog(
+  req,
+  {
+    category,
+    component,
+    event,
+    detail = null,
+    action,
+    severity = 'info',
+    conid = null,
+    database = null,
+    schemaName = null,
+    pureName = null,
+    sumint1 = null,
+    sumint2 = null,
+    sessionGroup = null,
+    sessionParam = null,
+    message = null,
+  }
+) {
+  if (!process.env.STORAGE_DATABASE) {
+    return;
+  }
+  const config = require('../controllers/config');
+  const settings = await config.getCachedSettings();
+  if (settings?.['storage.useAuditLog'] != 'true') {
+    return;
+  }
+
+  const { login, userId } = req?.user || {};
+  const sessionId = req?.headers?.['x-api-session-id'];
+
+  auditLogQueue.push({
+    userId,
+    login,
+    created: new Date().getTime(),
+    category,
+    component,
+    event,
+    detail,
+    action,
+    severity,
+    conid,
+    database,
+    schemaName,
+    pureName,
+    sumint1,
+    sumint2,
+    sessionId,
+    sessionGroup,
+    sessionParam,
+    message,
+  });
+  if (!isProcessing && !isPlanned) {
+    setTimeout(() => {
+      isPlanned = true;
+      processAuditLogQueue();
+    }, 0);
+  }
+}
+
+function maskLogFields(script) {
+  return _.cloneDeepWith(script, (value, key) => {
+    if (_.isString(key) && key.toLowerCase().includes('password')) {
+      return '****';
+    }
+    if (key == 'query') {
+      return '****';
+    }
+  });
+}
+
+function extractConnectionId(connection) {
+  if (
+    connection?.server == process.env.STORAGE_SERVER &&
+    connection?.database == process.env.STORAGE_DATABASE &&
+    connection?.port == process.env.STORAGE_PORT
+  ) {
+    return '__storage';
+  }
+  if (connection?.conid) {
+    return connection.conid;
+  }
+  return null;
+}
+
+function analyseJsonRunnerScript(script) {
+  const [assignSource, assignTarget, copyStream] = _.isArray(script?.commands) ? script?.commands : [];
+  if (assignSource?.type != 'assign') {
+    return null;
+  }
+  if (assignTarget?.type != 'assign') {
+    return null;
+  }
+  if (copyStream?.type != 'copyStream') {
+    return null;
+  }
+
+  if (assignTarget?.functionName == 'tableWriter') {
+    const pureName = assignTarget?.props?.pureName;
+    const schemaName = assignTarget?.props?.schemaName;
+    const connection = assignTarget?.props?.connection;
+    if (pureName && connection) {
+      return {
+        category: 'import',
+        component: 'RunnersController',
+        event: 'import.table',
+        action: 'import',
+        severity: 'info',
+        message: `Importing table ${pureName}`,
+        pureName: pureName,
+        conid: extractConnectionId(connection),
+        database: connection?.database,
+        schemaName: schemaName,
+        detail: maskLogFields(script),
+      };
+    }
+    return null;
+  }
+
+  if (assignSource?.functionName == 'tableReader') {
+    const pureName = assignSource?.props?.pureName;
+    const schemaName = assignSource?.props?.schemaName;
+    const connection = assignSource?.props?.connection;
+    if (pureName && connection) {
+      return {
+        category: 'export',
+        component: 'RunnersController',
+        event: 'export.table',
+        action: 'export',
+        severity: 'info',
+        message: `Exporting table ${pureName}`,
+        pureName: pureName,
+        conid: extractConnectionId(connection),
+        database: connection?.database,
+        schemaName: schemaName,
+        detail: maskLogFields(script),
+      };
+    }
+    return null;
+  }
+
+  if (assignSource?.functionName == 'queryReader') {
+    const connection = assignSource?.props?.connection;
+    if (connection) {
+      return {
+        category: 'export',
+        component: 'RunnersController',
+        event: 'export.query',
+        action: 'export',
+        severity: 'info',
+        message: 'Exporting query',
+        conid: extractConnectionId(connection),
+        database: connection?.database,
+        detail: maskLogFields(script),
+      };
+    }
+    return null;
+  }
+
+  return null;
+}
+
+function logJsonRunnerScript(req, script) {
+  const analysed = analyseJsonRunnerScript(script);
+
+  if (analysed) {
+    sendToAuditLog(req, analysed);
+  } else {
+    sendToAuditLog(req, {
+      category: 'shell',
+      component: 'RunnersController',
+      event: 'script.run.json',
+      action: 'script',
+      severity: 'info',
+      detail: maskLogFields(script),
+      message: 'Running JSON script',
+    });
+  }
+}
+
+module.exports = {
+  sendToAuditLog,
+  logJsonRunnerScript,
+};

package/src/utility/authProxy.js

@@ -31,12 +31,16 @@ function isAuthProxySupported() {
   return true;
 }
 
+function getE2ETestHeaders() {
+  return processArgs.runE2eTests ? { 'x-api-key': 'bcf6e1a0-5763-4060-9391-18fda005722d' } : {};
+}
+
 function getAxiosParamsWithLicense() {
   return {
     headers: {
       'Content-Type': 'application/json',
       Authorization: `Bearer ${licenseKey ?? process.env.DBGATE_LICENSE}`,
-      'x-api-key': processArgs.runE2eTests ? 'bcf6e1a0-5763-4060-9391-18fda005722d' : null,
+      ...getE2ETestHeaders(),
     },
   };
 }
@@ -44,10 +48,13 @@ function getAxiosParamsWithLicense() {
 function getLicenseHttpHeaders() {
   const licenseValue = licenseKey ?? process.env.DBGATE_LICENSE;
   if (!licenseValue) {
-    return {};
+    return {
+      ...getE2ETestHeaders(),
+    };
   }
   return {
     'x-license': licenseValue,
+    ...getE2ETestHeaders(),
   };
 }
 
@@ -178,6 +185,27 @@ async function obtainRefreshedLicense() {
   }
 }
 
+async function tryToGetRefreshedLicense(oldLicenseKey) {
+  try {
+    const respToken = await axios.default.post(
+      `${AUTH_PROXY_URL}/refresh-license`,
+      {},
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${oldLicenseKey}`,
+        },
+      }
+    );
+    return respToken.data;
+  } catch (err) {
+    return {
+      status: 'error',
+      message: err.message,
+    };
+  }
+}
+
 /**
  * @param {import('dbgate-types').DatabaseInfo} structure
  * @returns {import('dbgate-types').DatabaseInfoTiny}
@@ -296,4 +324,5 @@ module.exports = {
   callCompleteOnCursorApi,
   callRefactorSqlQueryApi,
   getLicenseHttpHeaders,
+  tryToGetRefreshedLicense,
 };

package/src/utility/checkLicense.js

@@ -147,6 +147,8 @@ function checkLicenseKey(licenseKey) {
       expiration: new Date(decoded.exp * 1000).toISOString(),
       daysLeft: Math.round((decoded.exp * 1000 - Date.now()) / (24 * 60 * 60 * 1000)),
       licenseTypeObj,
+      licenseId: decoded.licenseId,
+      users: decoded.users,
     };
   } catch (err) {
     try {
@@ -159,6 +161,7 @@ function checkLicenseKey(licenseKey) {
             status: 'error',
             isExpired: true,
             errorMessage: 'License key is expired',
+            expiration: exp * 1000,
           };
         }
       }

package/src/utility/cloudIntf.js

@@ -1,4 +1,5 @@
 const axios = require('axios');
+const crypto = require('crypto');
 const fs = require('fs-extra');
 const _ = require('lodash');
 const path = require('path');
@@ -34,11 +35,12 @@ const DBGATE_CLOUD_URL = process.env.LOCAL_DBGATE_CLOUD
   ? 'https://cloud.dbgate.udolni.net'
   : 'https://cloud.dbgate.io';
 
-async function createDbGateIdentitySession(client) {
+async function createDbGateIdentitySession(client, redirectUri) {
   const resp = await axios.default.post(
     `${DBGATE_IDENTITY_URL}/api/create-session`,
     {
       client,
+      redirectUri,
     },
     {
       headers: {
@@ -70,7 +72,7 @@ function startCloudTokenChecking(sid, callback) {
       });
       // console.log('CHECK RESP:', resp.data);
 
-      if (resp.data.email) {
+      if (resp.data?.email) {
         clearInterval(interval);
         callback(resp.data);
       }
@@ -80,6 +82,34 @@ function startCloudTokenChecking(sid, callback) {
   }, 500);
 }
 
+async function readCloudTokenHolder(sid) {
+  const resp = await axios.default.get(`${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+    },
+  });
+  if (resp.data?.email) {
+    return resp.data;
+  }
+  return null;
+}
+
+async function readCloudTestTokenHolder(email) {
+  const resp = await axios.default.post(
+    `${DBGATE_IDENTITY_URL}/api/test-token`,
+    { email },
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+      },
+    }
+  );
+  if (resp.data?.email) {
+    return resp.data;
+  }
+  return null;
+}
+
 async function loadCloudFiles() {
   try {
     const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
@@ -187,7 +217,7 @@ async function updateCloudFiles(isRefresh) {
     {
       headers: {
         ...getLicenseHttpHeaders(),
-        ...(await getCloudSigninHeaders()),
+        ...(await getCloudInstanceHeaders()),
         'x-app-version': currentVersion.version,
       },
     }
@@ -271,6 +301,17 @@ async function callCloudApiGet(endpoint, signinHolder = null, additionalHeaders
   return resp.data;
 }
 
+async function getCloudInstanceHeaders() {
+  if (!(await fs.exists(path.join(datadir(), 'cloud-instance.txt')))) {
+    const newInstanceId = crypto.randomUUID();
+    await fs.writeFile(path.join(datadir(), 'cloud-instance.txt'), newInstanceId);
+  }
+  const instanceId = await fs.readFile(path.join(datadir(), 'cloud-instance.txt'), 'utf-8');
+  return {
+    'x-cloud-instance': instanceId,
+  };
+}
+
 async function callCloudApiPost(endpoint, body, signinHolder = null) {
   if (!signinHolder) {
     signinHolder = await getCloudSigninHolder();
@@ -396,4 +437,6 @@ module.exports = {
   loadCachedCloudConnection,
   putCloudContent,
   removeCloudCachedConnection,
+  readCloudTokenHolder,
+  readCloudTestTokenHolder,
 };

package/src/utility/getChartExport.js

@@ -1,4 +1,12 @@
-const getChartExport = (title, config, imageFile) => {
+const getChartExport = (title, config, imageFile, plugins) => {
+  const PLUGIN_TAGS = {
+    zoom: '<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>',
+    dataLabels:
+      '<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-datalabels/2.2.0/chartjs-plugin-datalabels.min.js" integrity="sha512-JPcRR8yFa8mmCsfrw4TNte1ZvF1e3+1SdGMslZvmrzDYxS69J7J49vkFL8u6u8PlPJK+H3voElBtUCzaXj+6ig==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>',
+    outlabels:
+      '<script src="https://cdn.jsdelivr.net/npm/@energiency/chartjs-plugin-piechart-outlabels@1.3.4/dist/chartjs-plugin-piechart-outlabels.min.js"></script>',
+  };
+
   return `<html>
 <meta charset='utf-8'>
 
@@ -8,7 +16,7 @@ const getChartExport = (title, config, imageFile) => {
     <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js" integrity="sha512-qTXRIMyZIFb8iQcfjXWCO8+M5Tbc38Qi5WzdPOYZHIlZpzBHG3L3by84BBBOiRGiEb7KKtAOAs5qYdUiZiQNNQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.min.js" integrity="sha512-oh5t+CdSBsaVVAvxcZKy3XJdP7ZbYUBSRCXDTVn0ODewMDDNnELsrG9eDm8rVZAQg7RsDD/8K3MjPAFB13o6eA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js" integrity="sha512-UXumZrZNiOwnTcZSHLOfcTs0aos2MzBWHXOHOuB0J/R44QB0dwY5JgfbvljXcklVf65Gc4El6RjZ+lnwd2az2g==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    ${plugins.map(plugin => PLUGIN_TAGS[plugin] ?? '')}
 
     <style>
         a { text-decoration: none }
@@ -45,7 +53,7 @@ const getChartExport = (title, config, imageFile) => {
     </div>
 
     <div class="footer">
-        Exported from <a href='https://dbgate.org/' target='_blank'>DbGate</a>, powered by <a href='https://www.chartjs.org/' target='_blank'>Chart.js</a>
+        Exported from <a href='https://dbgate.io/' target='_blank'>DbGate</a>, powered by <a href='https://www.chartjs.org/' target='_blank'>Chart.js</a>
     </div>
 </body>
 

package/src/utility/getMapExport.js

@@ -18,7 +18,7 @@ const getMapExport = (geoJson) => {
     leaflet
       .tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
         maxZoom: 19,
-        attribution: '<a href="https://dbgate.org" title="Exported from DbGate">DbGate</a> | © OpenStreetMap',
+        attribution: '<a href="https://dbgate.io" title="Exported from DbGate">DbGate</a> | © OpenStreetMap',
       })
       .addTo(map);
       

package/src/utility/loginchecker.js

@@ -0,0 +1,69 @@
+// *** This file is part of DbGate Premium ***
+
+const _ = require('lodash');
+const { sendToAuditLog } = require('./auditlog');
+const { checkLicense } = require('./checkLicense');
+const LOGIN_LIMIT_ERROR = 'Your limit of concurrent logins has been reached';
+const jwt = require('jsonwebtoken');
+
+// map string (user key) => time to expiration
+let activeLoggedUsers = {};
+
+// map string (user key) => token expiration time
+let activeLoggedTokens = {};
+
+function markUserAsActive(licenseUid, accessToken) {
+  activeLoggedUsers[licenseUid] = Date.now() + 60 * 1000; // mark user as active for 1 minute
+}
+
+async function isLoginLicensed(req, licenseUid) {
+  const license = await checkLicense();
+  activeLoggedUsers = _.pickBy(activeLoggedUsers, value => value > Date.now());
+  activeLoggedTokens = _.pickBy(activeLoggedTokens, value => value > Date.now());
+  if (licenseUid in activeLoggedUsers) {
+    return true;
+  }
+  if (licenseUid in activeLoggedTokens) {
+    return true;
+  }
+  if (license.licenseId == 'f0346efe-ebc2-4822-9a83-4b4668a897e5' && license?.users != null) {
+    const uniqUsers = _.uniq([...Object.keys(activeLoggedUsers), ...Object.keys(activeLoggedTokens)]);
+    const currentUserCount = uniqUsers.length;
+
+    if (currentUserCount >= license?.users) {
+      sendToAuditLog(req, {
+        category: 'auth',
+        component: 'StorageAuthProvider',
+        action: 'loginFail',
+        event: 'login.licenseFailed',
+        severity: 'warn',
+        detail: { licenseUid, activeUsers: currentUserCount, licenseUsers: license?.users },
+        message: `Login failed, too many active users: ${currentUserCount + 1}, license allows only ${license?.users}`,
+      });
+
+      return false;
+    }
+  }
+  return true;
+}
+
+function markTokenAsLoggedIn(licenseUid, token) {
+  if (licenseUid != 'anonymous') {
+    const decoded = jwt.decode(token);
+    activeLoggedTokens[licenseUid] = decoded.exp * 1000;
+  }
+  activeLoggedUsers[licenseUid] = Date.now() + 60 * 1000; // mark user as active for 1 minute
+}
+
+function markLoginAsLoggedOut(licenseUid) {
+  delete activeLoggedUsers[licenseUid];
+  delete activeLoggedTokens[licenseUid];
+}
+
+module.exports = {
+  markUserAsActive,
+  markTokenAsLoggedIn,
+  isLoginLicensed,
+  LOGIN_LIMIT_ERROR,
+  markLoginAsLoggedOut,
+};