dbgate-api 6.6.0 → 6.6.2

package/src/utility/DatastoreProxy.js

@@ -61,7 +61,7 @@ class DatastoreProxy {
         this.subprocess = null;
       });
       this.subprocess.on('error', err => {
-        logger.error(extractErrorLogData(err), 'Error in data store subprocess');
+        logger.error(extractErrorLogData(err), 'DBGM-00167 Error in data store subprocess');
         this.subprocess = null;
       });
       this.subprocess.send({ msgtype: 'open', file: this.file });
@@ -77,7 +77,7 @@ class DatastoreProxy {
       try {
         this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
       } catch (err) {
-        logger.error(extractErrorLogData(err), 'Error getting rows');
+        logger.error(extractErrorLogData(err), 'DBGM-00168 Error getting rows');
         this.subprocess = null;
       }
     });
@@ -91,7 +91,7 @@ class DatastoreProxy {
       try {
         this.subprocess.send({ msgtype: 'notify', msgid });
       } catch (err) {
-        logger.error(extractErrorLogData(err), 'Error notifying subprocess');
+        logger.error(extractErrorLogData(err), 'DBGM-00169 Error notifying subprocess');
         this.subprocess = null;
       }
     });

package/src/utility/JsonLinesDatastore.js

@@ -7,7 +7,6 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
-const requirePluginFunction = require('./requirePluginFunction');
 const esort = require('external-sorting');
 const { jsldir } = require('./directories');
 const LineReader = require('./LineReader');
@@ -23,7 +22,10 @@ class JsonLinesDatastore {
     this.notifyChangedCallback = null;
     this.currentFilter = null;
     this.currentSort = null;
-    this.rowFormatter = requirePluginFunction(formatterFunction);
+    if (formatterFunction) {
+      const requirePluginFunction = require('./requirePluginFunction');
+      this.rowFormatter = requirePluginFunction(formatterFunction);
+    }
     this.sortedFiles = {};
   }
 

package/src/utility/appLogStore.js

@@ -0,0 +1,119 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { logsdir } = require('./directories');
+const { format, addDays, startOfDay } = require('date-fns');
+const LineReader = require('./LineReader');
+const socket = require('./socket');
+const _ = require('lodash');
+
+async function getLogFiles(timeFrom, timeTo) {
+  const dir = logsdir();
+  const files = await fs.readdir(dir);
+  const startPrefix = format(timeFrom, 'yyyy-MM-dd');
+  const endPrefix = format(addDays(timeTo, 1), 'yyyy-MM-dd');
+  const logFiles = files
+    .filter(file => file.endsWith('.ndjson'))
+    .filter(file => file >= startPrefix && file < endPrefix);
+  return logFiles.sort().map(x => path.join(dir, x));
+}
+
+const RECENT_LOG_LIMIT = 1000;
+
+let recentLogs = null;
+const beforeRecentLogs = [];
+
+function adjustRecentLogs() {
+  if (recentLogs.length > RECENT_LOG_LIMIT) {
+    recentLogs.splice(0, recentLogs.length - RECENT_LOG_LIMIT);
+  }
+}
+
+function prepareEntryForExport(entry, lastEntry) {
+  return {
+    date: format(new Date(entry.time), 'yyyy-MM-dd'),
+    time: format(new Date(entry.time), 'HH:mm:ss'),
+    dtime: lastEntry ? entry.time - lastEntry.time : 0,
+    msgcode: entry.msgcode || '',
+    message: entry.msg || '',
+    ..._.omit(entry, ['time', 'msg', 'msgcode']),
+    conid: entry.conid || '',
+    database: entry.database || '',
+    engine: entry.engine || '',
+    ts: entry.time,
+  };
+}
+
+async function copyAppLogsIntoFile(timeFrom, timeTo, fileName, prepareForExport) {
+  const writeStream = fs.createWriteStream(fileName);
+
+  let lastEntry = null;
+  for (const file of await getLogFiles(timeFrom, timeTo)) {
+    const readStream = fs.createReadStream(file);
+    const reader = new LineReader(readStream);
+    do {
+      const line = await reader.readLine();
+      if (line == null) break;
+      try {
+        const logEntry = JSON.parse(line);
+        if (logEntry.time >= timeFrom && logEntry.time <= timeTo) {
+          writeStream.write(
+            JSON.stringify(prepareForExport ? prepareEntryForExport(logEntry, lastEntry) : logEntry) + '\n'
+          );
+          lastEntry = logEntry;
+        }
+      } catch (e) {
+        continue;
+      }
+    } while (true);
+  }
+}
+
+async function initializeRecentLogProvider() {
+  const logs = [];
+  for (const file of await getLogFiles(startOfDay(new Date()), new Date())) {
+    const fileStream = fs.createReadStream(file);
+    const reader = new LineReader(fileStream);
+    do {
+      const line = await reader.readLine();
+      if (line == null) break;
+      try {
+        const logEntry = JSON.parse(line);
+        logs.push(logEntry);
+        if (logs.length > RECENT_LOG_LIMIT) {
+          logs.shift();
+        }
+      } catch (e) {
+        continue;
+      }
+    } while (true);
+  }
+  recentLogs = logs;
+  recentLogs.push(...beforeRecentLogs);
+}
+
+let counter = 0;
+function pushToRecentLogs(msg) {
+  const finalMsg = {
+    ...msg,
+    counter,
+  };
+  counter += 1;
+  if (recentLogs) {
+    recentLogs.push(finalMsg);
+    adjustRecentLogs();
+    socket.emit('applog-event', finalMsg);
+  } else {
+    beforeRecentLogs.push(finalMsg);
+  }
+}
+
+function getRecentAppLogRecords() {
+  return recentLogs ?? beforeRecentLogs;
+}
+
+module.exports = {
+  initializeRecentLogProvider,
+  getRecentAppLogRecords,
+  pushToRecentLogs,
+  copyAppLogsIntoFile,
+};

package/src/utility/childProcessChecker.js

@@ -12,7 +12,7 @@ function childProcessChecker() {
       // This will come once parent dies.
       // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED
       //     and call process.exit()
-      logger.error(extractErrorLogData(err), 'parent died');
+      logger.error(extractErrorLogData(err), 'DBGM-00163 parent died');
       process.exit(1);
     }
   }, 1000);

package/src/utility/cloudIntf.js

@@ -77,7 +77,7 @@ function startCloudTokenChecking(sid, callback) {
         callback(resp.data);
       }
     } catch (err) {
-      logger.error(extractErrorLogData(err), 'Error checking cloud token');
+      logger.error(extractErrorLogData(err), 'DBGM-00164 Error checking cloud token');
     }
   }, 500);
 }
@@ -125,7 +125,7 @@ async function getCloudUsedEngines() {
     const resp = await callCloudApiGet('content-engines');
     return resp || [];
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error getting cloud content list');
+    logger.error(extractErrorLogData(err), 'DBGM-00165 Error getting cloud content list');
     return [];
   }
 }
@@ -208,7 +208,7 @@ async function updateCloudFiles(isRefresh) {
     lastCheckedTm = _.max(cloudFiles.map(x => parseInt(x.modifiedTm)));
   }
 
-  logger.info({ tags, lastCheckedTm }, 'Downloading cloud files');
+  logger.info({ tags, lastCheckedTm }, 'DBGM-00082 Downloading cloud files');
 
   const resp = await axios.default.get(
     `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}&isRefresh=${
@@ -223,7 +223,7 @@ async function updateCloudFiles(isRefresh) {
     }
   );
 
-  logger.info(`Downloaded ${resp.data.length} cloud files`);
+  logger.info(`DBGM-00083 Downloaded ${resp.data.length} cloud files`);
 
   const filesByPath = lastCheckedTm == 0 ? {} : _.keyBy(cloudFiles, 'path');
   for (const file of resp.data) {
@@ -269,7 +269,7 @@ async function refreshPublicFiles(isRefresh) {
   try {
     await updateCloudFiles(isRefresh);
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error updating cloud files');
+    logger.error(extractErrorLogData(err), 'DBGM-00166 Error updating cloud files');
   }
 }
 

package/src/utility/connectUtility.js

@@ -132,7 +132,7 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
 
   connection.ssl = await extractConnectionSslParams(connection);
 
-  const conn = await driver.connect({ ...connection, ...additionalOptions });
+  const conn = await driver.connect({ conid: connectionLoaded?._id, ...connection, ...additionalOptions });
   return conn;
 }
 

package/src/utility/directories.js

@@ -14,11 +14,11 @@ const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
   if (!createDirectories[dir]) {
     if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) {
-      getLogger('directories').info(`Cleaning directory ${dir}`);
+      getLogger('directories').info(`DBGM-00170 Cleaning directory ${dir}`);
       cleanDirectory(dir, _.isNumber(clean) ? clean : null);
     }
     if (!fs.existsSync(dir)) {
-      getLogger('directories').info(`Creating directory ${dir}`);
+      getLogger('directories').info(`DBGM-00171 Creating directory ${dir}`);
       fs.mkdirSync(dir);
     }
     createDirectories[dir] = true;

package/src/utility/extractSingleFileFromZip.js

@@ -42,13 +42,13 @@ function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
 
             // When the file is finished writing, resolve
             writeStream.on('finish', () => {
-              logger.info(`File "${fileInZip}" extracted to "${outputPath}".`);
+              logger.info(`DBGM-00088 File "${fileInZip}" extracted to "${outputPath}".`);
               resolve(true);
             });
 
             // Handle write errors
             writeStream.on('error', writeErr => {
-              logger.error(extractErrorLogData(writeErr), `Error extracting "${fileInZip}" from "${zipPath}".`);
+              logger.error(extractErrorLogData(writeErr), `DBGM-00089 Error extracting "${fileInZip}" from "${zipPath}".`);
               reject(writeErr);
             });
           });
@@ -67,7 +67,7 @@ function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
 
       // Handle general errors
       zipFile.on('error', err => {
-        logger.error(extractErrorLogData(err), `ZIP file error in ${zipPath}.`);
+        logger.error(extractErrorLogData(err), `DBGM-00172 ZIP file error in ${zipPath}.`);
         reject(err);
       });
     });

package/src/utility/hasPermission.js

@@ -1,96 +1,303 @@
-const { compilePermissions, testPermission } = require('dbgate-tools');
+const { compilePermissions, testPermission, getPermissionsCacheKey } = require('dbgate-tools');
 const _ = require('lodash');
 const { getAuthProviderFromReq } = require('../auth/authProvider');
 
 const cachedPermissions = {};
 
-function hasPermission(tested, req) {
+async function loadPermissionsFromRequest(req) {
+  const authProvider = getAuthProviderFromReq(req);
   if (!req) {
-    // request object not available, allow all
-    return true;
+    return null;
   }
 
-  const permissions = getAuthProviderFromReq(req).getCurrentPermissions(req);
+  const loadedPermissions = await authProvider.getCurrentPermissions(req);
+  return loadedPermissions;
+}
 
-  if (!cachedPermissions[permissions]) {
-    cachedPermissions[permissions] = compilePermissions(permissions);
+function hasPermission(tested, loadedPermissions) {
+  if (!loadedPermissions) {
+    // not available, allow all
+    return true;
   }
 
-  return testPermission(tested, cachedPermissions[permissions]);
-
-  // const { user } = (req && req.auth) || {};
-  // const { login } = (process.env.OAUTH_PERMISSIONS && req && req.user) || {};
-  // const key = user || login || '';
-  // const logins = getLogins();
+  const permissionsKey = getPermissionsCacheKey(loadedPermissions);
+  if (!cachedPermissions[permissionsKey]) {
+    cachedPermissions[permissionsKey] = compilePermissions(loadedPermissions);
+  }
 
-  // if (!userPermissions[key]) {
-  //   if (logins) {
-  //     const login = logins.find(x => x.login == user);
-  //     userPermissions[key] = compilePermissions(login ? login.permissions : null);
-  //   } else {
-  //     userPermissions[key] = compilePermissions(process.env.PERMISSIONS);
-  //   }
-  // }
-  // return testPermission(tested, userPermissions[key]);
+  return testPermission(tested, cachedPermissions[permissionsKey]);
 }
 
-// let loginsCache = null;
-// let loginsLoaded = false;
-
-// function getLogins() {
-//   if (loginsLoaded) {
-//     return loginsCache;
-//   }
-
-//   const res = [];
-//   if (process.env.LOGIN && process.env.PASSWORD) {
-//     res.push({
-//       login: process.env.LOGIN,
-//       password: process.env.PASSWORD,
-//       permissions: process.env.PERMISSIONS,
-//     });
-//   }
-//   if (process.env.LOGINS) {
-//     const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim()));
-//     for (const login of logins) {
-//       const password = process.env[`LOGIN_PASSWORD_${login}`];
-//       const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
-//       if (password) {
-//         res.push({
-//           login,
-//           password,
-//           permissions,
-//         });
-//       }
-//     }
-//   } else if (process.env.OAUTH_PERMISSIONS) {
-//     const login_permission_keys = Object.keys(process.env).filter(key => _.startsWith(key, 'LOGIN_PERMISSIONS_'));
-//     for (const permissions_key of login_permission_keys) {
-//       const login = permissions_key.replace('LOGIN_PERMISSIONS_', '');
-//       const permissions = process.env[permissions_key];
-//       userPermissions[login] = compilePermissions(permissions);
-//     }
-//   }
-
-//   loginsCache = res.length > 0 ? res : null;
-//   loginsLoaded = true;
-//   return loginsCache;
-// }
-
-function connectionHasPermission(connection, req) {
+function connectionHasPermission(connection, loadedPermissions) {
   if (!connection) {
     return true;
   }
   if (_.isString(connection)) {
-    return hasPermission(`connections/${connection}`, req);
+    return hasPermission(`connections/${connection}`, loadedPermissions);
   } else {
-    return hasPermission(`connections/${connection._id}`, req);
+    return hasPermission(`connections/${connection._id}`, loadedPermissions);
+  }
+}
+
+async function testConnectionPermission(connection, req, loadedPermissions) {
+  if (!loadedPermissions) {
+    loadedPermissions = await loadPermissionsFromRequest(req);
+  }
+  if (process.env.STORAGE_DATABASE) {
+    if (hasPermission(`all-connections`, loadedPermissions)) {
+      return;
+    }
+    const conid = _.isString(connection) ? connection : connection?._id;
+    if (hasPermission('internal-storage', loadedPermissions) && conid == '__storage') {
+      return;
+    }
+    const authProvider = getAuthProviderFromReq(req);
+    if (!req) {
+      return;
+    }
+    if (!(await authProvider.checkCurrentConnectionPermission(req, conid))) {
+      throw new Error('DBGM-00263 Connection permission not granted');
+    }
+  } else {
+    if (!connectionHasPermission(connection, loadedPermissions)) {
+      throw new Error('DBGM-00264 Connection permission not granted');
+    }
+  }
+}
+
+async function loadDatabasePermissionsFromRequest(req) {
+  const authProvider = getAuthProviderFromReq(req);
+  if (!req) {
+    return null;
+  }
+
+  const databasePermissions = await authProvider.getCurrentDatabasePermissions(req);
+  return databasePermissions;
+}
+
+async function loadTablePermissionsFromRequest(req) {
+  const authProvider = getAuthProviderFromReq(req);
+  if (!req) {
+    return null;
+  }
+
+  const tablePermissions = await authProvider.getCurrentTablePermissions(req);
+  return tablePermissions;
+}
+
+function matchDatabasePermissionRow(conid, database, permissionRow) {
+  if (permissionRow.connection_id) {
+    if (conid != permissionRow.connection_id) {
+      return false;
+    }
+  }
+  if (permissionRow.database_names_list) {
+    const items = permissionRow.database_names_list.split('\n');
+    if (!items.find(item => item.trim()?.toLowerCase() === database?.toLowerCase())) {
+      return false;
+    }
+  }
+  if (permissionRow.database_names_regex) {
+    const regex = new RegExp(permissionRow.database_names_regex, 'i');
+    if (!regex.test(database)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function matchTablePermissionRow(objectTypeField, schemaName, pureName, permissionRow) {
+  if (permissionRow.table_names_list) {
+    const items = permissionRow.table_names_list.split('\n');
+    if (!items.find(item => item.trim()?.toLowerCase() === pureName?.toLowerCase())) {
+      return false;
+    }
+  }
+  if (permissionRow.table_names_regex) {
+    const regex = new RegExp(permissionRow.table_names_regex, 'i');
+    if (!regex.test(pureName)) {
+      return false;
+    }
+  }
+  if (permissionRow.schema_names_list) {
+    const items = permissionRow.schema_names_list.split('\n');
+    if (!items.find(item => item.trim()?.toLowerCase() === schemaName?.toLowerCase())) {
+      return false;
+    }
+  }
+  if (permissionRow.schema_names_regex) {
+    const regex = new RegExp(permissionRow.schema_names_regex, 'i');
+    if (!regex.test(schemaName)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+const DATABASE_ROLE_ID_NAMES = {
+  '-1': 'view',
+  '-2': 'read_content',
+  '-3': 'write_data',
+  '-4': 'run_script',
+  '-5': 'deny',
+};
+
+function getDatabaseRoleLevelIndex(roleName) {
+  if (!roleName) {
+    return 6;
+  }
+  if (roleName == 'run_script') {
+    return 5;
+  }
+  if (roleName == 'write_data') {
+    return 4;
+  }
+  if (roleName == 'read_content') {
+    return 3;
+  }
+  if (roleName == 'view') {
+    return 2;
+  }
+  if (roleName == 'deny') {
+    return 1;
+  }
+  return 6;
+}
+
+function getTablePermissionRoleLevelIndex(roleName) {
+  if (!roleName) {
+    return 6;
+  }
+  if (roleName == 'run_script') {
+    return 5;
+  }
+  if (roleName == 'create_update_delete') {
+    return 4;
+  }
+  if (roleName == 'update_only') {
+    return 3;
+  }
+  if (roleName == 'read') {
+    return 2;
+  }
+  if (roleName == 'deny') {
+    return 1;
   }
+  return 6;
 }
 
-function testConnectionPermission(connection, req) {
-  if (!connectionHasPermission(connection, req)) {
-    throw new Error('Connection permission not granted');
+function getDatabasePermissionRole(conid, database, loadedDatabasePermissions) {
+  let res = 'deny';
+  for (const permissionRow of loadedDatabasePermissions) {
+    if (!matchDatabasePermissionRow(conid, database, permissionRow)) {
+      continue;
+    }
+    res = DATABASE_ROLE_ID_NAMES[permissionRow.database_permission_role_id];
+  }
+  return res;
+}
+
+const TABLE_ROLE_ID_NAMES = {
+  '-1': 'read',
+  '-2': 'update_only',
+  '-3': 'create_update_delete',
+  '-4': 'run_script',
+  '-5': 'deny',
+};
+
+const TABLE_SCOPE_ID_NAMES = {
+  '-1': 'all_objects',
+  '-2': 'tables',
+  '-3': 'views',
+  '-4': 'tables_views_collections',
+  '-5': 'procedures',
+  '-6': 'functions',
+  '-7': 'triggers',
+  '-8': 'sql_objects',
+  '-9': 'collections',
+};
+
+function getTablePermissionRole(
+  conid,
+  database,
+  objectTypeField,
+  schemaName,
+  pureName,
+  loadedTablePermissions,
+  databasePermissionRole
+) {
+  let res =
+    databasePermissionRole == 'read_content'
+      ? 'read'
+      : databasePermissionRole == 'write_data'
+      ? 'create_update_delete'
+      : databasePermissionRole == 'run_script'
+      ? 'run_script'
+      : 'deny';
+  for (const permissionRow of loadedTablePermissions) {
+    if (!matchDatabasePermissionRow(conid, database, permissionRow)) {
+      continue;
+    }
+    if (!matchTablePermissionRow(objectTypeField, schemaName, pureName, permissionRow)) {
+      continue;
+    }
+    const scope = TABLE_SCOPE_ID_NAMES[permissionRow.table_permission_scope_id];
+    switch (scope) {
+      case 'tables':
+        if (objectTypeField != 'tables') continue;
+        break;
+      case 'views':
+        if (objectTypeField != 'views') continue;
+        break;
+      case 'tables_views_collections':
+        if (objectTypeField != 'tables' && objectTypeField != 'views' && objectTypeField != 'collections') continue;
+        break;
+      case 'procedures':
+        if (objectTypeField != 'procedures') continue;
+        break;
+      case 'functions':
+        if (objectTypeField != 'functions') continue;
+        break;
+      case 'triggers':
+        if (objectTypeField != 'triggers') continue;
+        break;
+      case 'sql_objects':
+        if (objectTypeField != 'procedures' && objectTypeField != 'functions' && objectTypeField != 'triggers')
+          continue;
+        break;
+      case 'collections':
+        if (objectTypeField != 'collections') continue;
+        break;
+    }
+    res = TABLE_ROLE_ID_NAMES[permissionRow.table_permission_role_id];
+  }
+  return res;
+}
+
+async function testStandardPermission(permission, req, loadedPermissions) {
+  if (!loadedPermissions) {
+    loadedPermissions = await loadPermissionsFromRequest(req);
+  }
+  if (!hasPermission(permission, loadedPermissions)) {
+    throw new Error('DBGM-00265 Permission not granted');
+  }
+}
+
+async function testDatabaseRolePermission(conid, database, requiredRole, req) {
+  if (!process.env.STORAGE_DATABASE) {
+    return;
+  }
+  const loadedPermissions = await loadPermissionsFromRequest(req);
+  if (hasPermission(`all-databases`, loadedPermissions)) {
+    return;
+  }
+  const databasePermissions = await loadDatabasePermissionsFromRequest(req);
+  const role = getDatabasePermissionRole(conid, database, databasePermissions);
+  const requiredIndex = getDatabaseRoleLevelIndex(requiredRole);
+  const roleIndex = getDatabaseRoleLevelIndex(role);
+  if (roleIndex < requiredIndex) {
+    throw new Error('DBGM-00266 Permission not granted');
   }
 }
 
@@ -98,4 +305,12 @@ module.exports = {
   hasPermission,
   connectionHasPermission,
   testConnectionPermission,
+  loadPermissionsFromRequest,
+  loadDatabasePermissionsFromRequest,
+  loadTablePermissionsFromRequest,
+  getDatabasePermissionRole,
+  getTablePermissionRole,
+  testStandardPermission,
+  testDatabaseRolePermission,
+  getTablePermissionRoleLevelIndex,
 };

package/src/utility/loadModelTransform.js

@@ -28,7 +28,7 @@ async function loadModelTransform(file) {
     }
     return null;
   } catch (err) {
-    logger.error(extractErrorLogData(err), `Error loading model transform ${file}`);
+    logger.error(extractErrorLogData(err), `DBGM-00173 Error loading model transform ${file}`);
     return null;
   }
 }