dbgate-api-premium 6.6.0 → 6.6.1

package/src/utility/JsonLinesDatastore.js

@@ -7,7 +7,6 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
-const requirePluginFunction = require('./requirePluginFunction');
 const esort = require('external-sorting');
 const { jsldir } = require('./directories');
 const LineReader = require('./LineReader');
@@ -23,7 +22,10 @@ class JsonLinesDatastore {
     this.notifyChangedCallback = null;
     this.currentFilter = null;
     this.currentSort = null;
-    this.rowFormatter = requirePluginFunction(formatterFunction);
+    if (formatterFunction) {
+      const requirePluginFunction = require('./requirePluginFunction');
+      this.rowFormatter = requirePluginFunction(formatterFunction);
+    }
     this.sortedFiles = {};
   }
 

package/src/utility/appLogStore.js

@@ -0,0 +1,119 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { logsdir } = require('./directories');
+const { format, addDays, startOfDay } = require('date-fns');
+const LineReader = require('./LineReader');
+const socket = require('./socket');
+const _ = require('lodash');
+
+async function getLogFiles(timeFrom, timeTo) {
+  const dir = logsdir();
+  const files = await fs.readdir(dir);
+  const startPrefix = format(timeFrom, 'yyyy-MM-dd');
+  const endPrefix = format(addDays(timeTo, 1), 'yyyy-MM-dd');
+  const logFiles = files
+    .filter(file => file.endsWith('.ndjson'))
+    .filter(file => file >= startPrefix && file < endPrefix);
+  return logFiles.sort().map(x => path.join(dir, x));
+}
+
+const RECENT_LOG_LIMIT = 1000;
+
+let recentLogs = null;
+const beforeRecentLogs = [];
+
+function adjustRecentLogs() {
+  if (recentLogs.length > RECENT_LOG_LIMIT) {
+    recentLogs.splice(0, recentLogs.length - RECENT_LOG_LIMIT);
+  }
+}
+
+function prepareEntryForExport(entry, lastEntry) {
+  return {
+    date: format(new Date(entry.time), 'yyyy-MM-dd'),
+    time: format(new Date(entry.time), 'HH:mm:ss'),
+    dtime: lastEntry ? entry.time - lastEntry.time : 0,
+    msgcode: entry.msgcode || '',
+    message: entry.msg || '',
+    ..._.omit(entry, ['time', 'msg', 'msgcode']),
+    conid: entry.conid || '',
+    database: entry.database || '',
+    engine: entry.engine || '',
+    ts: entry.time,
+  };
+}
+
+async function copyAppLogsIntoFile(timeFrom, timeTo, fileName, prepareForExport) {
+  const writeStream = fs.createWriteStream(fileName);
+
+  let lastEntry = null;
+  for (const file of await getLogFiles(timeFrom, timeTo)) {
+    const readStream = fs.createReadStream(file);
+    const reader = new LineReader(readStream);
+    do {
+      const line = await reader.readLine();
+      if (line == null) break;
+      try {
+        const logEntry = JSON.parse(line);
+        if (logEntry.time >= timeFrom && logEntry.time <= timeTo) {
+          writeStream.write(
+            JSON.stringify(prepareForExport ? prepareEntryForExport(logEntry, lastEntry) : logEntry) + '\n'
+          );
+          lastEntry = logEntry;
+        }
+      } catch (e) {
+        continue;
+      }
+    } while (true);
+  }
+}
+
+async function initializeRecentLogProvider() {
+  const logs = [];
+  for (const file of await getLogFiles(startOfDay(new Date()), new Date())) {
+    const fileStream = fs.createReadStream(file);
+    const reader = new LineReader(fileStream);
+    do {
+      const line = await reader.readLine();
+      if (line == null) break;
+      try {
+        const logEntry = JSON.parse(line);
+        logs.push(logEntry);
+        if (logs.length > RECENT_LOG_LIMIT) {
+          logs.shift();
+        }
+      } catch (e) {
+        continue;
+      }
+    } while (true);
+  }
+  recentLogs = logs;
+  recentLogs.push(...beforeRecentLogs);
+}
+
+let counter = 0;
+function pushToRecentLogs(msg) {
+  const finalMsg = {
+    ...msg,
+    counter,
+  };
+  counter += 1;
+  if (recentLogs) {
+    recentLogs.push(finalMsg);
+    adjustRecentLogs();
+    socket.emit('applog-event', finalMsg);
+  } else {
+    beforeRecentLogs.push(finalMsg);
+  }
+}
+
+function getRecentAppLogRecords() {
+  return recentLogs ?? beforeRecentLogs;
+}
+
+module.exports = {
+  initializeRecentLogProvider,
+  getRecentAppLogRecords,
+  pushToRecentLogs,
+  copyAppLogsIntoFile,
+};

package/src/utility/auditlog.js

@@ -92,7 +92,7 @@ async function processAuditLogQueue() {
           element.message || null
         );
       } catch (err) {
-        logger.error(extractErrorLogData(err), 'Error processing audit log entry');
+        logger.error(extractErrorLogData(err), 'DBGM-00159 Error processing audit log entry');
       }
     }
 

package/src/utility/authProxy.js

@@ -120,7 +120,7 @@ function startTokenChecking(sid, callback) {
         callback(resp.data.token);
       }
     } catch (err) {
-      logger.error(extractErrorLogData(err), 'Error checking token');
+      logger.error(extractErrorLogData(err), 'DBGM-00160 Error checking token');
     }
   }, 500);
 }
@@ -157,12 +157,12 @@ async function obtainRefreshedLicense() {
 
   const decoded = jwt.decode(licenseKey?.trim());
   if (!decoded?.end) {
-    logger.info('Invalid license found');
+    logger.info('DBGM-00078 Invalid license found');
     return null;
   }
 
   if (Date.now() > decoded.end * 1000) {
-    logger.info('License expired, trying to obtain fresh license');
+    logger.info('DBGM-00079 License expired, trying to obtain fresh license');
 
     try {
       const respToken = await axios.default.post(

package/src/utility/checkLicense.js

@@ -52,6 +52,12 @@ const licenseTypeById = {
     isForWeb: true,
     isForApp: true,
   },
+  '6c734e30-9b66-417d-91a2-0f9aeb739b32': {
+    name: 'Demo',
+    isPremium: true,
+    isForWeb: true,
+    isForApp: true,
+  },
 };
 
 function getLicenseByDecoded(decoded) {
@@ -105,10 +111,10 @@ async function getAwsMetadata() {
     awsMetadata = { amiId, region };
     awsMetadataLoaded = true;
 
-    logger.info(`Loaded AWS metadata, AMIID=${amiId}, region=${region}`);
+    logger.info(`DBGM-00080 Loaded AWS metadata, AMIID=${amiId}, region=${region}`);
     return { amiId, region };
   } catch (error) {
-    logger.error(extractErrorLogData(error), 'Error getting AWS metadata');
+    logger.error(extractErrorLogData(error), 'DBGM-00081 Error getting AWS metadata');
     awsMetadataLoaded = true;
     return null;
   }
@@ -134,7 +140,7 @@ function checkLicenseKey(licenseKey) {
       (platformInfo.isElectron && !licenseTypeObj.isForApp) ||
       (!platformInfo.isElectron && !licenseTypeObj.isForWeb)
     ) {
-      logger.error(`Incorrect license type, found ${licenseTypeObj?.name ?? 'n/a'}`);
+      logger.error(`DBGM-00161 Incorrect license type, found ${licenseTypeObj?.name ?? 'n/a'}`);
       return {
         status: 'error',
         error: `Incorrect license type, found ${licenseTypeObj?.name ?? 'n/a'}`,
@@ -167,7 +173,7 @@ function checkLicenseKey(licenseKey) {
       }
     } catch (err) {}
 
-    logger.error(extractErrorLogData(err), 'License token is invalid');
+    logger.error(extractErrorLogData(err), 'DBGM-00162 License token is invalid');
     return {
       status: 'error',
       error: err.message ?? 'License token is invalid',

package/src/utility/childProcessChecker.js

@@ -12,7 +12,7 @@ function childProcessChecker() {
       // This will come once parent dies.
       // One way can be to check for error code ERR_IPC_CHANNEL_CLOSED
       //     and call process.exit()
-      logger.error(extractErrorLogData(err), 'parent died');
+      logger.error(extractErrorLogData(err), 'DBGM-00163 parent died');
       process.exit(1);
     }
   }, 1000);

package/src/utility/cloudIntf.js

@@ -77,7 +77,7 @@ function startCloudTokenChecking(sid, callback) {
         callback(resp.data);
       }
     } catch (err) {
-      logger.error(extractErrorLogData(err), 'Error checking cloud token');
+      logger.error(extractErrorLogData(err), 'DBGM-00164 Error checking cloud token');
     }
   }, 500);
 }
@@ -125,7 +125,7 @@ async function getCloudUsedEngines() {
     const resp = await callCloudApiGet('content-engines');
     return resp || [];
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error getting cloud content list');
+    logger.error(extractErrorLogData(err), 'DBGM-00165 Error getting cloud content list');
     return [];
   }
 }
@@ -208,7 +208,7 @@ async function updateCloudFiles(isRefresh) {
     lastCheckedTm = _.max(cloudFiles.map(x => parseInt(x.modifiedTm)));
   }
 
-  logger.info({ tags, lastCheckedTm }, 'Downloading cloud files');
+  logger.info({ tags, lastCheckedTm }, 'DBGM-00082 Downloading cloud files');
 
   const resp = await axios.default.get(
     `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}&isRefresh=${
@@ -223,7 +223,7 @@ async function updateCloudFiles(isRefresh) {
     }
   );
 
-  logger.info(`Downloaded ${resp.data.length} cloud files`);
+  logger.info(`DBGM-00083 Downloaded ${resp.data.length} cloud files`);
 
   const filesByPath = lastCheckedTm == 0 ? {} : _.keyBy(cloudFiles, 'path');
   for (const file of resp.data) {
@@ -269,7 +269,7 @@ async function refreshPublicFiles(isRefresh) {
   try {
     await updateCloudFiles(isRefresh);
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error updating cloud files');
+    logger.error(extractErrorLogData(err), 'DBGM-00166 Error updating cloud files');
   }
 }
 

package/src/utility/cloudUpgrade.js

@@ -31,7 +31,7 @@ async function checkCloudUpgrade() {
       semver.gt(version, currentVersion.version) &&
       (!cloudDownloadedVersion || semver.gt(version, cloudDownloadedVersion))
     ) {
-      logger.info(`New version available: ${version}`);
+      logger.info(`DBGM-00084 New version available: ${version}`);
       const zipUrl = json.assets.find(x => x.name == 'cloud-build.zip').browser_download_url;
 
       const writer = fs.createWriteStream(process.env.CLOUD_UPGRADE_FILE);
@@ -50,16 +50,16 @@ async function checkCloudUpgrade() {
       });
       await fsp.writeFile(process.env.CLOUD_UPGRADE_FILE + '.version', version);
 
-      logger.info(`Downloaded new version from ${zipUrl}`);
+      logger.info(`DBGM-00085 Downloaded new version from ${zipUrl}`);
     } else {
       logger.info(
-        `Checked version ${version} is not newer than ${
+        `DBGM-00086 Checked version ${version} is not newer than ${
           cloudDownloadedVersion ?? currentVersion.version
         }, upgrade skippped`
       );
     }
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');
+    logger.error(extractErrorLogData(err), 'DBGM-00087 Error checking cloud upgrade');
   }
 }
 

package/src/utility/connectUtility.js

@@ -132,7 +132,7 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
 
   connection.ssl = await extractConnectionSslParams(connection);
 
-  const conn = await driver.connect({ ...connection, ...additionalOptions });
+  const conn = await driver.connect({ conid: connectionLoaded?._id, ...connection, ...additionalOptions });
   return conn;
 }
 

package/src/utility/directories.js

@@ -14,11 +14,11 @@ const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
   if (!createDirectories[dir]) {
     if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) {
-      getLogger('directories').info(`Cleaning directory ${dir}`);
+      getLogger('directories').info(`DBGM-00170 Cleaning directory ${dir}`);
       cleanDirectory(dir, _.isNumber(clean) ? clean : null);
     }
     if (!fs.existsSync(dir)) {
-      getLogger('directories').info(`Creating directory ${dir}`);
+      getLogger('directories').info(`DBGM-00171 Creating directory ${dir}`);
       fs.mkdirSync(dir);
     }
     createDirectories[dir] = true;

package/src/utility/extractSingleFileFromZip.js

@@ -42,13 +42,13 @@ function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
 
             // When the file is finished writing, resolve
             writeStream.on('finish', () => {
-              logger.info(`File "${fileInZip}" extracted to "${outputPath}".`);
+              logger.info(`DBGM-00088 File "${fileInZip}" extracted to "${outputPath}".`);
               resolve(true);
             });
 
             // Handle write errors
             writeStream.on('error', writeErr => {
-              logger.error(extractErrorLogData(writeErr), `Error extracting "${fileInZip}" from "${zipPath}".`);
+              logger.error(extractErrorLogData(writeErr), `DBGM-00089 Error extracting "${fileInZip}" from "${zipPath}".`);
               reject(writeErr);
             });
           });
@@ -67,7 +67,7 @@ function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
 
       // Handle general errors
       zipFile.on('error', err => {
-        logger.error(extractErrorLogData(err), `ZIP file error in ${zipPath}.`);
+        logger.error(extractErrorLogData(err), `DBGM-00172 ZIP file error in ${zipPath}.`);
         reject(err);
       });
     });

package/src/utility/loadModelTransform.js

@@ -28,7 +28,7 @@ async function loadModelTransform(file) {
     }
     return null;
   } catch (err) {
-    logger.error(extractErrorLogData(err), `Error loading model transform ${file}`);
+    logger.error(extractErrorLogData(err), `DBGM-00173 Error loading model transform ${file}`);
     return null;
   }
 }

package/src/utility/sshTunnel.js

@@ -40,7 +40,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
       tunnelConfig,
     });
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error connecting SSH');
+    logger.error(extractErrorLogData(err), 'DBGM-00174 Error connecting SSH');
   }
   return new Promise((resolve, reject) => {
     let promiseHandled = false;
@@ -57,18 +57,18 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
       }
     });
     subprocess.on('exit', code => {
-      logger.info(`SSH forward process exited with code ${code}`);
+      logger.info(`DBGM-00090 SSH forward process exited with code ${code}`);
       delete sshTunnelCache[tunnelCacheKey];
       if (!promiseHandled) {
         reject(
           new Error(
-            'SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'
+            'DBGM-00091 SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'
           )
         );
       }
     });
     subprocess.on('error', error => {
-      logger.error(extractErrorLogData(error), 'SSH forward process error');
+      logger.error(extractErrorLogData(error), 'DBGM-00092 SSH forward process error');
       delete sshTunnelCache[tunnelCacheKey];
       if (!promiseHandled) {
         reject(error);
@@ -97,13 +97,13 @@ async function getSshTunnel(connection) {
     };
     try {
       logger.info(
-        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+        `DBGM-00093 Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
 
       const subprocess = await callForwardProcess(connection, tunnelConfig, tunnelCacheKey);
 
       logger.info(
-        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+        `DBGM-00094 Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
 
       sshTunnelCache[tunnelCacheKey] = {
@@ -114,7 +114,7 @@ async function getSshTunnel(connection) {
       };
       return sshTunnelCache[tunnelCacheKey];
     } catch (err) {
-      logger.error(extractErrorLogData(err), 'Error creating SSH tunnel:');
+      logger.error(extractErrorLogData(err), 'DBGM-00095 Error creating SSH tunnel:');
       // error is not cached
       return {
         state: 'error',

package/src/utility/sshTunnelProxy.js

@@ -10,7 +10,7 @@ async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
   try {
     subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
   } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error sending to SSH tunnel');
+    logger.error(extractErrorLogData(err), 'DBGM-00175 Error sending to SSH tunnel');
   }
 }
 

package/src/utility/useController.js

@@ -12,11 +12,11 @@ module.exports = function useController(app, electron, route, controller) {
   const router = express.Router();
 
   if (controller._init) {
-    logger.info(`Calling init controller for controller ${route}`);
+    logger.info(`DBGM-00096 Calling init controller for controller ${route}`);
     try {
       controller._init();
     } catch (err) {
-      logger.error(extractErrorLogData(err), `Error initializing controller, exiting application`);
+      logger.error(extractErrorLogData(err), `DBGM-00097 Error initializing controller, exiting application`);
       process.exit(1);
     }
   }
@@ -78,7 +78,7 @@ module.exports = function useController(app, electron, route, controller) {
           const data = await controller[key]({ ...req.body, ...req.query }, req);
           res.json(data);
         } catch (err) {
-          logger.error(extractErrorLogData(err), `Error when processing route ${route}/${key}`);
+          logger.error(extractErrorLogData(err), `DBGM-00176 Error when processing route ${route}/${key}`);
           if (err instanceof MissingCredentialsError) {
             res.json({
               missingCredentials: true,