dbgate-api 5.2.1 → 5.2.2-alpha.11

package/src/index.js

@@ -1,5 +1,96 @@
-const shell = require('./shell');
+const { setLogger, getLogger, setLoggerName } = require('dbgate-tools');
 const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const { createLogger } = require('pinomin');
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const logger = createLogger({
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  });
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  setLogger(logger);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+}
+
+const shell = require('./shell');
 const dbgateTools = require('dbgate-tools');
 
 global['DBGATE_TOOLS'] = dbgateTools;
@@ -8,7 +99,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} 
+}
 
 if (processArgs.listenApi) {
   const main = require('./main');
@@ -17,5 +108,8 @@ if (processArgs.listenApi) {
 
 module.exports = {
   ...shell,
+  getLogger,
+  configureLogger,
+  // loadLogsContent,
   getMainModule: () => require('./main'),
 };

package/src/main.js

@@ -33,6 +33,9 @@ const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
 const { getLogins } = require('./utility/hasPermission');
 const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('main');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -60,8 +63,8 @@ function start() {
   } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
   } else if (process.env.DEVWEB) {
-    console.log('__dirname', __dirname);
-    console.log(path.join(__dirname, '../../web/public/build'));
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
   } else {
     app.get(getExpressPath('/'), (req, res) => {
@@ -109,7 +112,7 @@ function start() {
 
   if (platformInfo.isDocker) {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (docker build)', port);
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
     server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
@@ -119,27 +122,27 @@ function start() {
       ),
     }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port} (NPM build)`);
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
   } else if (process.env.DEVWEB) {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API & web listening on port (dev web build)', port);
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
     server.listen(port);
   } else {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port (dev API build)', port);
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
     server.listen(port);
   }
 
   function shutdown() {
-    console.log('\nShutting down DbGate API server');
+    logger.info('\nShutting down DbGate API server');
     server.close(() => {
-      console.log('Server shut down, terminating');
+      logger.info('Server shut down, terminating');
       process.exit(0);
     });
     setTimeout(() => {
-      console.log('Server close timeout, terminating');
+      logger.info('Server close timeout, terminating');
       process.exit(0);
     }, 1000);
   }

package/src/proc/databaseConnectionProcess.js

@@ -1,7 +1,7 @@
 const stableStringify = require('json-stable-stringify');
 const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -9,6 +9,8 @@ const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { dumpSqlSelect } = require('dbgate-sqltree');
 
+const logger = getLogger('dbconnProcess');
+
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
@@ -269,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
       setTimeout(() => {
-        console.log('Exiting because of unhandled exception');
+        getLogger.info('Exiting because of unhandled exception');
         process.exit(0);
       }, 500);
     }
@@ -336,7 +338,7 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      console.log('Database connection not alive, exiting');
+      logger.info('Database connection not alive, exiting');
       process.exit(0);
     }
   }, 10 * 1000);
@@ -345,9 +347,9 @@ function start() {
     if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
-    } catch (e) {
-      console.error('Error in DB connection', e);
-      process.send({ msgtype: 'error', error: e.message });
+    } catch (err) {
+      logger.error({ err }, 'Error in DB connection');
+      process.send({ msgtype: 'error', error: err.message });
     }
   });
 }

package/src/proc/serverConnectionProcess.js

@@ -1,9 +1,10 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const logger = getLogger('srvconnProcess');
 
 let systemConnection;
 let storedConnection;
@@ -101,7 +102,7 @@ async function handleDatabaseOp(op, { name }) {
   } else {
     const dmp = driver.createDumper();
     dmp[op](name);
-    console.log(`RUNNING SCRIPT: ${dmp.s}`);
+    logger.info({ sql: dmp.s }, 'Running script');
     await driver.query(systemConnection, dmp.s);
   }
   await handleRefresh();
@@ -146,7 +147,7 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      console.log('Server connection not alive, exiting');
+      logger.info('Server connection not alive, exiting');
       process.exit(0);
     }
   }, 10 * 1000);

package/src/proc/sessionProcess.js

@@ -10,12 +10,16 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+
+const logger = getLogger('sessionProcess');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
 let lastPing = null;
+let lastActivity = null;
 let currentProfiler = null;
 
 class TableWriter {
@@ -212,6 +216,8 @@ function waitConnected() {
 }
 
 async function handleStartProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
@@ -230,6 +236,8 @@ async function handleStartProfiler({ jslid }) {
 }
 
 async function handleStopProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
   const driver = requireEngineDriver(storedConnection);
   currentProfiler.writer.close();
   driver.stopProfiler(systemConnection, currentProfiler);
@@ -237,6 +245,8 @@ async function handleStopProfiler({ jslid }) {
 }
 
 async function handleExecuteQuery({ sql }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
@@ -270,6 +280,8 @@ async function handleExecuteQuery({ sql }) {
 }
 
 async function handleExecuteReader({ jslid, sql, fileName }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
 
   const driver = requireEngineDriver(storedConnection);
@@ -325,7 +337,20 @@ function start() {
   setInterval(() => {
     const time = new Date().getTime();
     if (time - lastPing > 25 * 1000) {
-      console.log('Session not alive, exiting');
+      logger.info('Session not alive, exiting');
+      process.exit(0);
+    }
+
+    const useSessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractBoolSettingsValue(storedConnection.globalSettings, 'session.autoClose', true)
+        : false;
+    const sessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractIntSettingsValue(storedConnection.globalSettings, 'session.autoCloseTimeout', 15, 1, 120)
+        : 15;
+    if (useSessionTimeout && time - lastActivity > sessionTimeout * 60 * 1000 && !currentProfiler) {
+      logger.info('Session not active, exiting');
       process.exit(0);
     }
   }, 10 * 1000);

package/src/proc/sshForwardProcess.js

@@ -3,6 +3,9 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('sshProcess');
 
 async function getSshConnection(connection) {
   const sshConfig = {
@@ -35,8 +38,8 @@ async function handleStart({ connection, tunnelConfig }) {
       tunnelConfig,
     });
   } catch (err) {
-    console.log('Error creating SSH tunnel connection:', err.message);
-    
+    logger.error({ err }, 'Error creating SSH tunnel connection:');
+
     process.send({
       msgtype: 'error',
       connection,

package/src/shell/archiveWriter.js

@@ -3,11 +3,14 @@ const fs = require('fs');
 const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger();
 
 function archiveWriter({ folderName, fileName }) {
   const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
-    console.log(`Creating directory ${dir}`);
+    logger.info(`Creating directory ${dir}`);
     fs.mkdirSync(dir);
   }
   const jsonlFile = path.join(dir, `${fileName}.jsonl`);

package/src/shell/dumpDatabase.js

@@ -1,5 +1,8 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('dumpDb');
 
 function doDump(dumper) {
   return new Promise((resolve, reject) => {
@@ -21,11 +24,11 @@ async function dumpDatabase({
   databaseName,
   schemaName,
 }) {
-  console.log(`Dumping database`);
+  logger.info(`Dumping database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   const dumper = await driver.createBackupDumper(pool, {
     outputFile,

package/src/shell/executeQuery.js

@@ -1,12 +1,15 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('execQuery');
 
 async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  console.log(`Execute query ${sql}`);
+  logger.info({ sql }, `Execute query`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   await driver.script(pool, sql);
 }

package/src/shell/generateModelSql.js

@@ -0,0 +1,30 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const {
+  extendDatabaseInfo,
+  databaseInfoFromYamlModel,
+  getAlterDatabaseScript,
+  DatabaseAnalyser,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function generateModelSql({ engine, driver, modelFolder, loadedDbModel, outputFile }) {
+  if (!driver) driver = requireEngineDriver(engine);
+
+  const dbInfo = extendDatabaseInfo(
+    loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder)
+  );
+
+  const { sql } = getAlterDatabaseScript(
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    {},
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    driver
+  );
+
+  fs.writeFileSync(outputFile, sql);
+}
+
+module.exports = generateModelSql;

package/src/shell/importDatabase.js

@@ -4,6 +4,9 @@ const connectUtility = require('../utility/connectUtility');
 const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
 const download = require('./download');
 const stream = require('stream');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('importDb');
 
 class ImportStream extends stream.Transform {
   constructor(pool, driver) {
@@ -38,11 +41,11 @@ function awaitStreamEnd(stream) {
 }
 
 async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
-  console.log(`Importing database`);
+  logger.info(`Importing database`);
 
   if (!driver) driver = requireEngineDriver(connection);
   const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
 
   const downloadedFile = await download(inputFile);
 

package/src/shell/index.js

@@ -23,6 +23,8 @@ const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 const dumpDatabase = require('./dumpDatabase');
 const importDatabase = require('./importDatabase');
+const loadDatabase = require('./loadDatabase');
+const generateModelSql = require('./generateModelSql');
 
 const dbgateApi = {
   queryReader,
@@ -49,6 +51,8 @@ const dbgateApi = {
   initializeApiEnvironment,
   dumpDatabase,
   importDatabase,
+  loadDatabase,
+  generateModelSql,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/jsonArrayWriter.js

@@ -1,6 +1,9 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
 
+const logger = getLogger('jsonArrayWriter');
+
 class StringifyStream extends stream.Transform {
   constructor() {
     super({ objectMode: true });
@@ -38,7 +41,7 @@ class StringifyStream extends stream.Transform {
 }
 
 async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream();
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);

package/src/shell/jsonLinesReader.js

@@ -1,6 +1,8 @@
 const fs = require('fs');
 const stream = require('stream');
 const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('jsonLinesReader');
 
 class ParseStream extends stream.Transform {
   constructor({ limitRows }) {
@@ -31,7 +33,7 @@ class ParseStream extends stream.Transform {
 }
 
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
-  console.log(`Reading file ${fileName}`);
+  logger.info(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);

package/src/shell/jsonLinesWriter.js

@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
+const logger = getLogger('jsonLinesWriter');
 
 class StringifyStream extends stream.Transform {
   constructor({ header }) {
@@ -21,7 +23,7 @@ class StringifyStream extends stream.Transform {
 }
 
 async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ header });
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);

package/src/shell/loadDatabase.js

@@ -0,0 +1,21 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const exportDbModel = require('../utility/exportDbModel');
+
+const logger = getLogger('analyseDb');
+
+async function loadDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, outputDir }) {
+  logger.info(`Analysing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  logger.info(`Connected.`);
+
+  const dbInfo = await driver.analyseFull(pool);
+  logger.info(`Analyse finished`);
+
+  await exportDbModel(dbInfo, outputDir);
+}
+
+module.exports = loadDatabase;

package/src/shell/queryReader.js

@@ -1,5 +1,7 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('queryReader');
 
 async function queryReader({
   connection,
@@ -14,12 +16,12 @@ async function queryReader({
   // if (!sql && !json) {
   //   throw new Error('One of sql or json must be set');
   // }
-  console.log(`Reading query ${query || sql}`);
+  logger.info({ sql: query || sql }, `Reading query`);
   // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
   const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
-  console.log(`Connected.`);
+  logger.info(`Connected.`);
   const reader =
     queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
   return reader;

package/src/shell/requirePlugin.js

@@ -3,6 +3,8 @@ const fs = require('fs');
 const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
 const platformInfo = require('../utility/platformInfo');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('requirePlugin');
 
 const loadedPlugins = {};
 
@@ -17,7 +19,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
   if (requiredPlugin == null) {
     let module;
     const modulePath = getPluginBackendPath(packageName);
-    console.log(`Loading module ${packageName} from ${modulePath}`);
+    logger.info(`Loading module ${packageName} from ${modulePath}`);
     try {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);

package/src/shell/runScript.js

@@ -1,5 +1,7 @@
+const { getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const processArgs = require('../utility/processArgs');
+const logger = getLogger();
 
 async function runScript(func) {
   if (processArgs.checkParent) {
@@ -9,7 +11,7 @@ async function runScript(func) {
     await func();
     process.exit(0);
   } catch (err) {
-    console.log(err);
+    logger.error('Error running script', err);
     process.exit(1);
   }
 }

package/src/shell/sqlDataWriter.js

@@ -1,8 +1,9 @@
 const fs = require('fs');
 const stream = require('stream');
 const path = require('path');
-const { driverBase } = require('dbgate-tools');
+const { driverBase, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const logger = getLogger('sqlDataWriter');
 
 class SqlizeStream extends stream.Transform {
   constructor({ fileName, dataName }) {
@@ -40,7 +41,7 @@ class SqlizeStream extends stream.Transform {
 }
 
 async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new SqlizeStream({ fileName, dataName });
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);