dbgate-api 5.2.1 → 5.2.2-alpha.11

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.2.1",
+  "version": "5.2.2-alpha.11",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -27,8 +27,8 @@
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
     "dbgate-query-splitter": "^4.9.3",
-    "dbgate-sqltree": "^5.2.1",
-    "dbgate-tools": "^5.2.1",
+    "dbgate-sqltree": "^5.2.2-alpha.11",
+    "dbgate-tools": "^5.2.2-alpha.11",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -49,6 +49,7 @@
     "ncp": "^2.0.0",
     "node-cron": "^2.0.3",
     "on-finished": "^2.4.1",
+    "pinomin": "^1.0.1",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
@@ -69,7 +70,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.2.1",
+    "dbgate-types": "^5.2.2-alpha.11",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
@@ -79,7 +80,7 @@
   },
   "optionalDependencies": {
     "better-sqlite3": "7.6.2",
-    "oracledb": "^5.5.0",
-    "msnodesqlv8": "^2.6.0"
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

package/src/controllers/archive.js

@@ -6,6 +6,9 @@ const socket = require('../utility/socket');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('archive');
 
 module.exports = {
   folders_meta: true,
@@ -68,7 +71,7 @@ module.exports = {
         ...fileType('.matview.sql', 'matview.sql'),
       ];
     } catch (err) {
-      console.log('Error reading archive files', err.message);
+      logger.error({ err }, 'Error reading archive files');
       return [];
     }
   },

package/src/controllers/auth.js

@@ -3,8 +3,11 @@ const jwt = require('jsonwebtoken');
 const getExpressPath = require('../utility/getExpressPath');
 const uuidv1 = require('uuid/v1');
 const { getLogins } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
 const AD = require('activedirectory2').promiseWrapper;
 
+const logger = getLogger('auth');
+
 const tokenSecret = uuidv1();
 
 function shouldAuthorizeApi() {
@@ -51,7 +54,7 @@ function authMiddleware(req, res, next) {
       return next();
     }
 
-    console.log('Sending invalid token error', err.message);
+    logger.error({ err }, 'Sending invalid token error');
 
     return unauthorizedResponse(req, res, 'invalid token');
   }
@@ -74,7 +77,7 @@ module.exports = {
 
     const payload = jwt.decode(access_token);
 
-    console.log('User payload returned from OAUTH:', payload);
+    logger.info({ payload }, 'User payload returned from OAUTH');
 
     const login =
       process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
@@ -122,7 +125,7 @@ module.exports = {
           accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
         };
       } catch (err) {
-        console.log('Failed active directory authentization', err.message);
+        logger.error({ err }, 'Failed active directory authentization');
         return {
           error: err.message,
         };

package/src/controllers/config.js

@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 const os = require('os');
 const path = require('path');
 const axios = require('axios');
-const { datadir } = require('../utility/directories');
+const { datadir, getLogsFilePath } = require('../utility/directories');
 const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
@@ -48,6 +48,8 @@ module.exports = {
       oauthScope: process.env.OAUTH_SCOPE,
       oauthLogout: process.env.OAUTH_LOGOUT,
       isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      logsFilePath: getLogsFilePath(),
+      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
   },

package/src/controllers/connections.js

@@ -12,9 +12,12 @@ const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
-const { safeJsonParse } = require('dbgate-tools');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
 const platformInfo = require('../utility/platformInfo');
 const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('connections');
 
 let volatileConnections = {};
 
@@ -86,13 +89,13 @@ function getPortalCollections() {
       sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
       sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
-    console.log('Using connections from ENV variables:');
-    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+
+    logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables');
     const noengine = connections.filter(x => !x.engine);
     if (noengine.length > 0) {
-      console.log(
-        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
-        noengine.map(x => x._id)
+      logger.warn(
+        { connections: noengine.map(x => x._id) },
+        'Invalid CONNECTIONS configutation, missing ENGINE for connection ID'
       );
     }
     return connections;
@@ -203,13 +206,20 @@ module.exports = {
 
   test_meta: true,
   test(connection) {
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'connectProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'connectProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     subprocess.send(connection);
     return new Promise(resolve => {
       subprocess.on('message', resp => {

package/src/controllers/databaseConnections.js

@@ -12,6 +12,7 @@ const {
   matchPairedObjects,
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
+  getLogger,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -28,6 +29,9 @@ const diff2htmlPage = require('../utility/diff2htmlPage');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('databaseConnections');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -60,7 +64,7 @@ module.exports = {
 
   handle_error(conid, database, props) {
     const { error } = props;
-    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+    logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
   },
   handle_response(conid, database, { msgid, ...response }) {
     const [resolve, reject] = this.requests[msgid];
@@ -85,13 +89,20 @@ module.exports = {
     if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
       throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
     }
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'databaseConnectionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'databaseConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
@@ -129,7 +140,12 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request do process');
+        this.close(conn.conid, conn.database);
+      }
     });
     return promise;
   },
@@ -137,7 +153,7 @@ module.exports = {
   queryData_meta: true,
   async queryData({ conid, database, sql }, req) {
     testConnectionPermission(conid, req);
-    console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing query');
     const opened = await this.ensureOpened(conid, database);
     // if (opened && opened.status && opened.status.name == 'error') {
     //   return opened.status;
@@ -157,7 +173,7 @@ module.exports = {
   runScript_meta: true,
   async runScript({ conid, database, sql }, req) {
     testConnectionPermission(conid, req);
-    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    logger.info({ conid, database, sql }, 'Processing script');
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
     return res;
@@ -278,6 +294,7 @@ module.exports = {
     if (existing) {
       existing.subprocess.send({ msgtype: 'ping' });
     } else {
+      // @ts-ignore
       existing = await this.ensureOpened(conid, database);
     }
 
@@ -308,7 +325,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
       this.closed[`${conid}/${database}`] = {
         status: {

package/src/controllers/runners.js

@@ -6,10 +6,17 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+} = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -29,13 +36,14 @@ const requirePluginsTemplate = (plugins, isExport) =>
 
 const scriptTemplate = (script, isExport) => `
 const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
 dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
 await dbgateApi.finalizer.run();
-console.log('Finished job script');
+logger.info('Finished job script');
 }
 dbgateApi.runScript(run);
 `;
@@ -59,19 +67,17 @@ module.exports = {
   requests: {},
 
   dispatchMessage(runid, message) {
-    if (message) console.log('...', message.message);
-    if (_.isString(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        message,
-        time: new Date(),
-        severity: 'info',
-      });
-    }
-    if (_.isPlainObject(message)) {
+    if (message) {
+      const json = safeJsonParse(message.message);
+
+      if (json) logger.info(json);
+      else logger.info(message.message);
+
       socket.emit(`runner-info-${runid}`, {
         time: new Date(),
         severity: 'info',
         ...message,
+        message: json ? json.msg : message.message,
       });
     }
   },
@@ -98,13 +104,15 @@ module.exports = {
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
-    console.log(`RUNNING SCRIPT ${scriptFile}`);
+    logger.info({ scriptFile }, 'Running script');
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
     const subprocess = fork(
       scriptFile,
       [
         '--checkParent', // ...process.argv.slice(3)
         '--is-forked-api',
+        '--process-display-name',
+        'script',
         ...processArgs.getPassArgs(),
       ],
       {
@@ -124,7 +132,7 @@ module.exports = {
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
       this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
-      console.log('... EXIT process', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
     });
     subprocess.on('error', error => {

package/src/controllers/scheduler.js

@@ -4,6 +4,9 @@ const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
 const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -21,7 +24,7 @@ module.exports = {
     if (!match) return;
     const pattern = match[1];
     if (!cron.validate(pattern)) return;
-    console.log(`Schedule script ${file} with pattern ${pattern}`);
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
     const task = cron.schedule(pattern, () => runners.start({ script: text }));
     this.tasks.push(task);
   },

package/src/controllers/serverConnections.js

@@ -10,6 +10,10 @@ const config = require('./config');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('serverConnection');
 
 module.exports = {
   opened: [],
@@ -53,13 +57,20 @@ module.exports = {
       if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
         throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
       }
-      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'serverConnectionProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'serverConnectionProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(subprocess);
       const newOpened = {
         conid,
         subprocess,
@@ -94,7 +105,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid);
       this.closed[conid] = {
         ...existing.status,
@@ -144,7 +161,12 @@ module.exports = {
         }
         this.lastPinged[conid] = new Date().getTime();
         const opened = await this.ensureOpened(conid);
-        opened.subprocess.send({ msgtype: 'ping' });
+        try {
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error({ err }, 'Error calling ping');
+          this.close(conid);
+        }
       })
     );
     return { status: 'ok' };
@@ -181,7 +203,12 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request');
+        this.close(conn.conid);
+      }
     });
     return promise;
   },

package/src/controllers/sessions.js

@@ -8,6 +8,11 @@ const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const { appdir } = require('../utility/directories');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const config = require('./config');
+
+const logger = getLogger('sessions');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -82,13 +87,20 @@ module.exports = {
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.getCore({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'sessionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'sessionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const newOpened = {
       conid,
       database,
@@ -109,7 +121,12 @@ module.exports = {
       socket.emit(`session-closed-${sesid}`);
     });
 
-    subprocess.send({ msgtype: 'connect', ...connection, database });
+    subprocess.send({
+      msgtype: 'connect',
+      ...connection,
+      database,
+      globalSettings: await config.getSettings(),
+    });
     return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
@@ -120,7 +137,7 @@ module.exports = {
       throw new Error('Invalid session');
     }
 
-    console.log(`Processing query, sesid=${sesid}, sql=${sql}`);
+    logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({ msgtype: 'executeQuery', sql });
 
@@ -158,7 +175,7 @@ module.exports = {
       throw new Error('Invalid session');
     }
 
-    console.log(`Starting profiler, sesid=${sesid}`);
+    logger.info({ sesid }, 'Starting profiler');
     session.loadingReader_jslid = jslid;
     session.subprocess.send({ msgtype: 'startProfiler', jslid });
 

package/src/controllers/uploads.js

@@ -1,6 +1,8 @@
 const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('uploads');
 
 module.exports = {
   upload_meta: {
@@ -15,7 +17,7 @@ module.exports = {
     }
     const uploadName = uuidv1();
     const filePath = path.join(uploadsdir(), uploadName);
-    console.log(`Uploading file ${data.name}, size=${data.size}`);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);
 
     data.mv(filePath, () => {
       res.json({

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.2.1',
-  buildTime: '2023-01-06T17:06:00.024Z'
+  version: '5.2.2-alpha.11',
+  buildTime: '2023-01-28T19:21:55.036Z'
 };