dbgate-api-premium 5.5.7-alpha.45

package/src/controllers/uploads.js

@@ -0,0 +1,133 @@
+const crypto = require('crypto');
+const path = require('path');
+const { uploadsdir, getLogsFilePath } = require('../utility/directories');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('uploads');
+const axios = require('axios');
+const os = require('os');
+const fs = require('fs/promises');
+const { read } = require('./queryHistory');
+const platformInfo = require('../utility/platformInfo');
+const _ = require('lodash');
+const serverConnections = require('./serverConnections');
+const config = require('./config');
+const gistSecret = require('../gistSecret');
+const currentVersion = require('../currentVersion');
+
+module.exports = {
+  upload_meta: {
+    method: 'post',
+    raw: true,
+  },
+  upload(req, res) {
+    const { data } = req.files || {};
+    if (!data) {
+      res.json(null);
+      return;
+    }
+    const uploadName = crypto.randomUUID();
+    const filePath = path.join(uploadsdir(), uploadName);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);
+
+    data.mv(filePath, () => {
+      res.json({
+        originalName: data.name,
+        uploadName,
+        filePath,
+      });
+    });
+  },
+
+  get_meta: {
+    method: 'get',
+    raw: true,
+  },
+  get(req, res) {
+    res.sendFile(path.join(uploadsdir(), req.query.file));
+  },
+
+  async getGistToken() {
+    const settings = await config.getSettings();
+
+    return settings['other.gistCreateToken'] || gistSecret;
+  },
+
+  uploadErrorToGist_meta: true,
+  async uploadErrorToGist() {
+    const logs = await fs.readFile(getLogsFilePath(), { encoding: 'utf-8' });
+    const connections = await serverConnections.getOpenedConnectionReport();
+    try {
+      const response = await axios.default.post(
+        'https://api.github.com/gists',
+        {
+          description: `DbGate ${currentVersion.version} error report`,
+          public: false,
+          files: {
+            'logs.jsonl': {
+              content: logs,
+            },
+            'os.json': {
+              content: JSON.stringify(
+                {
+                  release: os.release(),
+                  arch: os.arch(),
+                  machine: os.machine(),
+                  platform: os.platform(),
+                  type: os.type(),
+                },
+                null,
+                2
+              ),
+            },
+            'platform.json': {
+              content: JSON.stringify(
+                _.omit(
+                  {
+                    ...platformInfo,
+                  },
+                  ['defaultKeyfile', 'sshAuthSock']
+                ),
+                null,
+                2
+              ),
+            },
+            'connections.json': {
+              content: JSON.stringify(connections, null, 2),
+            },
+            'version.json': {
+              content: JSON.stringify(currentVersion, null, 2),
+            },
+          },
+        },
+        {
+          headers: {
+            Authorization: `token ${await this.getGistToken()}`,
+            'Content-Type': 'application/json',
+            Accept: 'application/vnd.github.v3+json',
+          },
+        }
+      );
+
+      return response.data;
+    } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error uploading gist');
+
+      return {
+        apiErrorMessage: err.message,
+      };
+      // console.error('Error creating gist:', error.response ? error.response.data : error.message);
+    }
+  },
+
+  deleteGist_meta: true,
+  async deleteGist({ url }) {
+    const response = await axios.default.delete(url, {
+      headers: {
+        Authorization: `token ${await this.getGistToken()}`,
+        'Content-Type': 'application/json',
+        Accept: 'application/vnd.github.v3+json',
+      },
+    });
+    return true;
+  },
+};

package/src/currentVersion.js

@@ -0,0 +1,5 @@
+
+module.exports = { 
+  version: '5.5.7-alpha.45',
+  buildTime: '2024-11-14T07:29:43.540Z'
+};

package/src/gistSecret.js

@@ -0,0 +1,2 @@
+
+module.exports = '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';

package/src/index.js

@@ -0,0 +1,139 @@
+const { setLogConfig, getLogger, setLoggerName, extractErrorLogData } = require('dbgate-tools');
+const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const currentVersion = require('./currentVersion');
+
+const logger = getLogger('apiIndex');
+
+process.on('uncaughtException', err => {
+  logger.fatal(extractErrorLogData(err), 'Uncaught exception, exiting process');
+  process.exit(1);
+});
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const consoleLogLevel = process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info';
+  const fileLogLevel = process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'debug';
+
+  const logConfig = {
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: consoleLogLevel,
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: fileLogLevel,
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  };
+
+  // logger.info(`Initialized logging, console log level: ${consoleLogLevel}, file log level: ${fileLogLevel}`);
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  // @ts-ignore
+  setLogConfig(logConfig);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+  logger.info(`Starting API process version ${currentVersion.version}`);
+
+  if (process.env.DEBUG_PRINT_ENV_VARIABLES) {
+    logger.info('Debug print environment variables:');
+    for (const key of Object.keys(process.env)) {
+      logger.info(`  ${key}: ${JSON.stringify(process.env[key])}`);
+    }
+  }
+}
+
+const shell = require('./shell/index');
+
+global.DBGATE_PACKAGES = {
+  'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+};
+
+if (processArgs.startProcess) {
+  const proc = require('./proc');
+  const module = proc[processArgs.startProcess];
+  module.start();
+}
+
+if (processArgs.listenApi) {
+  const main = require('./main');
+  main.start();
+}
+
+module.exports = {
+  ...shell,
+  getLogger,
+  configureLogger,
+  currentVersion,
+  // loadLogsContent,
+  getMainModule: () => require('./main'),
+};

package/src/main.js

@@ -0,0 +1,202 @@
+const express = require('express');
+const basicAuth = require('express-basic-auth');
+const bodyParser = require('body-parser');
+const fileUpload = require('express-fileupload');
+const http = require('http');
+const cors = require('cors');
+const getPort = require('get-port');
+const path = require('path');
+
+const useController = require('./utility/useController');
+const socket = require('./utility/socket');
+
+const connections = require('./controllers/connections');
+const serverConnections = require('./controllers/serverConnections');
+const databaseConnections = require('./controllers/databaseConnections');
+const metadata = require('./controllers/metadata');
+const sessions = require('./controllers/sessions');
+const runners = require('./controllers/runners');
+const jsldata = require('./controllers/jsldata');
+const config = require('./controllers/config');
+const storage = require('./controllers/storage');
+const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
+const uploads = require('./controllers/uploads');
+const plugins = require('./controllers/plugins');
+const files = require('./controllers/files');
+const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');
+const onFinished = require('on-finished');
+
+const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const getExpressPath = require('./utility/getExpressPath');
+const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
+const { getDefaultAuthProvider } = require('./auth/authProvider');
+const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
+
+const logger = getLogger('main');
+
+function start() {
+  // console.log('process.argv', process.argv);
+
+  const app = express();
+
+  const server = http.createServer(app);
+
+  if (process.env.BASIC_AUTH && !process.env.STORAGE_DATABASE) {
+    async function authorizer(username, password, cb) {
+      try {
+        const resp = await getDefaultAuthProvider().login(username, password);
+        if (resp.accessToken) {
+          cb(null, true);
+        } else {
+          cb(null, false);
+        }
+      } catch (err) {
+        cb(err, false);
+      }
+    }
+    app.use(
+      basicAuth({
+        authorizer,
+        authorizeAsync: true,
+        challenge: true,
+        realm: 'DbGate Web App',
+      })
+    );
+  }
+
+  app.use(cors());
+
+  if (platformInfo.isDocker) {
+    // server static files inside docker container
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+  } else if (process.env.DEVWEB) {
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+  } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+  }
+
+  app.use(auth.authMiddleware);
+
+  app.get(getExpressPath('/stream'), async function (req, res) {
+    const strmid = req.query.strmid;
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.addSseResponse(res, strmid);
+    onFinished(req, () => {
+      socket.removeSseResponse(strmid);
+    });
+  });
+
+  app.use(bodyParser.json({ limit: '50mb' }));
+
+  app.use(
+    getExpressPath('/uploads'),
+    fileUpload({
+      limits: { fileSize: 4 * 1024 * 1024 },
+    })
+  );
+
+  useAllControllers(app, null);
+
+  // if (process.env.PAGES_DIRECTORY) {
+  //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
+  // }
+
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
+
+  if (platformInfo.isDocker) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
+    server.listen(port);
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (AWS AMI build)`);
+    server.listen(port);
+  } else if (platformInfo.isNpmDist) {
+    getPort({
+      port: parseInt(
+        // @ts-ignore
+        process.env.PORT || 3000
+      ),
+    }).then(port => {
+      server.listen(port, () => {
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
+      });
+    });
+  } else if (process.env.DEVWEB) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
+    server.listen(port);
+  } else {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
+    server.listen(port);
+  }
+
+  function shutdown() {
+    logger.info('\nShutting down DbGate API server');
+    server.close(() => {
+      logger.info('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      logger.info('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
+
+  if (process.env.CLOUD_UPGRADE_FILE) {
+    startCloudUpgradeTimer();
+  }
+}
+
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/storage', storage);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
+}
+
+function setElectronSender(electronSender) {
+  socket.setElectronSender(electronSender);
+}
+
+module.exports = { start, useAllControllers, setElectronSender, configController: config };

package/src/packagedPluginsContent.js

@@ -0,0 +1 @@
+module.exports = () => null;

package/src/proc/connectProcess.js

@@ -0,0 +1,38 @@
+const childProcessChecker = require('../utility/childProcessChecker');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const _ = require('lodash');
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;
+
+function start() {
+  childProcessChecker();
+  process.on('message', async connection => {
+    if (handleProcessCommunication(connection)) return;
+    try {
+      const driver = requireEngineDriver(connection);
+      const dbhan = await connectUtility(driver, connection, 'app');
+      const res = await driver.getVersion(dbhan);
+      process.send({ msgtype: 'connected', ...res });
+      await driver.close(dbhan);
+    } catch (e) {
+      console.error(e);
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
+    }
+  });
+}
+
+module.exports = { start };