dbgate-api 4.7.3 → 4.7.4-alpha.12

package/src/main.js

@@ -29,6 +29,8 @@ const queryHistory = require('./controllers/queryHistory');
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -37,12 +39,11 @@ function start() {
 
   const server = http.createServer(app);
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
@@ -85,15 +86,11 @@ function start() {
   if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
-  } else {
-    if (!platformInfo.isNpmDist) {
-      app.get(getExpressPath('/'), (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
 
-  if (platformInfo.isNpmDist) {
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port', port);
+    server.listen(port);
+  } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({
       port: parseInt(
@@ -105,7 +102,19 @@ function start() {
         console.log(`DbGate API listening on port ${port}`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    console.log('__dirname', __dirname);
+    console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API & web listening on port', port);
+    server.listen(port);
   } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+
     const port = process.env.PORT || 3000;
     console.log('DbGate API listening on port', port);
     server.listen(port);

package/src/proc/connectProcess.js

@@ -20,7 +20,7 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {

package/src/proc/databaseConnectionProcess.js

@@ -7,6 +7,7 @@ const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
 
 let systemConnection;
 let storedConnection;
@@ -107,7 +108,7 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
   await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
@@ -154,6 +155,7 @@ async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     await driver.script(systemConnection, sql);
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
@@ -165,6 +167,7 @@ async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
@@ -172,21 +175,67 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
-async function handleCollectionData({ msgid, options }) {
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s });
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
-    const result = await driver.readCollection(systemConnection, options);
+    const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
+async function handleCollectionData({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
+  }
+}
+
 async function handleUpdateCollection({ msgid, changeSet }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const result = await driver.updateCollection(systemConnection, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
@@ -248,10 +297,16 @@ const messageHandlers = {
   runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
   generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
   // runCommand: handleRunCommand,
 };
 

package/src/proc/serverConnectionProcess.js

@@ -58,11 +58,14 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
-      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
     }
   } catch (err) {
     setStatus({
@@ -80,7 +83,7 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
   if (driver.createDatabase) {
     await driver.createDatabase(systemConnection, name);

package/src/proc/sessionProcess.js

@@ -17,11 +17,15 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +36,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -98,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -135,11 +166,22 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -163,6 +205,19 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
@@ -176,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

package/src/shell/executeQuery.js

@@ -5,7 +5,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   console.log(`Execute query ${sql}`);
 
   if (!driver) driver = requireEngineDriver(connection);
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
   console.log(`Connected.`);
 
   await driver.script(pool, sql);

package/src/shell/generateDeploySql.js

@@ -21,7 +21,7 @@ async function generateDeploySql({
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
   if (!analysedStructure) {
     analysedStructure = await driver.analyseFull(pool);
   }

package/src/shell/queryReader.js

@@ -1,14 +1,26 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function queryReader({ connection, sql }) {
-  console.log(`Reading query ${sql}`);
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  console.log(`Reading query ${query || sql}`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return await driver.readQuery(pool, sql);
+  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
 }
 
 module.exports = queryReader;

package/src/shell/tableReader.js

@@ -4,12 +4,12 @@ const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, 'read');
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
-  if (driver.dialect.nosql) {
+  if (driver.databaseEngineTypes.includes('document')) {
     // @ts-ignore
     console.log(`Reading collection ${fullNameToString(fullName)}`);
     // @ts-ignore

package/src/shell/tableWriter.js

@@ -8,7 +8,7 @@ async function tableWriter({ connection, schemaName, pureName, driver, systemCon
   if (!driver) {
     driver = requireEngineDriver(connection);
   }
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
 
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);

package/src/utility/connectUtility.js

@@ -4,11 +4,47 @@ const fs = require('fs-extra');
 const { decryptConnection } = require('./crypting');
 const { getSshTunnel } = require('./sshTunnel');
 const { getSshTunnelProxy } = require('./sshTunnelProxy');
+const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
+
+async function loadConnection(driver, storedConnection, connectionMode) {
+  const { allowShellConnection } = platformInfo;
+
+  if (connectionMode == 'app') {
+    return storedConnection;
+  }
+
+  if (storedConnection._id || !allowShellConnection) {
+    if (!storedConnection._id) {
+      throw new Error('Missing connection _id');
+    }
+
+    await connections._init();
+    const loaded = await connections.getCore({ conid: storedConnection._id });
+    const loadedWithDb = {
+      ...loaded,
+      database: storedConnection.database,
+    };
+
+    if (loaded.isReadOnly) {
+      if (connectionMode == 'read') return loadedWithDb;
+      if (connectionMode == 'write') throw new Error('Cannot write readonly connection');
+      if (connectionMode == 'script') {
+        if (driver.readOnlySessions) return loadedWithDb;
+        throw new Error('Cannot write readonly connection');
+      }
+    }
+    return loadedWithDb;
+  }
+  return storedConnection;
+}
+
+async function connectUtility(driver, storedConnection, connectionMode) {
+  const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
 
-async function connectUtility(driver, storedConnection) {
   const connection = {
-    database: storedConnection.defaultDatabase,
-    ...decryptConnection(storedConnection),
+    database: connectionLoaded.defaultDatabase,
+    ...decryptConnection(connectionLoaded),
   };
 
   if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();

package/src/utility/crypting.js

@@ -55,7 +55,7 @@ function encryptPasswordField(connection, field) {
       [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
     };
   }
-return connection;
+  return connection;
 }
 
 function decryptPasswordField(connection, field) {
@@ -75,6 +75,11 @@ function encryptConnection(connection) {
   return connection;
 }
 
+function maskConnection(connection) {
+  if (!connection) return connection;
+  return _.omit(connection, ['password', 'sshPassword', 'sshKeyfilePassword']);
+}
+
 function decryptConnection(connection) {
   connection = decryptPasswordField(connection, 'password');
   connection = decryptPasswordField(connection, 'sshPassword');
@@ -95,5 +100,6 @@ module.exports = {
   loadEncryptionKey,
   encryptConnection,
   decryptConnection,
+  maskConnection,
   pickSafeConnectionInfo,
 };

package/src/utility/directories.js

@@ -54,7 +54,7 @@ function packagedPluginsDir() {
   }
   if (platformInfo.isNpmDist) {
     // node_modules
-    return global['dbgateApiPackagedPluginsPath'];
+    return global['PLUGINS_DIR'];
   }
   if (platformInfo.isElectronBundle) {
     return path.resolve(__dirname, '../../plugins');

package/src/utility/hasPermission.js

@@ -1,12 +1,56 @@
 const { compilePermissions, testPermission } = require('dbgate-tools');
+const _ = require('lodash');
 
-let compiled = undefined;
+const userPermissions = {};
 
-function hasPermission(tested) {
-  if (compiled === undefined) {
-    compiled = compilePermissions(process.env.PERMISSIONS);
+function hasPermission(tested, req) {
+  const { user } = (req && req.auth) || {};
+  const key = user || '';
+  const logins = getLogins();
+  if (!userPermissions[key] && logins) {
+    const login = logins.find(x => x.login == user);
+    userPermissions[key] = compilePermissions(login ? login.permissions : null);
   }
-  return testPermission(tested, compiled);
+  return testPermission(tested, userPermissions[key]);
 }
 
-module.exports = hasPermission;
+let loginsCache = null;
+let loginsLoaded = false;
+
+function getLogins() {
+  if (loginsLoaded) {
+    return loginsCache;
+  }
+
+  const res = [];
+  if (process.env.LOGIN && process.env.PASSWORD) {
+    res.push({
+      login: process.env.LOGIN,
+      password: process.env.PASSWORD,
+      permissions: process.env.PERMISSIONS,
+    });
+  }
+  if (process.env.LOGINS) {
+    const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim()));
+    for (const login of logins) {
+      const password = process.env[`LOGIN_PASSWORD_${login}`];
+      const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
+      if (password) {
+        res.push({
+          login,
+          password,
+          permissions,
+        });
+      }
+    }
+  }
+
+  loginsCache = res.length > 0 ? res : null;
+  loginsLoaded = true;
+  return loginsCache;
+}
+
+module.exports = {
+  hasPermission,
+  getLogins,
+};

package/src/utility/platformInfo.js

@@ -10,7 +10,7 @@ const isMac = platform === 'darwin';
 const isLinux = platform === 'linux';
 const isDocker = fs.existsSync('/home/dbgate-docker/public');
 const isDevMode = process.env.DEVMODE == '1';
-const isNpmDist = !!global['dbgateApiModulePath'];
+const isNpmDist = !!global['IS_NPM_DIST'];
 const isForkedApi = processArgs.isForkedApi;
 
 // function moduleAvailable(name) {
@@ -39,6 +39,8 @@ const platformInfo = {
   environment: process.env.NODE_ENV,
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
+  allowShellConnection: !!process.env.SHELL_CONNECTION || !!isElectron(),
+  allowShellScripting: !!process.env.SHELL_SCRIPTING || !!isElectron(),
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
 };
 

package/src/utility/useController.js

@@ -62,7 +62,7 @@ module.exports = function useController(app, electron, route, controller) {
         //   controller._init_called = true;
         // }
         try {
-          let params = [{ ...req.body, ...req.query }];
+          let params = [{ ...req.body, ...req.query }, req];
           if (rawParams) params = [req, res];
           const data = await controller[key](...params);
           res.json(data);