dbgate-api 4.7.4-alpha.1 → 4.7.4-alpha.7

package/env/singledb/.env

@@ -8,6 +8,8 @@ USER_mysql=root
 PASSWORD_mysql=test
 PORT_mysql=3307
 ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
 
 SINGLE_CONNECTION=mysql
 SINGLE_DATABASE=Chinook

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.7.4-alpha.1",
+  "version": "4.7.4-alpha.7",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.7.4-alpha.1",
-    "dbgate-sqltree": "^4.7.4-alpha.1",
-    "dbgate-tools": "^4.7.4-alpha.1",
+    "dbgate-query-splitter": "^4.7.4-alpha.7",
+    "dbgate-sqltree": "^4.7.4-alpha.7",
+    "dbgate-tools": "^4.7.4-alpha.7",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.7.4-alpha.1",
+    "dbgate-types": "^4.7.4-alpha.7",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/config.js

@@ -33,6 +33,8 @@ module.exports = {
       runAsPortal: !!connections.portalConnections,
       singleDatabase: connections.singleDatabase,
       hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellConnection,
       permissions,
       ...currentVersion,
     };

package/src/controllers/connections.js

@@ -5,12 +5,14 @@ const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
+const { safeJsonParse } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
 
 function getNamedArgs() {
   const res = {};
@@ -56,6 +58,7 @@ function getPortalCollections() {
       singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
       isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],
@@ -163,7 +166,9 @@ module.exports = {
 
   list_meta: true,
   async list() {
-    return portalConnections || this.datastore.find();
+    return portalConnections && !platformInfo.allowShellConnection
+      ? portalConnections.map(maskConnection)
+      : this.datastore.find();
   },
 
   test_meta: true,
@@ -242,14 +247,21 @@ module.exports = {
     return res;
   },
 
-  get_meta: true,
-  async get({ conid }) {
+  async getCore({ conid, mask = false }) {
     if (!conid) return null;
-    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
     const res = await this.datastore.get(conid);
     return res || null;
   },
 
+  get_meta: true,
+  async get({ conid }) {
+    return this.getCore({ conid, mask: true });
+  },
+
   newSqliteDatabase_meta: true,
   async newSqliteDatabase({ file }) {
     const sqliteDir = path.join(filesdir(), 'sqlite');

package/src/controllers/databaseConnections.js

@@ -79,7 +79,7 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
+    const connection = await connections.getCore({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
       '--is-forked-api',
       '--start-process',
@@ -392,8 +392,8 @@ module.exports = {
     const targetDb = generateDbPairingId(
       extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
     );
-    // const sourceConnection = await connections.get({conid:sourceConid})
-    const connection = await connections.get({ conid: targetConid });
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
     const driver = requireEngineDriver(connection);
     const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
     const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);

package/src/controllers/plugins.js

@@ -73,7 +73,7 @@ module.exports = {
     const res = [];
     for (const packageName of _.union(files1, files2)) {
       if (packageName == 'dist') continue;
-      // if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
           const manifest = {

package/src/controllers/runners.js

@@ -6,9 +6,10 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -149,11 +150,17 @@ module.exports = {
 
   start_meta: true,
   async start({ script }) {
-    if (process.env.DISABLE_SHELL) {
-      return { errorMessage: 'Shell is disabled' };
+    const runid = uuidv1();
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
     }
 
-    const runid = uuidv1();
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

package/src/controllers/serverConnections.js

@@ -37,7 +37,7 @@ module.exports = {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
+      const connection = await connections.getCore({ conid });
       const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
         '--is-forked-api',
         '--start-process',

package/src/controllers/sessions.js

@@ -54,6 +54,9 @@ module.exports = {
       this.dispatchMessage(sesid, 'Query execution finished');
     }
     const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
     if (session.killOnDone) {
       this.kill({ sesid });
     }
@@ -78,7 +81,7 @@ module.exports = {
   create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
+    const connection = await connections.getCore({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
       '--is-forked-api',
       '--start-process',
@@ -124,6 +127,7 @@ module.exports = {
     const session = this.opened.find(x => x.sesid == sesid);
     session.killOnDone = true;
     const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
     const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
 
     session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
@@ -131,6 +135,15 @@ module.exports = {
     return { jslid };
   },
 
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.7.4-alpha.1',
-  buildTime: '2022-03-17T18:49:14.355Z'
+  version: '4.7.4-alpha.7',
+  buildTime: '2022-03-20T12:32:18.367Z'
 };

package/src/proc/connectProcess.js

@@ -20,7 +20,7 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {

package/src/proc/databaseConnectionProcess.js

@@ -108,7 +108,7 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
   await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;

package/src/proc/serverConnectionProcess.js

@@ -58,11 +58,14 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
-      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
     }
   } catch (err) {
     setStatus({
@@ -80,7 +83,7 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
   if (driver.createDatabase) {
     await driver.createDatabase(systemConnection, name);

package/src/proc/sessionProcess.js

@@ -181,7 +181,7 @@ async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }

package/src/shell/executeQuery.js

@@ -5,7 +5,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   console.log(`Execute query ${sql}`);
 
   if (!driver) driver = requireEngineDriver(connection);
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
   console.log(`Connected.`);
 
   await driver.script(pool, sql);

package/src/shell/generateDeploySql.js

@@ -21,7 +21,7 @@ async function generateDeploySql({
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
   if (!analysedStructure) {
     analysedStructure = await driver.analyseFull(pool);
   }

package/src/shell/queryReader.js

@@ -1,14 +1,26 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function queryReader({ connection, sql }) {
-  console.log(`Reading query ${sql}`);
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  console.log(`Reading query ${query || sql}`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return await driver.readQuery(pool, sql);
+  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
 }
 
 module.exports = queryReader;

package/src/shell/tableReader.js

@@ -4,7 +4,7 @@ const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, 'read');
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };

package/src/shell/tableWriter.js

@@ -8,7 +8,7 @@ async function tableWriter({ connection, schemaName, pureName, driver, systemCon
   if (!driver) {
     driver = requireEngineDriver(connection);
   }
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
 
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);

package/src/utility/connectUtility.js

@@ -4,11 +4,47 @@ const fs = require('fs-extra');
 const { decryptConnection } = require('./crypting');
 const { getSshTunnel } = require('./sshTunnel');
 const { getSshTunnelProxy } = require('./sshTunnelProxy');
+const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
+
+async function loadConnection(driver, storedConnection, connectionMode) {
+  const { allowShellConnection } = platformInfo;
+
+  if (connectionMode == 'app') {
+    return storedConnection;
+  }
+
+  if (storedConnection._id || !allowShellConnection) {
+    if (!storedConnection._id) {
+      throw new Error('Missing connection _id');
+    }
+
+    await connections._init();
+    const loaded = await connections.getCore({ conid: storedConnection._id });
+    const loadedWithDb = {
+      ...loaded,
+      database: storedConnection.database,
+    };
+
+    if (loaded.isReadOnly) {
+      if (connectionMode == 'read') return loadedWithDb;
+      if (connectionMode == 'write') throw new Error('Cannot write readonly connection');
+      if (connectionMode == 'script') {
+        if (driver.readOnlySessions) return loadedWithDb;
+        throw new Error('Cannot write readonly connection');
+      }
+    }
+    return loadedWithDb;
+  }
+  return storedConnection;
+}
+
+async function connectUtility(driver, storedConnection, connectionMode) {
+  const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
 
-async function connectUtility(driver, storedConnection) {
   const connection = {
-    database: storedConnection.defaultDatabase,
-    ...decryptConnection(storedConnection),
+    database: connectionLoaded.defaultDatabase,
+    ...decryptConnection(connectionLoaded),
   };
 
   if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();

package/src/utility/crypting.js

@@ -55,7 +55,7 @@ function encryptPasswordField(connection, field) {
       [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
     };
   }
-return connection;
+  return connection;
 }
 
 function decryptPasswordField(connection, field) {
@@ -75,6 +75,11 @@ function encryptConnection(connection) {
   return connection;
 }
 
+function maskConnection(connection) {
+  if (!connection) return connection;
+  return _.omit(connection, ['password', 'sshPassword', 'sshKeyfilePassword']);
+}
+
 function decryptConnection(connection) {
   connection = decryptPasswordField(connection, 'password');
   connection = decryptPasswordField(connection, 'sshPassword');
@@ -95,5 +100,6 @@ module.exports = {
   loadEncryptionKey,
   encryptConnection,
   decryptConnection,
+  maskConnection,
   pickSafeConnectionInfo,
 };

package/src/utility/platformInfo.js

@@ -39,6 +39,8 @@ const platformInfo = {
   environment: process.env.NODE_ENV,
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
+  allowShellConnection: !!process.env.SHELL_CONNECTION || !!isElectron(),
+  allowShellScripting: !!process.env.SHELL_SCRIPTING || !!isElectron(),
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
 };