dbgate-api 4.8.0 → 4.8.3

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.8.0",
+  "version": "4.8.3",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.8.0",
-    "dbgate-sqltree": "^4.8.0",
-    "dbgate-tools": "^4.8.0",
+    "dbgate-query-splitter": "^4.8.3",
+    "dbgate-sqltree": "^4.8.3",
+    "dbgate-tools": "^4.8.3",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.8.0",
+    "dbgate-types": "^4.8.3",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/databaseConnections.js

@@ -176,8 +176,8 @@ module.exports = {
   },
 
   loadKeys_meta: true,
-  async loadKeys({ conid, database, root }) {
-    return this.loadDataCore('loadKeys', { conid, database, root });
+  async loadKeys({ conid, database, root, filter }) {
+    return this.loadDataCore('loadKeys', { conid, database, root, filter });
   },
 
   exportKeys_meta: true,

package/src/controllers/jsldata.js

@@ -1,3 +1,4 @@
+const { filterName } = require('dbgate-tools');
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
@@ -148,6 +149,19 @@ module.exports = {
     return {};
   },
 
+  loadFieldValues_meta: true,
+  async loadFieldValues({ jslid, field, search }) {
+    const datastore = await this.ensureDatastore(jslid);
+    const res = new Set();
+    await datastore.enumRows(row => {
+      if (!filterName(search, row[field])) return true;
+      res.add(row[field]);
+      return res.size < 100;
+    });
+    // @ts-ignore
+    return [...res].map(value => ({ value }));
+  },
+
   async notifyChangedStats(stats) {
     // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];

package/src/controllers/plugins.js

@@ -125,6 +125,7 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
     // await this.saveRemovePlugins();
+    return true;
   },
 
   uninstall_meta: true,
@@ -134,7 +135,8 @@ module.exports = {
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins.push(packageName);
-    await this.saveRemovePlugins();
+    // await this.saveRemovePlugins();
+    return true;
   },
 
   upgrade_meta: true,
@@ -148,6 +150,7 @@ module.exports = {
     }
 
     socket.emitChanged(`installed-plugins-changed`);
+    return true;
   },
 
   command_meta: true,

package/src/controllers/runners.js

@@ -104,6 +104,7 @@ module.exports = {
       scriptFile,
       [
         '--checkParent', // ...process.argv.slice(3)
+        '--is-forked-api',
         ...processArgs.getPassArgs(),
       ],
       {

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.8.0',
-  buildTime: '2022-03-28T17:32:31.391Z'
+  version: '4.8.3',
+  buildTime: '2022-04-05T20:11:01.385Z'
 };

package/src/proc/databaseConnectionProcess.js

@@ -197,8 +197,8 @@ async function handleCollectionData({ msgid, options }) {
   return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
 }
 
-async function handleLoadKeys({ msgid, root }) {
-  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root));
+async function handleLoadKeys({ msgid, root, filter }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root, filter));
 }
 
 async function handleExportKeys({ msgid, options }) {

package/src/proc/index.js

@@ -3,6 +3,7 @@ const databaseConnectionProcess = require('./databaseConnectionProcess');
 const serverConnectionProcess = require('./serverConnectionProcess');
 const sessionProcess = require('./sessionProcess');
 const jslDatastoreProcess = require('./jslDatastoreProcess');
+const sshForwardProcess = require('./sshForwardProcess');
 
 module.exports = {
   connectProcess,
@@ -10,4 +11,5 @@ module.exports = {
   serverConnectionProcess,
   sessionProcess,
   jslDatastoreProcess,
+  sshForwardProcess,
 };

package/src/proc/sshForwardProcess.js

@@ -0,0 +1,68 @@
+const fs = require('fs-extra');
+const platformInfo = require('../utility/platformInfo');
+const childProcessChecker = require('../utility/childProcessChecker');
+const { SSHConnection } = require('node-ssh-forward');
+const { handleProcessCommunication } = require('../utility/processComm');
+
+async function getSshConnection(connection) {
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
+    username: connection.sshLogin,
+    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  return sshConn;
+}
+
+async function handleStart({ connection, tunnelConfig }) {
+  try {
+    const sshConn = await getSshConnection(connection);
+    await sshConn.forward(tunnelConfig);
+
+    process.send({
+      msgtype: 'connected',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    process.send({
+      msgtype: 'error',
+      connection,
+      tunnelConfig,
+      errorMessage: err.message,
+    });
+  }
+}
+
+const messageHandlers = {
+  connect: handleStart,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+function start() {
+  childProcessChecker();
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
+    try {
+      await handleMessage(message);
+    } catch (e) {
+      console.error('sshForwardProcess - unhandled error', e);
+    }
+  });
+}
+
+module.exports = { start };

package/src/utility/JsonLinesDatastore.js

@@ -159,10 +159,23 @@ class JsonLinesDatastore {
     }
   }
 
+  async enumRows(eachRow) {
+    await lock.acquire('reader', async () => {
+      await this._ensureReader(0, null);
+      for (;;) {
+        const line = await this._readLine(true);
+        if (line == null) break;
+        const shouldContinue = eachRow(line);
+        if (!shouldContinue) break;
+      }
+    });
+  }
+
   async getRows(offset, limit, filter) {
     const res = [];
     await lock.acquire('reader', async () => {
       await this._ensureReader(offset, filter);
+      // console.log(JSON.stringify(this.currentFilter, undefined, 2));
       for (let i = 0; i < limit; i += 1) {
         const line = await this._readLine(true);
         if (line == null) break;

package/src/utility/directories.js

@@ -8,7 +8,7 @@ const processArgs = require('./processArgs');
 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
   if (!createDirectories[dir]) {
-    if (clean && fs.existsSync(dir)) {
+    if (clean && fs.existsSync(dir) && !platformInfo.isForkedApi) {
       console.log(`Cleaning directory ${dir}`);
       cleanDirectory(dir);
     }

package/src/utility/sshTunnel.js

@@ -1,13 +1,11 @@
-const { SSHConnection } = require('node-ssh-forward');
-const fs = require('fs-extra');
 const portfinder = require('portfinder');
 const stableStringify = require('json-stable-stringify');
 const _ = require('lodash');
-const platformInfo = require('./platformInfo');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
+const { fork } = require('child_process');
+const processArgs = require('../utility/processArgs');
 
-const sshConnectionCache = {};
 const sshTunnelCache = {};
 
 const CONNECTION_FIELDS = [
@@ -22,37 +20,42 @@ const CONNECTION_FIELDS = [
 ];
 const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];
 
-async function getSshConnection(connection) {
-  const connectionCacheKey = stableStringify(_.pick(connection, CONNECTION_FIELDS));
-  if (sshConnectionCache[connectionCacheKey]) return sshConnectionCache[connectionCacheKey];
+function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
+  let subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+    '--is-forked-api',
+    '--start-process',
+    'sshForwardProcess',
+    ...processArgs.getPassArgs(),
+  ]);
 
-  const sshConfig = {
-    endHost: connection.sshHost || '',
-    endPort: connection.sshPort || 22,
-    bastionHost: connection.sshBastionHost || '',
-    agentForward: connection.sshMode == 'agent',
-    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
-    username: connection.sshLogin,
-    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
-    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
-    privateKey:
-      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
-    skipAutoPrivateKey: true,
-    noReadline: true,
-  };
-
-  const sshConn = new SSHConnection(sshConfig);
-  sshConnectionCache[connectionCacheKey] = sshConn;
-  return sshConn;
+  subprocess.send({
+    msgtype: 'connect',
+    connection,
+    tunnelConfig,
+  });
+  return new Promise((resolve, reject) => {
+    subprocess.on('message', resp => {
+      // @ts-ignore
+      const { msgtype, errorMessage } = resp;
+      if (msgtype == 'connected') {
+        resolve(subprocess);
+      }
+      if (msgtype == 'error') {
+        reject(errorMessage);
+      }
+    });
+    subprocess.on('exit', code => {
+      console.log('SSH forward process exited');
+      delete sshTunnelCache[tunnelCacheKey];
+    });
+  });
 }
 
 async function getSshTunnel(connection) {
   const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
 
   return await lock.acquire(tunnelCacheKey, async () => {
-    const sshConn = await getSshConnection(connection);
     if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
-
     const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
     // workaround for `getPortPromise` not releasing the port quickly enough
     await new Promise(resolve => setTimeout(resolve, 500));
@@ -66,7 +69,8 @@ async function getSshTunnel(connection) {
         `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
 
-      const tunnel = await sshConn.forward(tunnelConfig);
+      const subprocess = await callForwardProcess(connection, tunnelConfig, tunnelCacheKey);
+
       console.log(
         `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
       );
@@ -74,6 +78,7 @@ async function getSshTunnel(connection) {
       sshTunnelCache[tunnelCacheKey] = {
         state: 'ok',
         localPort,
+        subprocess,
       };
       return sshTunnelCache[tunnelCacheKey];
     } catch (err) {

package/src/utility/useController.js

@@ -31,9 +31,14 @@ module.exports = function useController(app, electron, route, controller) {
         const handler = `${route.substring(1)}-${_.kebabCase(key)}`;
         // console.log('REGISTERING HANDLER', handler);
         electron.ipcMain.handle(handler, async (event, args) => {
-          const data = await controller[key](args);
-          // console.log('HANDLED API', handler, data);
-          return data;
+          try {
+            const data = await controller[key](args);
+            // console.log('HANDLED API', handler, data);
+            if (data === undefined) return null;
+            return data;
+          } catch (err) {
+            return { apiErrorMessage: err.message };
+          }
         });
       }
 
@@ -68,7 +73,7 @@ module.exports = function useController(app, electron, route, controller) {
           res.json(data);
         } catch (e) {
           console.log(e);
-          res.status(500).json({ error: e.message });
+          res.status(500).json({ apiErrorMessage: e.message });
         }
       });
     }