dbgate-api 4.6.0 → 4.6.1

package/.env-portal

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres,mongo,mongo2
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -24,4 +24,17 @@ LABEL_mongo2=Mongo Server
 SERVER_mongo2=localhost
 ENGINE_mongo2=mongo@dbgate-plugin-mongo
 
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
 # docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.6.0",
+  "version": "4.6.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.6.0",
-    "dbgate-sqltree": "^4.6.0",
-    "dbgate-tools": "^4.6.0",
+    "dbgate-query-splitter": "^4.6.1",
+    "dbgate-sqltree": "^4.6.1",
+    "dbgate-tools": "^4.6.1",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -44,7 +44,6 @@
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
@@ -64,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.6.0",
+    "dbgate-types": "^4.6.1",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/apps.js

@@ -0,0 +1,264 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged(`app-files-changed-${folder}`);
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+};

package/src/controllers/connections.js

@@ -1,7 +1,6 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
 const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
@@ -9,6 +8,7 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
 
@@ -53,6 +53,24 @@ function getPortalCollections() {
       defaultDatabase: process.env[`DATABASE_${id}`],
       singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
     console.log('Using connections from ENV variables:');
     console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
@@ -136,7 +154,7 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
@@ -173,18 +191,22 @@ module.exports = {
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
   update_meta: true,
   async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.update({ _id }, { $set: values });
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -192,22 +214,24 @@ module.exports = {
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }) {
     if (portalConnections) return;
-    const conn = await this.datastore.find({ _id: conid });
-    let databases = conn[0].databases || [];
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
     if (databases.find(x => x.name == database)) {
       databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
     } else {
       databases = [...databases, { name: database, ...values }];
     }
-    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    const res = await this.datastore.patch(conid, { databases });
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
     return res;
   },
 
   delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.remove(connection._id);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -215,8 +239,8 @@ module.exports = {
   get_meta: true,
   async get({ conid }) {
     if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
-    const res = await this.datastore.find({ _id: conid });
-    return res[0] || null;
+    const res = await this.datastore.get(conid);
+    return res || null;
   },
 
   newSqliteDatabase_meta: true,

package/src/controllers/files.js

@@ -1,12 +1,13 @@
 const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir, archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -74,6 +75,11 @@ module.exports = {
         encoding: 'utf-8',
       });
       return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
     } else {
       if (!hasPermission(`files/${folder}/read`)) return null;
       const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
@@ -88,6 +94,12 @@ module.exports = {
       await fs.writeFile(path.join(dir, file), serialize(format, data));
       socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
       return true;
+    } else if (folder.startsWith('app:')) {
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed-${app}`);
+      apps.emitChangedDbApp(folder);
+      return true;
     } else {
       if (!hasPermission(`files/${folder}/write`)) return false;
       const dir = path.join(filesdir(), folder);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.6.0',
-  buildTime: '2022-01-26T16:57:37.630Z'
+  version: '4.6.1',
+  buildTime: '2022-01-31T19:49:04.817Z'
 };

package/src/main.js

@@ -19,6 +19,7 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -157,6 +158,7 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/files', files);
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
 }
 
 function initializeElectronSender(electronSender) {

package/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,142 @@
+const AsyncLock = require('async-lock');
+const fs = require('fs-extra');
+const uuidv1 = require('uuid/v1');
+
+const lock = new AsyncLock();
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
+  async _save() {
+    if (!this.loadedOk) {
+      // don't override data
+      return;
+    }
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loadPerformed) {
+      await lock.acquire('reader', async () => {
+        if (!this.loadPerformed) {
+          if (!(await fs.exists(this.filename))) {
+            this.loadedOk = true;
+            this.loadPerformed = true;
+            return;
+          }
+          try {
+            const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+            this.data = text
+              .split('\n')
+              .filter(x => x.trim())
+              .map(x => JSON.parse(x));
+            this.loadedOk = true;
+          } catch (err) {
+            console.error(`Error loading file ${this.filename}`, err);
+          }
+          this.loadPerformed = true;
+        }
+      });
+    }
+  }
+
+  async insert(obj) {
+    await this._ensureLoaded();
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: uuidv1(),
+        };
+    this.data.push(elem);
+    await this._save();
+    return elem;
+  }
+
+  async get(id) {
+    await this._ensureLoaded();
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    await this._ensureLoaded();
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._save();
+    return obj;
+  }
+
+  async patch(id, values) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._save();
+    return this.data.find(x => x._id == id);
+  }
+
+  async remove(id) {
+    await this._ensureLoaded();
+    const removed = this.data.find(x => x._id == id);
+    this.data = this.data.filter(x => x._id != id);
+    await this._save();
+    return removed;
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}
+
+module.exports = JsonLinesDatabase;

package/src/utility/JsonLinesDatastore.js

@@ -4,7 +4,7 @@ const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 
-async function fetchNextLine(reader) {
+function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
     if (!reader.hasNextLine()) {
       resolve(null);
@@ -62,7 +62,7 @@ class JsonLinesDatastore {
 
   async _readLine(parse) {
     for (;;) {
-      const line = await fetchNextLine(this.reader);
+      const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
         // EOF
         return null;

package/src/utility/directories.js

@@ -38,6 +38,7 @@ const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
+const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
 
 function packagedPluginsDir() {
@@ -103,6 +104,7 @@ module.exports = {
   rundir,
   uploadsdir,
   archivedir,
+  appdir,
   ensureDirectory,
   pluginsdir,
   filesdir,