dbgate-api 4.4.2 → 4.5.0

package/.env-portal

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres
+CONNECTIONS=mysql,postgres,mongo,mongo2
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -15,3 +15,13 @@ USER_postgres=postgres
 PASSWORD_postgres=test
 PORT_postgres=5433
 ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.4.2",
+  "version": "4.5.0",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -19,16 +19,15 @@
   "dependencies": {
     "async-lock": "^1.2.4",
     "axios": "^0.21.1",
-    "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.4.2",
-    "dbgate-sqltree": "^4.4.2",
-    "dbgate-tools": "^4.4.2",
+    "dbgate-query-splitter": "^4.5.0",
+    "dbgate-sqltree": "^4.5.0",
+    "dbgate-tools": "^4.5.0",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -39,6 +38,7 @@
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "is-electron": "^2.2.1",
     "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
@@ -49,7 +49,6 @@
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
-    "socket.io": "^2.3.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
@@ -65,7 +64,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.4.2",
+    "dbgate-types": "^4.5.0",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
@@ -74,6 +73,7 @@
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.4.0"
+    "better-sqlite3": "7.4.5",
+    "msnodesqlv8": "^2.4.4"
   }
 }

package/src/controllers/archive.js

@@ -10,7 +10,7 @@ const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 
 module.exports = {
-  folders_meta: 'get',
+  folders_meta: true,
   async folders() {
     const folders = await fs.readdir(archivedir());
     return [
@@ -27,14 +27,14 @@ module.exports = {
     ];
   },
 
-  createFolder_meta: 'post',
+  createFolder_meta: true,
   async createFolder({ folder }) {
     await fs.mkdir(path.join(archivedir(), folder));
     socket.emitChanged('archive-folders-changed');
     return true;
   },
 
-  createLink_meta: 'post',
+  createLink_meta: true,
   async createLink({ linkedFolder }) {
     const folder = await this.getNewArchiveFolder({ database: path.parse(linkedFolder).name + '.link' });
     fs.writeFile(path.join(archivedir(), folder), linkedFolder);
@@ -43,7 +43,7 @@ module.exports = {
     return folder;
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ folder }) {
     const dir = resolveArchiveFolder(folder);
     if (!(await fs.exists(dir))) return [];
@@ -70,23 +70,23 @@ module.exports = {
     ];
   },
 
-  refreshFiles_meta: 'post',
+  refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  refreshFolders_meta: 'post',
+  refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  deleteFile_meta: 'post',
+  deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  renameFile_meta: 'post',
+  renameFile_meta: true,
   async renameFile({ folder, file, newFile, fileType }) {
     await fs.rename(
       path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
@@ -95,14 +95,14 @@ module.exports = {
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  renameFolder_meta: 'post',
+  renameFolder_meta: true,
   async renameFolder({ folder, newFolder }) {
     const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
     await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  deleteFolder_meta: 'post',
+  deleteFolder_meta: true,
   async deleteFolder({ folder }) {
     if (!folder) throw new Error('Missing folder parameter');
     if (folder.endsWith('.link')) {
@@ -113,13 +113,14 @@ module.exports = {
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  saveFreeTable_meta: 'post',
+  saveFreeTable_meta: true,
   async saveFreeTable({ folder, file, data }) {
-    saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
+    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
+    socket.emitChanged(`archive-files-changed-${folder}`);
     return true;
   },
 
-  loadFreeTable_meta: 'post',
+  loadFreeTable_meta: true,
   async loadFreeTable({ folder, file }) {
     return new Promise((resolve, reject) => {
       const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));

package/src/controllers/config.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const axios = require('axios');
 const { datadir } = require('../utility/directories');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
@@ -20,7 +21,7 @@ module.exports = {
     }
   },
 
-  get_meta: 'get',
+  get_meta: true,
   async get() {
     const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
 
@@ -32,17 +33,17 @@ module.exports = {
     };
   },
 
-  platformInfo_meta: 'get',
+  platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
   },
 
-  getSettings_meta: 'get',
+  getSettings_meta: true,
   async getSettings() {
     return this.settingsValue;
   },
 
-  updateSettings_meta: 'post',
+  updateSettings_meta: true,
   async updateSettings(values) {
     if (!hasPermission(`settings/change`)) return false;
     try {
@@ -58,4 +59,10 @@ module.exports = {
       return false;
     }
   },
+
+  changelog_meta: true,
+  async changelog() {
+    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+    return resp.data;
+  },
 };

package/src/controllers/connections.js

@@ -8,6 +8,9 @@ const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+
+const processArgs = require('../utility/processArgs');
 
 function getNamedArgs() {
   const res = {};
@@ -37,7 +40,7 @@ function getDatabaseFileLabel(databaseFile) {
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
@@ -45,11 +48,22 @@ function getPortalCollections() {
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
       defaultDatabase: process.env[`DATABASE_${id}`],
       singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
     }));
+    console.log('Using connections from ENV variables:');
+    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      console.log(
+        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
+        noengine.map(x => x._id)
+      );
+    }
+    return connections;
   }
 
   const args = getNamedArgs();
@@ -126,29 +140,34 @@ module.exports = {
     }
   },
 
-  list_meta: 'get',
+  list_meta: true,
   async list() {
     return portalConnections || this.datastore.find();
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
-    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
-      }
+  test_meta: true,
+  test(connection) {
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'connectProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
-  save_meta: 'post',
+  save_meta: true,
   async save(connection) {
     if (portalConnections) return;
     let res;
@@ -162,7 +181,30 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
+  update_meta: true,
+  async update({ _id, values }) {
+    if (portalConnections) return;
+    const res = await this.datastore.update({ _id }, { $set: values });
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  updateDatabase_meta: true,
+  async updateDatabase({ conid, database, values }) {
+    if (portalConnections) return;
+    const conn = await this.datastore.find({ _id: conid });
+    let databases = conn[0].databases || [];
+    if (databases.find(x => x.name == database)) {
+      databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
+    } else {
+      databases = [...databases, { name: database, ...values }];
+    }
+    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
     const res = await this.datastore.remove(_.pick(connection, '_id'));
@@ -170,14 +212,14 @@ module.exports = {
     return res;
   },
 
-  get_meta: 'get',
+  get_meta: true,
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
+    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
     const res = await this.datastore.find({ _id: conid });
-    return res[0];
+    return res[0] || null;
   },
 
-  newSqliteDatabase_meta: 'post',
+  newSqliteDatabase_meta: true,
   async newSqliteDatabase({ file }) {
     const sqliteDir = path.join(filesdir(), 'sqlite');
     if (!(await fs.exists(sqliteDir))) {

package/src/controllers/databaseConnections.js

@@ -25,6 +25,7 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { createTwoFilesPatch } = require('diff');
 const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -74,10 +75,12 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], [
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'databaseConnectionProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
@@ -121,7 +124,7 @@ module.exports = {
     return promise;
   },
 
-  queryData_meta: 'post',
+  queryData_meta: true,
   async queryData({ conid, database, sql }) {
     console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -132,7 +135,7 @@ module.exports = {
     return res;
   },
 
-  runScript_meta: 'post',
+  runScript_meta: true,
   async runScript({ conid, database, sql }) {
     console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -140,21 +143,21 @@ module.exports = {
     return res;
   },
 
-  collectionData_meta: 'post',
+  collectionData_meta: true,
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
     return res.result;
   },
 
-  updateCollection_meta: 'post',
+  updateCollection_meta: true,
   async updateCollection({ conid, database, changeSet }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
     return res.result;
   },
 
-  status_meta: 'get',
+  status_meta: true,
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
@@ -176,7 +179,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ conid, database }) {
     let existing = this.opened.find(x => x.conid == conid && x.database == database);
 
@@ -192,7 +195,7 @@ module.exports = {
     };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, database, keepOpen }) {
     if (!keepOpen) this.close(conid, database);
 
@@ -200,7 +203,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  syncModel_meta: 'post',
+  syncModel_meta: true,
   async syncModel({ conid, database }) {
     const conn = await this.ensureOpened(conid, database);
     conn.subprocess.send({ msgtype: 'syncModel' });
@@ -224,13 +227,13 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  disconnect_meta: true,
   async disconnect({ conid, database }) {
     await this.close(conid, database, true);
     return { status: 'ok' };
   },
 
-  structure_meta: 'get',
+  structure_meta: true,
   async structure({ conid, database }) {
     if (conid == '__model') {
       const model = await importDbModel(database);
@@ -247,13 +250,14 @@ module.exports = {
     // };
   },
 
-  serverVersion_meta: 'get',
+  serverVersion_meta: true,
   async serverVersion({ conid, database }) {
+    if (!conid) return null;
     const opened = await this.ensureOpened(conid, database);
-    return opened.serverVersion;
+    return opened.serverVersion || null;
   },
 
-  sqlPreview_meta: 'post',
+  sqlPreview_meta: true,
   async sqlPreview({ conid, database, objects, options }) {
     // wait for structure
     await this.structure({ conid, database });
@@ -263,7 +267,7 @@ module.exports = {
     return res;
   },
 
-  exportModel_meta: 'post',
+  exportModel_meta: true,
   async exportModel({ conid, database }) {
     const archiveFolder = await archive.getNewArchiveFolder({ database });
     await fs.mkdir(path.join(archivedir(), archiveFolder));
@@ -273,10 +277,13 @@ module.exports = {
     return { archiveFolder };
   },
 
-  generateDeploySql_meta: 'post',
+  generateDeploySql_meta: true,
   async generateDeploySql({ conid, database, archiveFolder }) {
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'generateDeploySql', modelFolder: resolveArchiveFolder(archiveFolder) });
+    const res = await this.sendRequest(opened, {
+      msgtype: 'generateDeploySql',
+      modelFolder: resolveArchiveFolder(archiveFolder),
+    });
     return res;
 
     // const connection = await connections.get({ conid });
@@ -285,7 +292,7 @@ module.exports = {
     //   analysedStructure: await this.structure({ conid, database }),
     //   modelFolder: resolveArchiveFolder(archiveFolder),
     // });
-    
+
     // const deployedModel = generateDbPairingId(await importDbModel(path.join(archivedir(), archiveFolder)));
     // const currentModel = generateDbPairingId(await this.structure({ conid, database }));
     // const currentModelPaired = matchPairedObjects(deployedModel, currentModel);
@@ -300,7 +307,7 @@ module.exports = {
     // };
     // return sql;
   },
-  // runCommand_meta: 'post',
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);
@@ -343,7 +350,7 @@ module.exports = {
     return res;
   },
 
-  generateDbDiffReport_meta: 'post',
+  generateDbDiffReport_meta: true,
   async generateDbDiffReport({ filePath, sourceConid, sourceDatabase, targetConid, targetDatabase }) {
     const unifiedDiff = await this.getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase });
 

package/src/controllers/files.js

@@ -20,7 +20,7 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  list_meta: 'get',
+  list_meta: true,
   async list({ folder }) {
     if (!hasPermission(`files/${folder}/read`)) return [];
     const dir = path.join(filesdir(), folder);
@@ -29,7 +29,7 @@ module.exports = {
     return files;
   },
 
-  listAll_meta: 'get',
+  listAll_meta: true,
   async listAll() {
     const folders = await fs.readdir(filesdir());
     const res = [];
@@ -42,7 +42,7 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
+  delete_meta: true,
   async delete({ folder, file }) {
     if (!hasPermission(`files/${folder}/write`)) return;
     await fs.unlink(path.join(filesdir(), folder, file));
@@ -50,7 +50,7 @@ module.exports = {
     socket.emitChanged(`all-files-changed`);
   },
 
-  rename_meta: 'post',
+  rename_meta: true,
   async rename({ folder, file, newFile }) {
     if (!hasPermission(`files/${folder}/write`)) return;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
@@ -58,7 +58,7 @@ module.exports = {
     socket.emitChanged(`all-files-changed`);
   },
 
-  copy_meta: 'post',
+  copy_meta: true,
   async copy({ folder, file, newFile }) {
     if (!hasPermission(`files/${folder}/write`)) return;
     await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
@@ -66,7 +66,7 @@ module.exports = {
     socket.emitChanged(`all-files-changed`);
   },
 
-  load_meta: 'post',
+  load_meta: true,
   async load({ folder, file, format }) {
     if (folder.startsWith('archive:')) {
       const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
@@ -80,7 +80,7 @@ module.exports = {
     }
   },
 
-  save_meta: 'post',
+  save_meta: true,
   async save({ folder, file, data, format }) {
     if (folder.startsWith('archive:')) {
       const dir = resolveArchiveFolder(folder.substring('archive:'.length));
@@ -101,12 +101,12 @@ module.exports = {
     }
   },
 
-  saveAs_meta: 'post',
+  saveAs_meta: true,
   async saveAs({ filePath, data, format }) {
     await fs.writeFile(filePath, serialize(format, data));
   },
 
-  favorites_meta: 'get',
+  favorites_meta: true,
   async favorites() {
     if (!hasPermission(`files/favorites/read`)) return [];
     const dir = path.join(filesdir(), 'favorites');
@@ -125,7 +125,7 @@ module.exports = {
     return res;
   },
 
-  generateUploadsFile_meta: 'get',
+  generateUploadsFile_meta: true,
   async generateUploadsFile() {
     const fileName = `${uuidv1()}.html`;
     return {
@@ -134,7 +134,7 @@ module.exports = {
     };
   },
 
-  exportChart_meta: 'post',
+  exportChart_meta: true,
   async exportChart({ filePath, title, config, image }) {
     const fileName = path.parse(filePath).base;
     const imageFile = fileName.replace('.html', '-preview.png');

package/src/controllers/jsldata.js

@@ -104,7 +104,7 @@ module.exports = {
     return datastore;
   },
 
-  getInfo_meta: 'get',
+  getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
@@ -112,13 +112,13 @@ module.exports = {
     return null;
   },
 
-  getRows_meta: 'post',
+  getRows_meta: true,
   async getRows({ jslid, offset, limit, filters }) {
     const datastore = await this.ensureDatastore(jslid);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
-  getStats_meta: 'get',
+  getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
     if (fs.existsSync(file)) {
@@ -146,7 +146,7 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: 'post',
+  saveFreeTable_meta: true,
   async saveFreeTable({ jslid, data }) {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;