dbgate-api 4.7.3-alpha.5 → 4.7.4-alpha.10

package/.env

@@ -1 +1,14 @@
 DEVMODE=1
+# PERMISSIONS=~widgets/app,~widgets/plugins
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1
+
+
+DEVWEB=1
+LOGINS=admin,test
+
+LOGIN_PASSWORD_admin=admin
+LOGIN_PERMISSIONS_admin=*
+
+LOGIN_PASSWORD_test=test
+LOGIN_PERMISSIONS_test=~*, widgets/database

package/env/singledb/.env

@@ -8,6 +8,8 @@ USER_mysql=root
 PASSWORD_mysql=test
 PORT_mysql=3307
 ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
 
 SINGLE_CONNECTION=mysql
 SINGLE_DATABASE=Chinook

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.7.3-alpha.5",
+  "version": "4.7.4-alpha.10",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.7.3-alpha.5",
-    "dbgate-sqltree": "^4.7.3-alpha.5",
-    "dbgate-tools": "^4.7.3-alpha.5",
+    "dbgate-query-splitter": "^4.7.4-alpha.10",
+    "dbgate-sqltree": "^4.7.4-alpha.10",
+    "dbgate-tools": "^4.7.4-alpha.10",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.7.3-alpha.5",
+    "dbgate-types": "^4.7.4-alpha.10",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/config.js

@@ -3,7 +3,7 @@ const os = require('os');
 const path = require('path');
 const axios = require('axios');
 const { datadir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
@@ -26,17 +26,31 @@ module.exports = {
   // },
 
   get_meta: true,
-  async get() {
-    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
+  async get(_params, req) {
+    const logins = getLogins();
+    const login = logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
+    const permissions = login ? login.permissions : null;
 
     return {
       runAsPortal: !!connections.portalConnections,
       singleDatabase: connections.singleDatabase,
+      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellConnection,
       permissions,
+      login,
       ...currentVersion,
     };
   },
 
+  logout_meta: {
+    method: 'get',
+    raw: true,
+  },
+  logout(req, res) {
+    res.status(401).send('Logged out<br><a href="../..">Back to DbGate</a>');
+  },
+
   platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
@@ -64,8 +78,8 @@ module.exports = {
   },
 
   updateSettings_meta: true,
-  async updateSettings(values) {
-    if (!hasPermission(`settings/change`)) return false;
+  async updateSettings(values, req) {
+    if (!hasPermission(`settings/change`, req)) return false;
 
     const res = await lock.acquire('update', async () => {
       const currentValue = await this.getSettings();

package/src/controllers/connections.js

@@ -5,12 +5,14 @@ const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
+const { safeJsonParse } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
 
 function getNamedArgs() {
   const res = {};
@@ -55,6 +57,8 @@ function getPortalCollections() {
         (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
       singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],
@@ -162,7 +166,9 @@ module.exports = {
 
   list_meta: true,
   async list() {
-    return portalConnections || this.datastore.find();
+    return portalConnections && !platformInfo.allowShellConnection
+      ? portalConnections.map(maskConnection)
+      : this.datastore.find();
   },
 
   test_meta: true,
@@ -199,6 +205,9 @@ module.exports = {
     }
     socket.emitChanged('connection-list-changed');
     socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
     // for (const db of connection.databases || []) {
     //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
     // }
@@ -238,13 +247,21 @@ module.exports = {
     return res;
   },
 
-  get_meta: true,
-  async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
+  async getCore({ conid, mask = false }) {
+    if (!conid) return null;
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
     const res = await this.datastore.get(conid);
     return res || null;
   },
 
+  get_meta: true,
+  async get({ conid }) {
+    return this.getCore({ conid, mask: true });
+  },
+
   newSqliteDatabase_meta: true,
   async newSqliteDatabase({ file }) {
     const sqliteDir = path.join(filesdir(), 'sqlite');

package/src/controllers/databaseConnections.js

@@ -33,6 +33,10 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
@@ -75,7 +79,7 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
+    const connection = await connections.getCore({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
       '--is-forked-api',
       '--start-process',
@@ -136,6 +140,13 @@ module.exports = {
     return res;
   },
 
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
   runScript_meta: true,
   async runScript({ conid, database, sql }) {
     console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
@@ -148,14 +159,64 @@ module.exports = {
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
-    return res.result;
+    return res.result || null;
+  },
+
+  async loadDataCore(msgtype, { conid, database, ...args }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root }) {
+    return this.loadDataCore('loadKeys', { conid, database, root });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }) {
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }) {
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }) {
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }) {
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
   },
 
   updateCollection_meta: true,
   async updateCollection({ conid, database, changeSet }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
-    return res.result;
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
   },
 
   status_meta: true,
@@ -228,6 +289,12 @@ module.exports = {
     }
   },
 
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
+    }
+  },
+
   disconnect_meta: true,
   async disconnect({ conid, database }) {
     await this.close(conid, database, true);
@@ -325,8 +392,8 @@ module.exports = {
     const targetDb = generateDbPairingId(
       extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
     );
-    // const sourceConnection = await connections.get({conid:sourceConid})
-    const connection = await connections.get({ conid: targetConid });
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
     const driver = requireEngineDriver(connection);
     const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
     const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);

package/src/controllers/files.js

@@ -3,7 +3,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
@@ -23,8 +23,8 @@ function deserialize(format, text) {
 
 module.exports = {
   list_meta: true,
-  async list({ folder }) {
-    if (!hasPermission(`files/${folder}/read`)) return [];
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
@@ -32,11 +32,11 @@ module.exports = {
   },
 
   listAll_meta: true,
-  async listAll() {
+  async listAll(_params, req) {
     const folders = await fs.readdir(filesdir());
     const res = [];
     for (const folder of folders) {
-      if (!hasPermission(`files/${folder}/read`)) continue;
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
       const dir = path.join(filesdir(), folder);
       const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
@@ -45,31 +45,34 @@ module.exports = {
   },
 
   delete_meta: true,
-  async delete({ folder, file }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
   rename_meta: true,
-  async rename({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
   copy_meta: true,
-  async copy({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
   load_meta: true,
-  async load({ folder, file, format }) {
+  async load({ folder, file, format }, req) {
     if (folder.startsWith('archive:')) {
       const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
         encoding: 'utf-8',
@@ -81,20 +84,22 @@ module.exports = {
       });
       return deserialize(format, text);
     } else {
-      if (!hasPermission(`files/${folder}/read`)) return null;
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
       const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
       return deserialize(format, text);
     }
   },
 
   save_meta: true,
-  async save({ folder, file, data, format }) {
+  async save({ folder, file, data, format }, req) {
     if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
       const dir = resolveArchiveFolder(folder.substring('archive:'.length));
       await fs.writeFile(path.join(dir, file), serialize(format, data));
       socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
       return true;
     } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
       const app = folder.substring('app:'.length);
       await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
       socket.emitChanged(`app-files-changed-${app}`);
@@ -102,7 +107,7 @@ module.exports = {
       apps.emitChangedDbApp(folder);
       return true;
     } else {
-      if (!hasPermission(`files/${folder}/write`)) return false;
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
       const dir = path.join(filesdir(), folder);
       if (!(await fs.exists(dir))) {
         await fs.mkdir(dir);
@@ -123,8 +128,8 @@ module.exports = {
   },
 
   favorites_meta: true,
-  async favorites() {
-    if (!hasPermission(`files/favorites/read`)) return [];
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
     const dir = path.join(filesdir(), 'favorites');
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);

package/src/controllers/jsldata.js

@@ -111,18 +111,22 @@ module.exports = {
   getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) {
-      const parsed = JSON.parse(firstLine);
-      if (parsed.__isStreamHeader) {
-        return parsed;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
       }
-      return {
-        __isStreamHeader: true,
-        __isDynamicStructure: true,
-      };
+      return null;
+    } catch (err) {
+      return null;
     }
-    return null;
   },
 
   getRows_meta: true,

package/src/controllers/plugins.js

@@ -7,7 +7,7 @@ const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const _ = require('lodash');
 const packagedPluginsContent = require('../packagedPluginsContent');
 
@@ -73,7 +73,7 @@ module.exports = {
     const res = [];
     for (const packageName of _.union(files1, files2)) {
       if (packageName == 'dist') continue;
-      // if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
           const manifest = {
@@ -115,8 +115,8 @@ module.exports = {
   // },
 
   install_meta: true,
-  async install({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (!(await fs.exists(dir))) {
@@ -128,8 +128,8 @@ module.exports = {
   },
 
   uninstall_meta: true,
-  async uninstall({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
@@ -138,8 +138,8 @@ module.exports = {
   },
 
   upgrade_meta: true,
-  async upgrade({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (await fs.exists(dir)) {

package/src/controllers/runners.js

@@ -6,9 +6,10 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -150,6 +151,16 @@ module.exports = {
   start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

package/src/controllers/scheduler.js

@@ -3,7 +3,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -26,8 +26,8 @@ module.exports = {
     this.tasks.push(task);
   },
 
-  async reload() {
-    if (!hasPermission('files/shell/read')) return;
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
     const shellDir = path.join(filesdir(), 'shell');
     await this.unload();
     if (!(await fs.exists(shellDir))) return;

package/src/controllers/serverConnections.js

@@ -37,7 +37,7 @@ module.exports = {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
+      const connection = await connections.getCore({ conid });
       const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
         '--is-forked-api',
         '--start-process',

package/src/controllers/sessions.js

@@ -4,8 +4,10 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -46,9 +48,18 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -60,12 +71,17 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
   create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
+    const connection = await connections.getCore({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
       '--is-forked-api',
       '--start-process',
@@ -105,6 +121,29 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.7.3-alpha.5',
-  buildTime: '2022-03-14T18:31:03.883Z'
+  version: '4.7.4-alpha.10',
+  buildTime: '2022-03-21T19:56:58.513Z'
 };

package/src/main.js

@@ -29,6 +29,8 @@ const queryHistory = require('./controllers/queryHistory');
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -37,12 +39,11 @@ function start() {
 
   const server = http.createServer(app);
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
@@ -85,15 +86,7 @@ function start() {
   if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
-  } else {
-    if (!platformInfo.isNpmDist) {
-      app.get(getExpressPath('/'), (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
-
-  if (platformInfo.isNpmDist) {
+  } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({
       port: parseInt(
@@ -105,7 +98,19 @@ function start() {
         console.log(`DbGate API listening on port ${port}`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    console.log('__dirname', __dirname);
+    console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API & web listening on port', port);
+    server.listen(port);
   } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+
     const port = process.env.PORT || 3000;
     console.log('DbGate API listening on port', port);
     server.listen(port);

package/src/proc/connectProcess.js

@@ -20,7 +20,7 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {

package/src/proc/databaseConnectionProcess.js

@@ -7,6 +7,7 @@ const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
 
 let systemConnection;
 let storedConnection;
@@ -107,7 +108,7 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
   await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
@@ -154,6 +155,7 @@ async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     await driver.script(systemConnection, sql);
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
@@ -165,6 +167,7 @@ async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
@@ -172,21 +175,67 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
-async function handleCollectionData({ msgid, options }) {
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s });
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
-    const result = await driver.readCollection(systemConnection, options);
+    const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
+async function handleCollectionData({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
+  }
+}
+
 async function handleUpdateCollection({ msgid, changeSet }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const result = await driver.updateCollection(systemConnection, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
@@ -248,10 +297,16 @@ const messageHandlers = {
   runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
   generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
   // runCommand: handleRunCommand,
 };
 

package/src/proc/serverConnectionProcess.js

@@ -58,11 +58,14 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
-      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
     }
   } catch (err) {
     setStatus({
@@ -80,7 +83,7 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
   if (driver.createDatabase) {
     await driver.createDatabase(systemConnection, name);

package/src/proc/sessionProcess.js

@@ -17,11 +17,15 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +36,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -98,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -135,11 +166,22 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -163,6 +205,19 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
@@ -176,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

package/src/shell/executeQuery.js

@@ -5,7 +5,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   console.log(`Execute query ${sql}`);
 
   if (!driver) driver = requireEngineDriver(connection);
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
   console.log(`Connected.`);
 
   await driver.script(pool, sql);

package/src/shell/generateDeploySql.js

@@ -21,7 +21,7 @@ async function generateDeploySql({
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
   if (!analysedStructure) {
     analysedStructure = await driver.analyseFull(pool);
   }

package/src/shell/queryReader.js

@@ -1,14 +1,26 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function queryReader({ connection, sql }) {
-  console.log(`Reading query ${sql}`);
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  console.log(`Reading query ${query || sql}`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return await driver.readQuery(pool, sql);
+  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
 }
 
 module.exports = queryReader;

package/src/shell/tableReader.js

@@ -4,12 +4,12 @@ const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, 'read');
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
-  if (driver.dialect.nosql) {
+  if (driver.databaseEngineTypes.includes('document')) {
     // @ts-ignore
     console.log(`Reading collection ${fullNameToString(fullName)}`);
     // @ts-ignore

package/src/shell/tableWriter.js

@@ -8,7 +8,7 @@ async function tableWriter({ connection, schemaName, pureName, driver, systemCon
   if (!driver) {
     driver = requireEngineDriver(connection);
   }
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
 
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);

package/src/utility/connectUtility.js

@@ -4,11 +4,47 @@ const fs = require('fs-extra');
 const { decryptConnection } = require('./crypting');
 const { getSshTunnel } = require('./sshTunnel');
 const { getSshTunnelProxy } = require('./sshTunnelProxy');
+const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
+
+async function loadConnection(driver, storedConnection, connectionMode) {
+  const { allowShellConnection } = platformInfo;
+
+  if (connectionMode == 'app') {
+    return storedConnection;
+  }
+
+  if (storedConnection._id || !allowShellConnection) {
+    if (!storedConnection._id) {
+      throw new Error('Missing connection _id');
+    }
+
+    await connections._init();
+    const loaded = await connections.getCore({ conid: storedConnection._id });
+    const loadedWithDb = {
+      ...loaded,
+      database: storedConnection.database,
+    };
+
+    if (loaded.isReadOnly) {
+      if (connectionMode == 'read') return loadedWithDb;
+      if (connectionMode == 'write') throw new Error('Cannot write readonly connection');
+      if (connectionMode == 'script') {
+        if (driver.readOnlySessions) return loadedWithDb;
+        throw new Error('Cannot write readonly connection');
+      }
+    }
+    return loadedWithDb;
+  }
+  return storedConnection;
+}
+
+async function connectUtility(driver, storedConnection, connectionMode) {
+  const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
 
-async function connectUtility(driver, storedConnection) {
   const connection = {
-    database: storedConnection.defaultDatabase,
-    ...decryptConnection(storedConnection),
+    database: connectionLoaded.defaultDatabase,
+    ...decryptConnection(connectionLoaded),
   };
 
   if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();

package/src/utility/crypting.js

@@ -55,7 +55,7 @@ function encryptPasswordField(connection, field) {
       [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
     };
   }
-return connection;
+  return connection;
 }
 
 function decryptPasswordField(connection, field) {
@@ -75,6 +75,11 @@ function encryptConnection(connection) {
   return connection;
 }
 
+function maskConnection(connection) {
+  if (!connection) return connection;
+  return _.omit(connection, ['password', 'sshPassword', 'sshKeyfilePassword']);
+}
+
 function decryptConnection(connection) {
   connection = decryptPasswordField(connection, 'password');
   connection = decryptPasswordField(connection, 'sshPassword');
@@ -95,5 +100,6 @@ module.exports = {
   loadEncryptionKey,
   encryptConnection,
   decryptConnection,
+  maskConnection,
   pickSafeConnectionInfo,
 };

package/src/utility/hasPermission.js

@@ -1,12 +1,56 @@
 const { compilePermissions, testPermission } = require('dbgate-tools');
+const _ = require('lodash');
 
-let compiled = undefined;
+const userPermissions = {};
 
-function hasPermission(tested) {
-  if (compiled === undefined) {
-    compiled = compilePermissions(process.env.PERMISSIONS);
+function hasPermission(tested, req) {
+  const { user } = (req && req.auth) || {};
+  const key = user || '';
+  const logins = getLogins();
+  if (!userPermissions[key] && logins) {
+    const login = logins.find(x => x.login == user);
+    userPermissions[key] = compilePermissions(login ? login.permissions : null);
   }
-  return testPermission(tested, compiled);
+  return testPermission(tested, userPermissions[key]);
 }
 
-module.exports = hasPermission;
+let loginsCache = null;
+let loginsLoaded = false;
+
+function getLogins() {
+  if (loginsLoaded) {
+    return loginsCache;
+  }
+
+  const res = [];
+  if (process.env.LOGIN && process.env.PASSWORD) {
+    res.push({
+      login: process.env.LOGIN,
+      password: process.env.PASSWORD,
+      permissions: process.env.PERMISSIONS,
+    });
+  }
+  if (process.env.LOGINS) {
+    const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim()));
+    for (const login of logins) {
+      const password = process.env[`LOGIN_PASSWORD_${login}`];
+      const permissions = process.env[`LOGIN_PERMISSIONS_${login}`];
+      if (password) {
+        res.push({
+          login,
+          password,
+          permissions,
+        });
+      }
+    }
+  }
+
+  loginsCache = res.length > 0 ? res : null;
+  loginsLoaded = true;
+  return loginsCache;
+}
+
+module.exports = {
+  hasPermission,
+  getLogins,
+};

package/src/utility/platformInfo.js

@@ -39,6 +39,8 @@ const platformInfo = {
   environment: process.env.NODE_ENV,
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
+  allowShellConnection: !!process.env.SHELL_CONNECTION || !!isElectron(),
+  allowShellScripting: !!process.env.SHELL_SCRIPTING || !!isElectron(),
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
 };
 

package/src/utility/useController.js

@@ -62,7 +62,7 @@ module.exports = function useController(app, electron, route, controller) {
         //   controller._init_called = true;
         // }
         try {
-          let params = [{ ...req.body, ...req.query }];
+          let params = [{ ...req.body, ...req.query }, req];
           if (rawParams) params = [req, res];
           const data = await controller[key](...params);
           res.json(data);