dbgate-api-premium 6.3.2 → 6.4.0

package/src/controllers/files.js

@@ -9,6 +9,9 @@ const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
 const apps = require('./apps');
 const getMapExport = require('../utility/getMapExport');
+const dbgateApi = require('../shell');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('files');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -219,4 +222,60 @@ module.exports = {
       return path.join(dir, file);
     }
   },
+
+  createZipFromJsons_meta: true,
+  async createZipFromJsons({ db, filePath }) {
+    logger.info(`Creating zip file from JSONS ${filePath}`);
+    await dbgateApi.zipJsonLinesData(db, filePath);
+    return true;
+  },
+
+  getJsonsFromZip_meta: true,
+  async getJsonsFromZip({ filePath }) {
+    const res = await dbgateApi.unzipJsonLinesData(filePath);
+    return res;
+  },
+
+  downloadText_meta: true,
+  async downloadText({ uri }, req) {
+    if (!uri) return null;
+    const filePath = await dbgateApi.download(uri);
+    const text = await fs.readFile(filePath, {
+      encoding: 'utf-8',
+    });
+    return text;
+  },
+
+  saveUploadedFile_meta: true,
+  async saveUploadedFile({ filePath, fileName }) {
+    const FOLDERS = ['sql', 'sqlite'];
+    for (const folder of FOLDERS) {
+      if (fileName.toLowerCase().endsWith('.' + folder)) {
+        logger.info(`Saving ${folder} file ${fileName}`);
+        await fs.copyFile(filePath, path.join(filesdir(), folder, fileName));
+
+        socket.emitChanged(`files-changed`, { folder: folder });
+        socket.emitChanged(`all-files-changed`);
+        return {
+          name: path.basename(filePath),
+          folder: folder,
+        };
+      }
+    }
+
+    throw new Error(`${fileName} doesn't have one of supported extensions: ${FOLDERS.join(', ')}`);
+  },
+
+  exportFile_meta: true,
+  async exportFile({ folder, file, filePath }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), filePath);
+    return true;
+  },
+
+  simpleCopy_meta: true,
+  async simpleCopy({ sourceFilePath, targetFilePath }, req) {
+    await fs.copyFile(sourceFilePath, targetFilePath);
+    return true;
+  },
 };

package/src/controllers/jsldata.js

@@ -8,6 +8,8 @@ const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
+const crypto = require('crypto');
+const dbgateApi = require('../shell');
 
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
@@ -293,4 +295,11 @@ module.exports = {
       })),
     };
   },
+
+  downloadJslData_meta: true,
+  async downloadJslData({ uri }) {
+    const jslid = crypto.randomUUID();
+    await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
+    return { jslid };
+  },
 };

package/src/controllers/runners.js

@@ -8,7 +8,7 @@ const { fork, spawn } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const {
   extractShellApiPlugins,
-  extractShellApiFunctionName,
+  compileShellApiFunctionName,
   jsonScriptToJavascript,
   getLogger,
   safeJsonParse,
@@ -58,7 +58,7 @@ dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
 require=null;
 async function run() {
-const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});
+const reader=await ${compileShellApiFunctionName(functionName)}(${JSON.stringify(props)});
 const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
 await dbgateApi.copyStream(reader, writer);
 }
@@ -96,9 +96,9 @@ module.exports = {
 
   handle_ping() {},
 
-  handle_freeData(runid, { freeData }) {
+  handle_dataResult(runid, { dataResult }) {
     const { resolve } = this.requests[runid];
-    resolve(freeData);
+    resolve(dataResult);
     delete this.requests[runid];
   },
 
@@ -273,7 +273,7 @@ module.exports = {
     const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
-      const js = jsonScriptToJavascript(script);
+      const js = await jsonScriptToJavascript(script);
       return this.startCore(runid, scriptTemplate(js, false));
     }
 
@@ -328,4 +328,24 @@ module.exports = {
     });
     return promise;
   },
+
+  scriptResult_meta: true,
+  async scriptResult({ script }) {
+    if (script.type != 'json') {
+      return { errorMessage: 'Only JSON scripts are allowed' };
+    }
+
+    const promise = new Promise(async (resolve, reject) => {
+      const runid = crypto.randomUUID();
+      this.requests[runid] = { resolve, reject, exitOnStreamError: true };
+      const cloned = _.cloneDeepWith(script, node => {
+        if (node?.$replace == 'runid') {
+          return runid;
+        }
+      });
+      const js = await jsonScriptToJavascript(cloned);
+      this.startCore(runid, scriptTemplate(js, false));
+    });
+    return promise;
+  },
 };

package/src/controllers/serverConnections.js

@@ -54,6 +54,9 @@ module.exports = {
       if (!connection) {
         throw new Error(`Connection with conid="${conid}" not found`);
       }
+      if (connection.singleDatabase) {
+        return null;
+      }
       if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
         throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
       }
@@ -98,6 +101,11 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
+      subprocess.on('error', err => {
+        logger.error(extractErrorLogData(err), 'Error in server connection subprocess');
+        if (newOpened.disconnected) return;
+        this.close(conid, false);
+      });
       subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
@@ -137,14 +145,14 @@ module.exports = {
     if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
-    return opened.databases;
+    return opened?.databases ?? [];
   },
 
   version_meta: true,
   async version({ conid }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
-    return opened.version;
+    return opened?.version ?? null;
   },
 
   serverStatus_meta: true,
@@ -165,6 +173,9 @@ module.exports = {
         }
         this.lastPinged[conid] = new Date().getTime();
         const opened = await this.ensureOpened(conid);
+        if (!opened) {
+          return Promise.resolve();
+        }
         try {
           opened.subprocess.send({ msgtype: 'ping' });
         } catch (err) {
@@ -189,6 +200,9 @@ module.exports = {
   async sendDatabaseOp({ conid, msgtype, name }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return null;
+    }
     if (opened.connection.isReadOnly) return false;
     const res = await this.sendRequest(opened, { msgtype, name });
     if (res.errorMessage) {
@@ -228,6 +242,9 @@ module.exports = {
   async loadDataCore(msgtype, { conid, ...args }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return null;
+    }
     const res = await this.sendRequest(opened, { msgtype, ...args });
     if (res.errorMessage) {
       console.error(res.errorMessage);
@@ -249,6 +266,9 @@ module.exports = {
   async summaryCommand({ conid, command, row }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    if (!opened) {
+      return null;
+    }
     if (opened.connection.isReadOnly) return false;
     return this.loadDataCore('summaryCommand', { conid, command, row });
   },

package/src/controllers/storage.js

@@ -12,15 +12,32 @@ const {
   storageReadConfig,
   storageWriteConfig,
   getStorageConnectionError,
+  storageSaveRelationDiff,
+  selectStorageIdentity,
 } = require('./storageDb');
 const { hasPermission } = require('../utility/hasPermission');
 const { changeSetToSql, removeSchemaFromChangeSet } = require('dbgate-datalib');
 const storageModel = require('../storageModel');
 const { dumpSqlCommand } = require('dbgate-sqltree');
-const { runCommandOnDriver, getLogger, runQueryFmt } = require('dbgate-tools');
+const {
+  runCommandOnDriver,
+  getLogger,
+  runQueryFmt,
+  getPredefinedPermissions,
+  runQueryOnDriver,
+} = require('dbgate-tools');
 const socket = require('../utility/socket');
 const { obtainRefreshedLicense } = require('../utility/authProxy');
 const { datadir } = require('../utility/directories');
+const {
+  loadEncryptionKeyFromExternal,
+  encryptUser,
+  encryptConnection,
+  encryptPasswordString,
+} = require('../utility/crypting');
+const crypto = require('crypto');
+const dataReplicator = require('../shell/dataReplicator');
+const storageReplicatorItems = require('../utility/storageReplicatorItems');
 
 const logger = getLogger('storage');
 
@@ -31,7 +48,6 @@ function mapConnection(connnection) {
   };
 }
 
-
 let refreshLicenseStarted = false;
 
 module.exports = {
@@ -48,6 +64,22 @@ module.exports = {
     }
 
     setAuthProviders(providers, providers[defIndex]);
+
+    const keyResult = await storageSelectFmt(
+      "select ~value from ~config where ~group = 'admin' and ~key = 'encryptionKey'"
+    );
+    const [conn, driver] = await getStorageConnection();
+
+    await loadEncryptionKeyFromExternal(keyResult[0]?.value, async key => {
+      await runQueryFmt(
+        driver,
+        conn,
+        'insert into ~config (~group, ~key, ~value) values (%v, %v, %v)',
+        'admin',
+        'encryptionKey',
+        key
+      );
+    });
   },
 
   async startRefreshLicense() {
@@ -135,7 +167,7 @@ module.exports = {
         displayName: 'Internal storage',
         defaultDatabase: process.env.STORAGE_DATABASE,
         singleDatabase: true,
-        ...await getDbConnectionParams(),
+        ...(await getDbConnectionParams()),
       };
     }
 
@@ -183,8 +215,8 @@ module.exports = {
       return null;
     }
 
-    await runQueryFmt(driver, conn, 'delete from auth_methods_config');
-    await runQueryFmt(driver, conn, 'delete from auth_methods where id > 0');
+    await runQueryFmt(driver, conn, 'delete from ~auth_methods_config');
+    await runQueryFmt(driver, conn, 'delete from ~auth_methods where ~id > 0');
 
     let id = 1;
     for (const method of authMethods) {
@@ -192,7 +224,7 @@ module.exports = {
         await runQueryFmt(
           driver,
           conn,
-          'update auth_methods set name=%v, is_disabled=%v, is_default = %v, is_collapsed = %v where id = %v',
+          'update ~auth_methods set ~name=%v, ~is_disabled=%v, ~is_default = %v, ~is_collapsed = %v where ~id = %v',
           method.name,
           method.isDisabled,
           method.isDefault,
@@ -228,7 +260,7 @@ module.exports = {
         await runQueryFmt(
           driver,
           conn,
-          'insert into auth_methods_config (%i, %i, %i) values (%v, %v, %v)',
+          'insert into ~auth_methods_config (%i, %i, %i) values (%v, %v, %v)',
           'auth_method_id',
           'key',
           'value',
@@ -400,7 +432,7 @@ module.exports = {
       driver,
       conn,
       "insert into ~config (~group, ~key, ~value) values ('admin', 'adminPassword', %v)",
-      newPassword
+      encryptPasswordString(newPassword)
     );
 
     return { status: 'ok' };
@@ -409,4 +441,305 @@ module.exports = {
   getStorageConnectionError() {
     return getStorageConnectionError();
   },
+
+  getUserList_meta: true,
+  async getUserList() {
+    const resp = await storageSelectFmt(`select ~users.~id,~users.~login,~users.~password,~users.~email from ~users`);
+    const usedRoles = await storageSelectFmt(`select ~roles.~name, ~user_roles.~user_id from 
+        ~user_roles inner join ~roles ON ~user_roles.~role_id = ~roles.~id`);
+    const usedPermissions = await storageSelectFmt(`select * from ~user_permissions`);
+    return resp.map(x => ({
+      ...x,
+      password: x.password?.startsWith('crypt:') ? 'crypted' : x.password ? 'plain' : 'no',
+      usedRoles: usedRoles.filter(y => y.user_id == x.id).map(y => y.name),
+      usedPermissions: usedPermissions.filter(y => y.user_id == x.id).map(y => y.permission),
+    }));
+  },
+
+  getUserDetail_meta: true,
+  async getUserDetail({ id }) {
+    const resp =
+      id == 'new'
+        ? {}
+        : await storageSelectFmt(
+            `select ~users.~id,~users.~login,~users.password, ~users.~email from ~users where ~users.~id = %v`,
+            id
+          );
+
+    const rolePermissions = id == 'new' ? [] : await storageReadUserRolePermissions(id);
+    const basePermissions = [...getPredefinedPermissions('logged-user'), ...rolePermissions];
+    const permissions =
+      id == 'new'
+        ? []
+        : (await storageSelectFmt('select * from ~user_permissions where ~user_id = %v', id)).map(x => x.permission);
+    const allRoles = await storageSelectFmt('select * from ~roles');
+    const usedRoles =
+      id == 'new'
+        ? []
+        : (
+            await storageSelectFmt(`select ~user_roles.~role_id from ~user_roles where ~user_roles.~user_id = %v`, id)
+          ).map(x => x.role_id);
+    const allConnections = await storageSelectFmt('select * from ~connections');
+    const usedConnections =
+      id == 'new'
+        ? []
+        : (await storageSelectFmt('select ~connection_id from ~user_connections where ~user_id = %v', id)).map(
+            x => x.connection_id
+          );
+
+    return {
+      ...resp[0],
+      basePermissions,
+      permissions,
+      allRoles,
+      usedRoles,
+      allConnections,
+      usedConnections,
+      encryptPassword: !!(resp[0]?.password?.startsWith('crypt:') || id == 'new'),
+    };
+  },
+
+  saveUserDetail_meta: true,
+  async saveUserDetail(user) {
+    const [conn, driver] = await getStorageConnection();
+
+    const { login, password, email, usedRoles, usedConnections, permissions } = encryptUser(user);
+    let id = user.id;
+
+    if (id == null) {
+      await runQueryFmt(
+        driver,
+        conn,
+        'insert into ~users (~login, ~password, ~email) values (%v, %v, %v)',
+        login,
+        password,
+        email
+      );
+      id = await selectStorageIdentity('users');
+    } else {
+      await runQueryFmt(
+        driver,
+        conn,
+        'update ~users set ~login=%v, ~password=%v, ~email=%v where ~id = %v',
+        login,
+        password,
+        email,
+        id
+      );
+    }
+
+    await storageSaveRelationDiff('user_permissions', 'user_id', 'permission', id, permissions);
+    await storageSaveRelationDiff('user_roles', 'user_id', 'role_id', id, usedRoles);
+    await storageSaveRelationDiff('user_connections', 'user_id', 'connection_id', id, usedConnections);
+
+    return true;
+  },
+
+  deleteUser_meta: true,
+  async deleteUser({ id }) {
+    const [conn, driver] = await getStorageConnection();
+
+    await runQueryFmt(driver, conn, 'delete from ~users where ~id = %v', id);
+
+    return true;
+  },
+
+  getConnectionList_meta: true,
+  async getConnectionList() {
+    const resp = await storageSelectFmt(
+      `select ~connections.~id,~connections.~engine,~connections.~displayName,~connections.~server,~connections.~user from ~connections`
+    );
+    return resp;
+  },
+
+  getConnectionDetail_meta: true,
+  async getConnectionDetail({ id }) {
+    if (id == 'new') {
+      return {
+        conid: crypto.randomUUID(),
+      };
+    }
+
+    const resp = await storageSelectFmt(`select * from ~connections where ~id = %v`, id);
+    return resp[0];
+  },
+
+  saveConnectionDetail_meta: true,
+  async saveConnectionDetail(savedConnection) {
+    const [conn, driver] = await getStorageConnection();
+
+    const connection = encryptConnection(savedConnection);
+
+    const tableConnections = storageModel.tables.find(x => x.pureName == 'connections');
+    const id = connection.id;
+
+    const usedColumns = _.difference(
+      _.intersection(
+        tableConnections.columns.map(x => x.columnName),
+        Object.keys(connection)
+      ),
+      ['id']
+    );
+
+    if (id) {
+      await runQueryOnDriver(conn, driver, dmp => {
+        dmp.put(`update ~connections set`);
+        dmp.putCollection(',', usedColumns, col => {
+          dmp.put(`%i = %v`, col, connection[col]);
+        });
+        dmp.put(` where ~id = %v`, id);
+      });
+    } else {
+      await runQueryFmt(
+        driver,
+        conn,
+        `insert into ~connections (%,i) values (%,v)`,
+        usedColumns,
+        usedColumns.map(x => connection[x])
+      );
+    }
+
+    socket.emitChanged('connection-list-changed');
+
+    return true;
+  },
+
+  deleteConnection_meta: true,
+  async deleteConnection({ id }) {
+    const [conn, driver] = await getStorageConnection();
+    if (!conn) {
+      return null;
+    }
+
+    await runQueryFmt(driver, conn, 'delete from ~connections where ~id = %v', id);
+
+    socket.emitChanged('connection-list-changed');
+
+    return true;
+  },
+
+  getRoleList_meta: true,
+  async getRoleList() {
+    const resp = await storageSelectFmt(`select ~roles.~id,~roles.~name from ~roles`);
+    const usedPermissions = await storageSelectFmt(`select * from ~role_permissions`);
+    return resp.map(x => ({
+      ...x,
+      usedPermissions: usedPermissions.filter(y => y.role_id == x.id).map(y => y.permission),
+    }));
+  },
+
+  getRoleDetail_meta: true,
+  async getRoleDetail({ id }) {
+    const resp =
+      id == 'new' ? {} : await storageSelectFmt(`select ~roles.~id,~roles.~name from ~roles where ~roles.~id = %v`, id);
+
+    const basePermissions = getPredefinedPermissions(id < 0 && resp[0]?.name ? resp[0]?.name : 'logged-user');
+    const permissions =
+      id == 'new'
+        ? []
+        : (await storageSelectFmt('select * from ~role_permissions where ~role_id = %v', id)).map(x => x.permission);
+    const allUsers = await storageSelectFmt('select * from ~users');
+    const usedUsers =
+      id == 'new'
+        ? []
+        : (
+            await storageSelectFmt(`select ~user_roles.~user_id from ~user_roles where ~user_roles.~role_id = %v`, id)
+          ).map(x => x.user_id);
+    const allConnections = await storageSelectFmt('select * from ~connections');
+    const usedConnections =
+      id == 'new'
+        ? []
+        : (await storageSelectFmt('select ~connection_id from ~role_connections where ~role_id = %v', id)).map(
+            x => x.connection_id
+          );
+
+    return {
+      ...resp[0],
+      basePermissions,
+      permissions,
+      allUsers,
+      usedUsers,
+      allConnections,
+      usedConnections,
+    };
+  },
+
+  deleteRole_meta: true,
+  async deleteRole({ id }) {
+    const [conn, driver] = await getStorageConnection();
+    if (!conn) {
+      return null;
+    }
+
+    await runQueryFmt(driver, conn, 'delete from ~roles where ~id = %v', id);
+
+    return true;
+  },
+
+  saveRoleDetail_meta: true,
+  async saveRoleDetail(role) {
+    const [conn, driver] = await getStorageConnection();
+
+    const { name, usedConnections, usedUsers, permissions } = encryptUser(role);
+    let id = role.id;
+
+    if (id == null) {
+      await runQueryFmt(driver, conn, 'insert into ~roles (~name) values (%v)', name);
+      id = await selectStorageIdentity('roles');
+    } else {
+      if (id > 0) {
+        await runQueryFmt(driver, conn, 'update ~roles set ~name=%v where ~id = %v', name, id);
+      }
+    }
+
+    await storageSaveRelationDiff('role_permissions', 'role_id', 'permission', id, permissions);
+    await storageSaveRelationDiff('user_roles', 'role_id', 'user_id', id, usedUsers);
+    await storageSaveRelationDiff('role_connections', 'role_id', 'connection_id', id, usedConnections);
+
+    return true;
+  },
+
+  async getExportedDatabase() {
+    const connections = await storageSelectFmt(`select * from ~connections`);
+    const user_permissions = await storageSelectFmt(`select * from ~user_permissions`);
+    const users = await storageSelectFmt(`select * from ~users`);
+    const auth_methods = await storageSelectFmt(`select * from ~auth_methods`);
+    const role_connections = await storageSelectFmt(`select * from ~role_connections`);
+    const user_connections = await storageSelectFmt(`select * from ~user_connections`);
+    const user_roles = await storageSelectFmt(`select * from ~user_roles`);
+    const config = await storageSelectFmt(`select * from ~config`);
+    const auth_methods_config = await storageSelectFmt(`select * from ~auth_methods_config`);
+    const role_permissions = await storageSelectFmt(`select * from ~role_permissions`);
+    const roles = await storageSelectFmt(`select * from ~roles`);
+
+    return {
+      connections,
+      user_permissions,
+      users,
+      auth_methods,
+      role_connections,
+      user_connections,
+      user_roles,
+      config,
+      auth_methods_config,
+      role_permissions,
+      roles,
+    };
+  },
+
+  async replicateImportedDatabase(db) {
+    const [conn, driver] = await getStorageConnection();
+    // @ts-ignore
+    await dataReplicator({
+      systemConnection: conn,
+      driver,
+      items: storageReplicatorItems
+        .map(item => ({
+          ...item,
+          jsonArray: db[item.name],
+        }))
+        .filter(x => x.jsonArray),
+    });
+    socket.emitChanged('connection-list-changed');
+  },
 };