dbgate-api 4.7.3 → 4.7.4-alpha.3

package/.env

@@ -1 +1,3 @@
 DEVMODE=1
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1

package/env/singledb/.env

@@ -8,6 +8,8 @@ USER_mysql=root
 PASSWORD_mysql=test
 PORT_mysql=3307
 ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
 
 SINGLE_CONNECTION=mysql
 SINGLE_DATABASE=Chinook

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.7.3",
+  "version": "4.7.4-alpha.3",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.7.3",
-    "dbgate-sqltree": "^4.7.3",
-    "dbgate-tools": "^4.7.3",
+    "dbgate-query-splitter": "^4.7.4-alpha.3",
+    "dbgate-sqltree": "^4.7.4-alpha.3",
+    "dbgate-tools": "^4.7.4-alpha.3",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.7.3",
+    "dbgate-types": "^4.7.4-alpha.3",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/config.js

@@ -32,6 +32,7 @@ module.exports = {
     return {
       runAsPortal: !!connections.portalConnections,
       singleDatabase: connections.singleDatabase,
+      hideAppEditor: !!process.env.HIDE_APP_EDITOR,
       permissions,
       ...currentVersion,
     };

package/src/controllers/connections.js

@@ -11,6 +11,7 @@ const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
+const { safeJsonParse } = require('dbgate-tools');
 
 function getNamedArgs() {
   const res = {};
@@ -55,6 +56,8 @@ function getPortalCollections() {
         (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
       singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],
@@ -199,6 +202,9 @@ module.exports = {
     }
     socket.emitChanged('connection-list-changed');
     socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
     // for (const db of connection.databases || []) {
     //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
     // }
@@ -240,6 +246,7 @@ module.exports = {
 
   get_meta: true,
   async get({ conid }) {
+    if (!conid) return null;
     if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
     const res = await this.datastore.get(conid);
     return res || null;

package/src/controllers/databaseConnections.js

@@ -33,6 +33,10 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
@@ -136,6 +140,13 @@ module.exports = {
     return res;
   },
 
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
   runScript_meta: true,
   async runScript({ conid, database, sql }) {
     console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
@@ -148,14 +159,64 @@ module.exports = {
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
-    return res.result;
+    return res.result || null;
+  },
+
+  async loadDataCore(msgtype, { conid, database, ...args }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root }) {
+    return this.loadDataCore('loadKeys', { conid, database, root });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }) {
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }) {
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }) {
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }) {
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
   },
 
   updateCollection_meta: true,
   async updateCollection({ conid, database, changeSet }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
-    return res.result;
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
   },
 
   status_meta: true,
@@ -228,6 +289,12 @@ module.exports = {
     }
   },
 
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
+    }
+  },
+
   disconnect_meta: true,
   async disconnect({ conid, database }) {
     await this.close(conid, database, true);

package/src/controllers/jsldata.js

@@ -111,18 +111,22 @@ module.exports = {
   getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) {
-      const parsed = JSON.parse(firstLine);
-      if (parsed.__isStreamHeader) {
-        return parsed;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
       }
-      return {
-        __isStreamHeader: true,
-        __isDynamicStructure: true,
-      };
+      return null;
+    } catch (err) {
+      return null;
     }
-    return null;
   },
 
   getRows_meta: true,

package/src/controllers/runners.js

@@ -148,7 +148,11 @@ module.exports = {
   },
 
   start_meta: true,
-  async start({ script }) {
+  async start({ script, isGeneratedScript }) {
+    if (!isGeneratedScript && process.env.DISABLE_SHELL) {
+      return { errorMessage: 'Shell is disabled' };
+    }
+
     const runid = uuidv1();
     return this.startCore(runid, scriptTemplate(script, false));
   },

package/src/controllers/sessions.js

@@ -4,8 +4,10 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -46,9 +48,15 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -60,6 +68,11 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
   create_meta: true,
@@ -105,6 +118,19 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.7.3',
-  buildTime: '2022-03-14T19:11:28.854Z'
+  version: '4.7.4-alpha.3',
+  buildTime: '2022-03-17T19:22:21.077Z'
 };

package/src/proc/databaseConnectionProcess.js

@@ -7,6 +7,7 @@ const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
 
 let systemConnection;
 let storedConnection;
@@ -154,6 +155,7 @@ async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     await driver.script(systemConnection, sql);
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
@@ -165,6 +167,7 @@ async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
@@ -172,21 +175,67 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
-async function handleCollectionData({ msgid, options }) {
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s });
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
-    const result = await driver.readCollection(systemConnection, options);
+    const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
+async function handleCollectionData({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
+  }
+}
+
 async function handleUpdateCollection({ msgid, changeSet }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const result = await driver.updateCollection(systemConnection, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
@@ -248,10 +297,16 @@ const messageHandlers = {
   runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
   generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
   // runCommand: handleRunCommand,
 };
 

package/src/proc/sessionProcess.js

@@ -17,11 +17,15 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +36,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -98,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -135,6 +166,17 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
@@ -163,6 +205,19 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
@@ -176,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

package/src/shell/tableReader.js

@@ -9,7 +9,7 @@ async function tableReader({ connection, pureName, schemaName }) {
 
   const fullName = { pureName, schemaName };
 
-  if (driver.dialect.nosql) {
+  if (driver.databaseEngineTypes.includes('document')) {
     // @ts-ignore
     console.log(`Reading collection ${fullNameToString(fullName)}`);
     // @ts-ignore