dbgate-api 4.6.3 → 4.7.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.6.3",
+  "version": "4.7.2",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.6.3",
-    "dbgate-sqltree": "^4.6.3",
-    "dbgate-tools": "^4.6.3",
+    "dbgate-query-splitter": "^4.7.2",
+    "dbgate-sqltree": "^4.7.2",
+    "dbgate-tools": "^4.7.2",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.6.3",
+    "dbgate-types": "^4.7.2",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/apps.js

@@ -21,11 +21,13 @@ module.exports = {
     const name = await this.getNewAppFolder({ name: folder });
     await fs.mkdir(path.join(appdir(), name));
     socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
     return name;
   },
 
   files_meta: true,
   async files({ folder }) {
+    if (!folder) return [];
     const dir = path.join(appdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
@@ -93,6 +95,8 @@ module.exports = {
     if (!folder) throw new Error('Missing folder parameter');
     await fs.rmdir(path.join(appdir(), folder), { recursive: true });
     socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
   },
 
   async getNewAppFolder({ name }) {

package/src/controllers/archive.js

@@ -141,6 +141,13 @@ module.exports = {
     });
   },
 
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
   async getNewArchiveFolder({ database }) {
     const isLink = database.endsWith(database);
     const name = isLink ? database.slice(0, -5) : database;

package/src/controllers/config.js

@@ -1,25 +1,29 @@
 const fs = require('fs-extra');
+const os = require('os');
 const path = require('path');
 const axios = require('axios');
 const { datadir } = require('../utility/directories');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
+const AsyncLock = require('async-lock');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
 
+const lock = new AsyncLock();
+
 module.exports = {
-  settingsValue: {},
+  // settingsValue: {},
 
-  async _init() {
-    try {
-      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
-    } catch (err) {
-      this.settingsValue = {};
-    }
-  },
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
 
   get_meta: true,
   async get() {
@@ -40,24 +44,45 @@ module.exports = {
 
   getSettings_meta: true,
   async getSettings() {
-    return this.settingsValue;
+    try {
+      return this.fillMissingSettings(
+        JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }))
+      );
+    } catch (err) {
+      return this.fillMissingSettings({});
+    }
+  },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+    }
+    return res;
   },
 
   updateSettings_meta: true,
   async updateSettings(values) {
     if (!hasPermission(`settings/change`)) return false;
-    try {
-      const updated = {
-        ...this.settingsValue,
-        ...values,
-      };
-      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
-      this.settingsValue = updated;
-      socket.emitChanged(`settings-changed`);
-      return updated;
-    } catch (err) {
-      return false;
-    }
+
+    const res = await lock.acquire('update', async () => {
+      const currentValue = await this.getSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
   },
 
   changelog_meta: true,

package/src/controllers/databaseConnections.js

@@ -62,9 +62,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },
@@ -109,7 +110,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
-      globalSettings: config.settingsValue,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },

package/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +11,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -108,7 +112,16 @@ module.exports = {
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
+    if (firstLine) {
+      const parsed = JSON.parse(firstLine);
+      if (parsed.__isStreamHeader) {
+        return parsed;
+      }
+      return {
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      };
+    }
     return null;
   },
 
@@ -132,7 +145,7 @@ module.exports = {
   },
 
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -151,4 +164,10 @@ module.exports = {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

package/src/controllers/queryHistory.js

@@ -48,7 +48,7 @@ module.exports = {
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');
-    socket.emitChanged('query-history-changed');
+    socket.emit('query-history-changed');
     return 'OK';
   },
 };

package/src/controllers/serverConnections.js

@@ -69,7 +69,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.6.3',
-  buildTime: '2022-02-07T20:42:15.958Z'
+  version: '4.7.2',
+  buildTime: '2022-03-07T17:31:11.445Z'
 };

package/src/main.js

@@ -28,8 +28,7 @@ const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
-
-let checkLocalhostOrigin = null;
+const getExpressPath = require('./utility/getExpressPath');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -50,32 +49,13 @@ function start() {
     );
   }
 
-  app.use(function (req, res, next) {
-    if (checkLocalhostOrigin) {
-      if (
-        req.headers.origin &&
-        req.headers.origin != checkLocalhostOrigin &&
-        req.headers.origin != `http://${checkLocalhostOrigin}`
-      ) {
-        console.log('API origin check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
-        console.log('API host check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-    }
-    next();
-  });
-
   app.use(cors());
 
-  app.get('/stream', async function (req, res) {
+  app.get(getExpressPath('/stream'), async function (req, res) {
     res.set({
       'Cache-Control': 'no-cache',
       'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
       Connection: 'keep-alive',
     });
     res.flushHeaders();
@@ -88,7 +68,7 @@ function start() {
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
-    '/uploads',
+    getExpressPath('/uploads'),
     fileUpload({
       limits: { fileSize: 4 * 1024 * 1024 },
     })
@@ -100,21 +80,21 @@ function start() {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
   // }
 
-  app.use('/runners/data', express.static(rundir()));
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
   if (platformInfo.isDocker) {
     // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/public'));
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
   } else {
     if (!platformInfo.isNpmDist) {
-      app.get('/', (req, res) => {
+      app.get(getExpressPath('/'), (req, res) => {
         res.send('DbGate API');
       });
     }
   }
 
   if (platformInfo.isNpmDist) {
-    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
         console.log(`DbGate API listening on port ${port}`);
@@ -165,4 +145,4 @@ function initializeElectronSender(electronSender) {
   socket.setElectronSender(electronSender);
 }
 
-module.exports = { start, useAllControllers, initializeElectronSender };
+module.exports = { start, useAllControllers, initializeElectronSender, configController: config };

package/src/proc/databaseConnectionProcess.js

@@ -18,6 +18,12 @@ let lastStatus = null;
 let analysedTime = 0;
 let serverVersion;
 
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -79,7 +85,7 @@ function handleSyncModel() {
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
+    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
     lastStatus = statusString;
   }
 }

package/src/shell/copyStream.js

@@ -2,12 +2,17 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
 
 function copyStream(input, output) {
   return new Promise((resolve, reject) => {
-    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(ensureHeader);
-    ensureHeader.pipe(output);
+
+    if (output.requireFixedStructure) {
+      const ensureHeader = new EnsureStreamHeaderStream();
+      input.pipe(ensureHeader);
+      ensureHeader.pipe(output);
+    } else {
+      input.pipe(output);
+    }
   });
 }
 

package/src/shell/jsonArrayWriter.js

@@ -11,10 +11,7 @@ class StringifyStream extends stream.Transform {
     let skip = false;
 
     if (!this.wasHeader) {
-      skip =
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns);
+      skip = chunk.__isStreamHeader;
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/jsonLinesReader.js

@@ -12,14 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (
-        !obj.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(obj.columns)
-      ) {
-        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
       }
-      
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {

package/src/shell/jsonLinesWriter.js

@@ -10,11 +10,7 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip =
-        (chunk.__isStreamHeader ||
-          // TODO remove isArray test
-          Array.isArray(chunk.columns)) &&
-        !this.header;
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/sqlDataWriter.js

@@ -14,11 +14,7 @@ class SqlizeStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      if (
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns)
-      ) {
+      if (chunk.__isStreamHeader) {
         skip = true;
         this.tableName = chunk.pureName;
         if (chunk.engine) {

package/src/utility/EnsureStreamHeaderStream.js

@@ -13,11 +13,7 @@ class EnsureStreamHeaderStream extends stream.Transform {
         return;
       }
 
-      if (
-        !chunk.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(chunk.columns)
-      ) {
+      if (!chunk.__isStreamHeader) {
         this.push({
           __isStreamHeader: true,
           __isComputedStructure: true,

package/src/utility/JsonLinesDatastore.js

@@ -27,6 +27,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
   }
@@ -37,6 +38,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     reader.close(() => {});
   }
@@ -61,6 +63,11 @@ class JsonLinesDatastore {
   }
 
   async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
     for (;;) {
       const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
@@ -70,7 +77,11 @@ class JsonLinesDatastore {
 
       if (!this.readedSchemaRow) {
         this.readedSchemaRow = true;
-        return true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
       }
       if (this.currentFilter) {
         const parsedLine = JSON.parse(line);
@@ -130,11 +141,21 @@ class JsonLinesDatastore {
       this.reader = reader;
       this.currentFilter = filter;
     }
-    if (!this.readedSchemaRow) {
-      await this._readLine(false); // skip structure
-    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
     while (this.readedDataRowCount < offset) {
-      await this._readLine(false);
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
     }
   }
 
@@ -148,7 +169,6 @@ class JsonLinesDatastore {
         res.push(line);
       }
     });
-    // console.log('RETURN', res.length);
     return res;
   }
 }

package/src/utility/getExpressPath.js

@@ -0,0 +1,10 @@
+function getExpressPath(path) {
+  path = path.replace(/\/*$/, '').replace(/^\/*/, '');
+  const root = (process.env.WEB_ROOT || '').replace(/^\/*/, '').replace(/\/*$/, '');
+  if (root) {
+    return `/${root}/${path}`;
+  }
+  return `/${path}`;
+}
+
+module.exports = getExpressPath;

package/src/utility/socket.js

@@ -5,6 +5,7 @@ let init = '';
 module.exports = {
   setSseResponse(value) {
     sseResponse = value;
+    setInterval(() => this.emit('ping'), 29 * 1000);
   },
   setElectronSender(value) {
     electronSender = value;
@@ -23,7 +24,8 @@ module.exports = {
     }
   },
   emitChanged(key) {
-    this.emit('clean-cache', key);
-    this.emit(key);
+    // console.log('EMIT CHANGED', key);
+    this.emit('changed-cache', key);
+    // this.emit(key);
   },
 };

package/src/utility/useController.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const express = require('express');
+const getExpressPath = require('./getExpressPath');
 
 /**
  * @param {string} route
@@ -74,6 +75,6 @@ module.exports = function useController(app, electron, route, controller) {
   }
 
   if (app) {
-    app.use(route, router);
+    app.use(getExpressPath(route), router);
   }
 };

package/webpack.config.js

@@ -10,16 +10,16 @@ var config = {
   target: 'node',
   node: {
     __dirname: false,
-  },  
+  },
   output: {
     path: path.resolve(__dirname, 'dist'),
     filename: 'bundle.js',
     libraryTarget: 'commonjs2',
   },
 
-//   optimization: {
-//     minimize: false,
-//   },
+  // optimization: {
+  //   minimize: false,
+  // },
 
   module: {
     rules: [
@@ -45,6 +45,9 @@ var config = {
       },
     }),
   ],
+  externals: {
+    'better-sqlite3': 'commonjs better-sqlite3',
+  },
 };
 
 module.exports = config;