dbgate-api 4.6.3 → 4.7.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.6.3",
+  "version": "4.7.0",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.6.3",
-    "dbgate-sqltree": "^4.6.3",
-    "dbgate-tools": "^4.6.3",
+    "dbgate-query-splitter": "^4.7.0",
+    "dbgate-sqltree": "^4.7.0",
+    "dbgate-tools": "^4.7.0",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -63,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.6.3",
+    "dbgate-types": "^4.7.0",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/apps.js

@@ -21,6 +21,7 @@ module.exports = {
     const name = await this.getNewAppFolder({ name: folder });
     await fs.mkdir(path.join(appdir(), name));
     socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
     return name;
   },
 
@@ -93,6 +94,8 @@ module.exports = {
     if (!folder) throw new Error('Missing folder parameter');
     await fs.rmdir(path.join(appdir(), folder), { recursive: true });
     socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
   },
 
   async getNewAppFolder({ name }) {

package/src/controllers/archive.js

@@ -141,6 +141,13 @@ module.exports = {
     });
   },
 
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
   async getNewArchiveFolder({ database }) {
     const isLink = database.endsWith(database);
     const name = isLink ? database.slice(0, -5) : database;

package/src/controllers/databaseConnections.js

@@ -62,9 +62,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },

package/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +11,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -108,7 +112,16 @@ module.exports = {
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
+    if (firstLine) {
+      const parsed = JSON.parse(firstLine);
+      if (parsed.__isStreamHeader) {
+        return parsed;
+      }
+      return {
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      };
+    }
     return null;
   },
 
@@ -132,7 +145,7 @@ module.exports = {
   },
 
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -151,4 +164,10 @@ module.exports = {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

package/src/controllers/queryHistory.js

@@ -48,7 +48,7 @@ module.exports = {
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');
-    socket.emitChanged('query-history-changed');
+    socket.emit('query-history-changed');
     return 'OK';
   },
 };

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.6.3',
-  buildTime: '2022-02-07T20:42:15.958Z'
+  version: '4.7.0',
+  buildTime: '2022-02-21T18:54:46.663Z'
 };

package/src/proc/databaseConnectionProcess.js

@@ -18,6 +18,12 @@ let lastStatus = null;
 let analysedTime = 0;
 let serverVersion;
 
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -79,7 +85,7 @@ function handleSyncModel() {
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
+    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
     lastStatus = statusString;
   }
 }

package/src/shell/copyStream.js

@@ -2,12 +2,17 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
 
 function copyStream(input, output) {
   return new Promise((resolve, reject) => {
-    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(ensureHeader);
-    ensureHeader.pipe(output);
+
+    if (output.requireFixedStructure) {
+      const ensureHeader = new EnsureStreamHeaderStream();
+      input.pipe(ensureHeader);
+      ensureHeader.pipe(output);
+    } else {
+      input.pipe(output);
+    }
   });
 }
 

package/src/shell/jsonArrayWriter.js

@@ -11,10 +11,7 @@ class StringifyStream extends stream.Transform {
     let skip = false;
 
     if (!this.wasHeader) {
-      skip =
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns);
+      skip = chunk.__isStreamHeader;
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/jsonLinesReader.js

@@ -12,14 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (
-        !obj.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(obj.columns)
-      ) {
-        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
       }
-      
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {

package/src/shell/jsonLinesWriter.js

@@ -10,11 +10,7 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip =
-        (chunk.__isStreamHeader ||
-          // TODO remove isArray test
-          Array.isArray(chunk.columns)) &&
-        !this.header;
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/sqlDataWriter.js

@@ -14,11 +14,7 @@ class SqlizeStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      if (
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns)
-      ) {
+      if (chunk.__isStreamHeader) {
         skip = true;
         this.tableName = chunk.pureName;
         if (chunk.engine) {

package/src/utility/EnsureStreamHeaderStream.js

@@ -13,11 +13,7 @@ class EnsureStreamHeaderStream extends stream.Transform {
         return;
       }
 
-      if (
-        !chunk.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(chunk.columns)
-      ) {
+      if (!chunk.__isStreamHeader) {
         this.push({
           __isStreamHeader: true,
           __isComputedStructure: true,

package/src/utility/JsonLinesDatastore.js

@@ -27,6 +27,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
   }
@@ -37,6 +38,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     reader.close(() => {});
   }
@@ -61,6 +63,11 @@ class JsonLinesDatastore {
   }
 
   async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
     for (;;) {
       const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
@@ -70,7 +77,11 @@ class JsonLinesDatastore {
 
       if (!this.readedSchemaRow) {
         this.readedSchemaRow = true;
-        return true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
       }
       if (this.currentFilter) {
         const parsedLine = JSON.parse(line);
@@ -130,11 +141,21 @@ class JsonLinesDatastore {
       this.reader = reader;
       this.currentFilter = filter;
     }
-    if (!this.readedSchemaRow) {
-      await this._readLine(false); // skip structure
-    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
     while (this.readedDataRowCount < offset) {
-      await this._readLine(false);
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
     }
   }
 
@@ -148,7 +169,6 @@ class JsonLinesDatastore {
         res.push(line);
       }
     });
-    // console.log('RETURN', res.length);
     return res;
   }
 }

package/src/utility/socket.js

@@ -23,7 +23,8 @@ module.exports = {
     }
   },
   emitChanged(key) {
-    this.emit('clean-cache', key);
-    this.emit(key);
+    // console.log('EMIT CHANGED', key);
+    this.emit('changed-cache', key);
+    // this.emit(key);
   },
 };

package/webpack.config.js

@@ -10,16 +10,16 @@ var config = {
   target: 'node',
   node: {
     __dirname: false,
-  },  
+  },
   output: {
     path: path.resolve(__dirname, 'dist'),
     filename: 'bundle.js',
     libraryTarget: 'commonjs2',
   },
 
-//   optimization: {
-//     minimize: false,
-//   },
+  // optimization: {
+  //   minimize: false,
+  // },
 
   module: {
     rules: [
@@ -45,6 +45,9 @@ var config = {
       },
     }),
   ],
+  externals: {
+    'better-sqlite3': 'commonjs better-sqlite3',
+  },
 };
 
 module.exports = config;