dbgate-api 4.6.0 → 4.7.0

package/.env-portal

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres,mongo,mongo2
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -24,4 +24,21 @@ LABEL_mongo2=Mongo Server
 SERVER_mongo2=localhost
 ENGINE_mongo2=mongo@dbgate-plugin-mongo
 
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
+LABEL_sqlite=sqlite
+FILE_sqlite=/home/jena/dbgate-data/files/sqlite/feeds.sqlite
+ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
 # docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.6.0",
+  "version": "4.7.0",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,9 +25,9 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.6.0",
-    "dbgate-sqltree": "^4.6.0",
-    "dbgate-tools": "^4.6.0",
+    "dbgate-query-splitter": "^4.7.0",
+    "dbgate-sqltree": "^4.7.0",
+    "dbgate-tools": "^4.7.0",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
@@ -44,7 +44,6 @@
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
@@ -64,7 +63,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.6.0",
+    "dbgate-types": "^4.7.0",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/apps.js

@@ -0,0 +1,267 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged(`app-files-changed-${folder}`);
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+};

package/src/controllers/archive.js

@@ -141,6 +141,13 @@ module.exports = {
     });
   },
 
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
   async getNewArchiveFolder({ database }) {
     const isLink = database.endsWith(database);
     const name = isLink ? database.slice(0, -5) : database;

package/src/controllers/connections.js

@@ -1,7 +1,6 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
 const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
@@ -9,6 +8,7 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
 
@@ -50,9 +50,29 @@ function getPortalCollections() {
       databaseUrl: process.env[`URL_${id}`],
       useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
-      defaultDatabase: process.env[`DATABASE_${id}`],
-      singleDatabase: !!process.env[`DATABASE_${id}`],
+      defaultDatabase:
+        process.env[`DATABASE_${id}`] ||
+        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
+      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
     console.log('Using connections from ENV variables:');
     console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
@@ -136,7 +156,7 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
@@ -173,18 +193,22 @@ module.exports = {
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
   update_meta: true,
   async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.update({ _id }, { $set: values });
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -192,22 +216,24 @@ module.exports = {
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }) {
     if (portalConnections) return;
-    const conn = await this.datastore.find({ _id: conid });
-    let databases = conn[0].databases || [];
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
     if (databases.find(x => x.name == database)) {
       databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
     } else {
       databases = [...databases, { name: database, ...values }];
     }
-    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    const res = await this.datastore.patch(conid, { databases });
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
     return res;
   },
 
   delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.remove(connection._id);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -215,8 +241,8 @@ module.exports = {
   get_meta: true,
   async get({ conid }) {
     if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
-    const res = await this.datastore.find({ _id: conid });
-    return res[0] || null;
+    const res = await this.datastore.get(conid);
+    return res || null;
   },
 
   newSqliteDatabase_meta: true,

package/src/controllers/databaseConnections.js

@@ -62,9 +62,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },

package/src/controllers/files.js

@@ -1,12 +1,13 @@
 const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir, archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -74,6 +75,11 @@ module.exports = {
         encoding: 'utf-8',
       });
       return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
     } else {
       if (!hasPermission(`files/${folder}/read`)) return null;
       const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
@@ -88,6 +94,12 @@ module.exports = {
       await fs.writeFile(path.join(dir, file), serialize(format, data));
       socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
       return true;
+    } else if (folder.startsWith('app:')) {
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed-${app}`);
+      apps.emitChangedDbApp(folder);
+      return true;
     } else {
       if (!hasPermission(`files/${folder}/write`)) return false;
       const dir = path.join(filesdir(), folder);

package/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +11,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -108,7 +112,16 @@ module.exports = {
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
+    if (firstLine) {
+      const parsed = JSON.parse(firstLine);
+      if (parsed.__isStreamHeader) {
+        return parsed;
+      }
+      return {
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      };
+    }
     return null;
   },
 
@@ -132,7 +145,7 @@ module.exports = {
   },
 
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -151,4 +164,10 @@ module.exports = {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

package/src/controllers/plugins.js

@@ -72,7 +72,8 @@ module.exports = {
 
     const res = [];
     for (const packageName of _.union(files1, files2)) {
-      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      if (packageName == 'dist') continue;
+      // if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
           const manifest = {

package/src/controllers/queryHistory.js

@@ -48,7 +48,7 @@ module.exports = {
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');
-    socket.emitChanged('query-history-changed');
+    socket.emit('query-history-changed');
     return 'OK';
   },
 };

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.6.0',
-  buildTime: '2022-01-26T16:57:37.630Z'
+  version: '4.7.0',
+  buildTime: '2022-02-21T18:54:46.663Z'
 };

package/src/main.js

@@ -19,6 +19,7 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -157,6 +158,7 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/files', files);
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
 }
 
 function initializeElectronSender(electronSender) {

package/src/proc/databaseConnectionProcess.js

@@ -18,6 +18,12 @@ let lastStatus = null;
 let analysedTime = 0;
 let serverVersion;
 
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -79,7 +85,7 @@ function handleSyncModel() {
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
+    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
     lastStatus = statusString;
   }
 }

package/src/shell/copyStream.js

@@ -2,12 +2,17 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
 
 function copyStream(input, output) {
   return new Promise((resolve, reject) => {
-    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(ensureHeader);
-    ensureHeader.pipe(output);
+
+    if (output.requireFixedStructure) {
+      const ensureHeader = new EnsureStreamHeaderStream();
+      input.pipe(ensureHeader);
+      ensureHeader.pipe(output);
+    } else {
+      input.pipe(output);
+    }
   });
 }
 

package/src/shell/jsonArrayWriter.js

@@ -11,10 +11,7 @@ class StringifyStream extends stream.Transform {
     let skip = false;
 
     if (!this.wasHeader) {
-      skip =
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns);
+      skip = chunk.__isStreamHeader;
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/jsonLinesReader.js

@@ -12,14 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (
-        !obj.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(obj.columns)
-      ) {
-        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
       }
-      
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {

package/src/shell/jsonLinesWriter.js

@@ -10,11 +10,7 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip =
-        (chunk.__isStreamHeader ||
-          // TODO remove isArray test
-          Array.isArray(chunk.columns)) &&
-        !this.header;
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
     }
     if (!skip) {

package/src/shell/sqlDataWriter.js

@@ -14,11 +14,7 @@ class SqlizeStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      if (
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns)
-      ) {
+      if (chunk.__isStreamHeader) {
         skip = true;
         this.tableName = chunk.pureName;
         if (chunk.engine) {

package/src/utility/EnsureStreamHeaderStream.js

@@ -13,11 +13,7 @@ class EnsureStreamHeaderStream extends stream.Transform {
         return;
       }
 
-      if (
-        !chunk.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(chunk.columns)
-      ) {
+      if (!chunk.__isStreamHeader) {
         this.push({
           __isStreamHeader: true,
           __isComputedStructure: true,

package/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,142 @@
+const AsyncLock = require('async-lock');
+const fs = require('fs-extra');
+const uuidv1 = require('uuid/v1');
+
+const lock = new AsyncLock();
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
+  async _save() {
+    if (!this.loadedOk) {
+      // don't override data
+      return;
+    }
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loadPerformed) {
+      await lock.acquire('reader', async () => {
+        if (!this.loadPerformed) {
+          if (!(await fs.exists(this.filename))) {
+            this.loadedOk = true;
+            this.loadPerformed = true;
+            return;
+          }
+          try {
+            const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+            this.data = text
+              .split('\n')
+              .filter(x => x.trim())
+              .map(x => JSON.parse(x));
+            this.loadedOk = true;
+          } catch (err) {
+            console.error(`Error loading file ${this.filename}`, err);
+          }
+          this.loadPerformed = true;
+        }
+      });
+    }
+  }
+
+  async insert(obj) {
+    await this._ensureLoaded();
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: uuidv1(),
+        };
+    this.data.push(elem);
+    await this._save();
+    return elem;
+  }
+
+  async get(id) {
+    await this._ensureLoaded();
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    await this._ensureLoaded();
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._save();
+    return obj;
+  }
+
+  async patch(id, values) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._save();
+    return this.data.find(x => x._id == id);
+  }
+
+  async remove(id) {
+    await this._ensureLoaded();
+    const removed = this.data.find(x => x._id == id);
+    this.data = this.data.filter(x => x._id != id);
+    await this._save();
+    return removed;
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}
+
+module.exports = JsonLinesDatabase;

package/src/utility/JsonLinesDatastore.js

@@ -4,7 +4,7 @@ const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 
-async function fetchNextLine(reader) {
+function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
     if (!reader.hasNextLine()) {
       resolve(null);
@@ -27,6 +27,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
   }
@@ -37,6 +38,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     reader.close(() => {});
   }
@@ -61,8 +63,13 @@ class JsonLinesDatastore {
   }
 
   async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
     for (;;) {
-      const line = await fetchNextLine(this.reader);
+      const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
         // EOF
         return null;
@@ -70,7 +77,11 @@ class JsonLinesDatastore {
 
       if (!this.readedSchemaRow) {
         this.readedSchemaRow = true;
-        return true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
       }
       if (this.currentFilter) {
         const parsedLine = JSON.parse(line);
@@ -130,11 +141,21 @@ class JsonLinesDatastore {
       this.reader = reader;
       this.currentFilter = filter;
     }
-    if (!this.readedSchemaRow) {
-      await this._readLine(false); // skip structure
-    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
     while (this.readedDataRowCount < offset) {
-      await this._readLine(false);
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
     }
   }
 
@@ -148,7 +169,6 @@ class JsonLinesDatastore {
         res.push(line);
       }
     });
-    // console.log('RETURN', res.length);
     return res;
   }
 }

package/src/utility/directories.js

@@ -38,6 +38,7 @@ const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
+const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
 
 function packagedPluginsDir() {
@@ -103,6 +104,7 @@ module.exports = {
   rundir,
   uploadsdir,
   archivedir,
+  appdir,
   ensureDirectory,
   pluginsdir,
   filesdir,

package/src/utility/socket.js

@@ -23,7 +23,8 @@ module.exports = {
     }
   },
   emitChanged(key) {
-    this.emit('clean-cache', key);
-    this.emit(key);
+    // console.log('EMIT CHANGED', key);
+    this.emit('changed-cache', key);
+    // this.emit(key);
   },
 };

package/webpack.config.js

@@ -10,16 +10,16 @@ var config = {
   target: 'node',
   node: {
     __dirname: false,
-  },  
+  },
   output: {
     path: path.resolve(__dirname, 'dist'),
     filename: 'bundle.js',
     libraryTarget: 'commonjs2',
   },
 
-//   optimization: {
-//     minimize: false,
-//   },
+  // optimization: {
+  //   minimize: false,
+  // },
 
   module: {
     rules: [
@@ -45,6 +45,9 @@ var config = {
       },
     }),
   ],
+  externals: {
+    'better-sqlite3': 'commonjs better-sqlite3',
+  },
 };
 
 module.exports = config;