dbgate-api-premium 5.5.7-alpha.45

package/src/controllers/files.js

@@ -0,0 +1,222 @@
+const fs = require('fs-extra');
+const path = require('path');
+const crypto = require('crypto');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
+const getChartExport = require('../utility/getChartExport');
+const { hasPermission } = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
+const getMapExport = require('../utility/getMapExport');
+
+function serialize(format, data) {
+  if (format == 'text') return data;
+  if (format == 'json') return JSON.stringify(data);
+  throw new Error(`Invalid format: ${format}`);
+}
+
+function deserialize(format, text) {
+  if (format == 'text') return text;
+  if (format == 'json') return JSON.parse(text);
+  throw new Error(`Invalid format: ${format}`);
+}
+
+module.exports = {
+  list_meta: true,
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
+    const dir = path.join(filesdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
+    return files;
+  },
+
+  listAll_meta: true,
+  async listAll(_params, req) {
+    const folders = await fs.readdir(filesdir());
+    const res = [];
+    for (const folder of folders) {
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
+      const dir = path.join(filesdir(), folder);
+      const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
+      res.push(...files);
+    }
+    return res;
+  },
+
+  delete_meta: true,
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.unlink(path.join(filesdir(), folder, file));
+    socket.emitChanged(`files-changed`, { folder });
+    socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  rename_meta: true,
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
+    socket.emitChanged(`files-changed`, { folder });
+    socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  refresh_meta: true,
+  async refresh({ folders }, req) {
+    for (const folder of folders) {
+      socket.emitChanged(`files-changed`, { folder });
+      socket.emitChanged(`all-files-changed`);
+    }
+    return true;
+  },
+
+  copy_meta: true,
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
+    socket.emitChanged(`files-changed`, { folder });
+    socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  load_meta: true,
+  async load({ folder, file, format }, req) {
+    if (folder.startsWith('archive:')) {
+      const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else {
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
+      const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+      return deserialize(format, text);
+    }
+  },
+
+  loadFrom_meta: true,
+  async loadFrom({ filePath, format }, req) {
+    const text = await fs.readFile(filePath, { encoding: 'utf-8' });
+    return deserialize(format, text);
+  },
+
+  save_meta: true,
+  async save({ folder, file, data, format }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) });
+      return true;
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed`, { app });
+      socket.emitChanged('used-apps-changed');
+      apps.emitChangedDbApp(folder);
+      return true;
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`files-changed`, { folder });
+      socket.emitChanged(`all-files-changed`);
+      if (folder == 'shell') {
+        scheduler.reload();
+      }
+      return true;
+    }
+  },
+
+  saveAs_meta: true,
+  async saveAs({ filePath, data, format }) {
+    await fs.writeFile(filePath, serialize(format, data));
+  },
+
+  favorites_meta: true,
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
+    const dir = path.join(filesdir(), 'favorites');
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+    const res = [];
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      const text = await fs.readFile(filePath, { encoding: 'utf-8' });
+      res.push({
+        file,
+        folder: 'favorites',
+        ...JSON.parse(text),
+      });
+    }
+    return res;
+  },
+
+  generateUploadsFile_meta: true,
+  async generateUploadsFile({ extension }) {
+    const fileName = `${crypto.randomUUID()}.${extension || 'html'}`;
+    return {
+      fileName,
+      filePath: path.join(uploadsdir(), fileName),
+    };
+  },
+
+  exportChart_meta: true,
+  async exportChart({ filePath, title, config, image }) {
+    const fileName = path.parse(filePath).base;
+    const imageFile = fileName.replace('.html', '-preview.png');
+    const html = getChartExport(title, config, imageFile);
+    await fs.writeFile(filePath, html);
+    if (image) {
+      const index = image.indexOf('base64,');
+      if (index > 0) {
+        const data = image.substr(index + 'base64,'.length);
+        const buf = Buffer.from(data, 'base64');
+        await fs.writeFile(filePath.replace('.html', '-preview.png'), buf);
+      }
+    }
+    return true;
+  },
+
+  exportMap_meta: true,
+  async exportMap({ filePath, geoJson }) {
+    await fs.writeFile(filePath, getMapExport(geoJson));
+    return true;
+  },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
+    return true;
+  },
+
+  getFileRealPath_meta: true,
+  async getFileRealPath({ folder, file }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      return path.join(dir, file);
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      return path.join(appdir(), app, file);
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      return path.join(dir, file);
+    }
+  },
+};

package/src/controllers/jsldata.js

@@ -0,0 +1,296 @@
+const { filterName } = require('dbgate-tools');
+const fs = require('fs');
+const lineReader = require('line-reader');
+const _ = require('lodash');
+const { __ } = require('lodash/fp');
+const DatastoreProxy = require('../utility/DatastoreProxy');
+const getJslFileName = require('../utility/getJslFileName');
+const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
+const socket = require('../utility/socket');
+
+function readFirstLine(file) {
+  return new Promise((resolve, reject) => {
+    lineReader.open(file, (err, reader) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      if (reader.hasNextLine()) {
+        reader.nextLine((err, line) => {
+          if (err) {
+            reader.close(() => reject(err)); // Ensure reader is closed on error
+            return;
+          }
+          reader.close(() => resolve(line)); // Ensure reader is closed after reading
+        });
+      } else {
+        reader.close(() => resolve(null)); // Properly close if no lines are present
+      }
+    });
+  });
+}
+
+module.exports = {
+  datastores: {},
+
+  // closeReader(jslid) {
+  //   // console.log('CLOSING READER');
+  //   if (!this.openedReaders[jslid]) return Promise.resolve();
+  //   return new Promise((resolve, reject) => {
+  //     this.openedReaders[jslid].reader.close((err) => {
+  //       if (err) reject(err);
+  //       delete this.openedReaders[jslid];
+  //       resolve();
+  //     });
+  //   });
+  // },
+
+  // readLine(readerInfo) {
+  //   return new Promise((resolve, reject) => {
+  //     const { reader } = readerInfo;
+  //     if (!reader.hasNextLine()) {
+  //       resolve(null);
+  //       return;
+  //     }
+  //     reader.nextLine((err, line) => {
+  //       if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1;
+  //       else readerInfo.readedSchemaRow = true;
+  //       if (err) reject(err);
+  //       resolve(line);
+  //     });
+  //   });
+  // },
+
+  // openReader(jslid) {
+  //   // console.log('OPENING READER');
+  //   // console.log(
+  //   //   'OPENING READER, LINES=',
+  //   //   fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length
+  //   // );
+  //   const file = getJslFileName(jslid);
+  //   return new Promise((resolve, reject) =>
+  //     lineReader.open(file, (err, reader) => {
+  //       if (err) reject(err);
+  //       const readerInfo = {
+  //         reader,
+  //         readedDataRowCount: 0,
+  //         readedSchemaRow: false,
+  //         isReading: true,
+  //       };
+  //       this.openedReaders[jslid] = readerInfo;
+  //       resolve(readerInfo);
+  //     })
+  //   );
+  // },
+
+  // async ensureReader(jslid, offset) {
+  //   if (this.openedReaders[jslid] && this.openedReaders[jslid].readedDataRowCount > offset) {
+  //     await this.closeReader(jslid);
+  //   }
+  //   let readerInfo = this.openedReaders[jslid];
+  //   if (!this.openedReaders[jslid]) {
+  //     readerInfo = await this.openReader(jslid);
+  //   }
+  //   readerInfo.isReading = true;
+  //   if (!readerInfo.readedSchemaRow) {
+  //     await this.readLine(readerInfo); // skip structure
+  //   }
+  //   while (readerInfo.readedDataRowCount < offset) {
+  //     await this.readLine(readerInfo);
+  //   }
+  //   return readerInfo;
+  // },
+
+  async ensureDatastore(jslid, formatterFunction) {
+    let datastore = this.datastores[jslid];
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
+      // datastore = new DatastoreProxy(getJslFileName(jslid));
+      this.datastores[jslid] = datastore;
+    }
+    return datastore;
+  },
+
+  async closeDataStore(jslid) {
+    const datastore = this.datastores[jslid];
+    if (datastore) {
+      await datastore._closeReader();
+      delete this.datastores[jslid];
+    }
+  },
+
+  getInfo_meta: true,
+  async getInfo({ jslid }) {
+    const file = getJslFileName(jslid);
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
+      }
+      return null;
+    } catch (err) {
+      return null;
+    }
+  },
+
+  getRows_meta: true,
+  async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
+  },
+
+  exists_meta: true,
+  async exists({ jslid }) {
+    const fileName = getJslFileName(jslid);
+    return fs.existsSync(fileName);
+  },
+
+  getStats_meta: true,
+  getStats({ jslid }) {
+    const file = `${getJslFileName(jslid)}.stats`;
+    if (fs.existsSync(file)) {
+      try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+      } catch (e) {
+        return {};
+      }
+    }
+    return {};
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    const res = new Set();
+    await datastore.enumRows(row => {
+      if (!filterName(search, row[field])) return true;
+      res.add(row[field]);
+      return res.size < 100;
+    });
+    // @ts-ignore
+    return [...res].map(value => ({ value }));
+  },
+
+  async notifyChangedStats(stats) {
+    // console.log('SENDING STATS', JSON.stringify(stats));
+    const datastore = this.datastores[stats.jslid];
+    if (datastore) await datastore.notifyChanged();
+    socket.emit(`jsldata-stats-${stats.jslid}`, stats);
+
+    // const readerInfo = this.openedReaders[stats.jslid];
+    // if (readerInfo && readerInfo.isReading) {
+    //   readerInfo.closeAfterReadAndSendStats = stats;
+    // } else {
+    //   await this.closeReader(stats.jslid);
+    //   socket.emit(`jsldata-stats-${stats.jslid}`, stats);
+    // }
+  },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
+
+  saveRows_meta: true,
+  async saveRows({ jslid, rows }) {
+    const fileStream = fs.createWriteStream(getJslFileName(jslid));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    return true;
+  },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
+};

package/src/controllers/metadata.js

@@ -0,0 +1,47 @@
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const databaseConnections = require('./databaseConnections');
+
+function pickObjectNames(array) {
+  return _.sortBy(array, x => `${x.schemaName}.${x.pureName}`).map(fp.pick(['pureName', 'schemaName']));
+}
+
+module.exports = {
+  // tableData_meta: true,
+  // async tableData({ conid, database, schemaName, pureName }) {
+  //   const opened = await databaseConnections.ensureOpened(conid, database);
+  //   const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
+  //   return res;
+  // },
+
+  listObjects_meta: true,
+  async listObjects({ conid, database }) {
+    const opened = await databaseConnections.ensureOpened(conid, database);
+    const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
+    return types.reduce(
+      (res, type) => ({
+        ...res,
+        [type]: pickObjectNames(opened.structure[type]),
+      }),
+      {}
+    );
+  },
+
+  tableInfo_meta: true,
+  async tableInfo({ conid, database, schemaName, pureName }) {
+    const opened = await databaseConnections.ensureOpened(conid, database);
+    const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
+    const allForeignKeys = _.flatten(opened.structure.tables.map(x => x.foreignKeys));
+    return {
+      ...table,
+      dependencies: allForeignKeys.filter(x => x.refSchemaName == schemaName && x.refTableName == pureName),
+    };
+  },
+
+  sqlObjectInfo_meta: true,
+  async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
+    const opened = await databaseConnections.ensureOpened(conid, database);
+    const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);
+    return res;
+  },
+};