dbgate-api-premium 5.5.7-alpha.45

package/src/controllers/plugins.js

@@ -0,0 +1,216 @@
+const fs = require('fs-extra');
+const axios = require('axios');
+const path = require('path');
+const { extractPackageName } = require('dbgate-tools');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const compareVersions = require('compare-versions');
+const requirePlugin = require('../shell/requirePlugin');
+const downloadPackage = require('../utility/downloadPackage');
+const { hasPermission } = require('../utility/hasPermission');
+const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
+
+module.exports = {
+  script_meta: true,
+  async script({ packageName }) {
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
+    const data = await fs.readFile(file, {
+      encoding: 'utf-8',
+    });
+    return data;
+  },
+
+  search_meta: true,
+  async search({ filter }) {
+    // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
+    const resp = await axios.default.get(
+      `http://registry.npmjs.com/-/v1/search?text=${encodeURIComponent(filter)}+keywords:dbgateplugin&size=25&from=0`
+    );
+    const { objects } = resp.data || {};
+    return (objects || []).map(x => x.package);
+  },
+
+  info_meta: true,
+  async info({ packageName }) {
+    // @ts-ignore
+    const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
+
+    try {
+      const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
+      const { latest } = infoResp.data['dist-tags'];
+      const manifest = infoResp.data.versions[latest];
+      const { readme } = infoResp.data;
+
+      return {
+        readme,
+        manifest,
+        isPackaged,
+      };
+    } catch (err) {
+      return {
+        isPackaged,
+        state: 'error',
+        error: err.message,
+      };
+    }
+  },
+
+  installed_meta: true,
+  async installed() {
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
+    const res = [];
+    for (const packageName of _.union(files1, files2)) {
+      if (packageName == 'dist') continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          if (!manifest.keywords) {
+            continue;
+          }
+          if (!manifest.keywords.includes('dbgateplugin')) {
+            continue;
+          }
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
+      }
+    }
+    return res;
+  },
+
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },
+
+  install_meta: true,
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
+    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
+    if (!(await fs.exists(dir))) {
+      await downloadPackage(packageName, dir);
+    }
+    socket.emitChanged(`installed-plugins-changed`);
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
+    return true;
+  },
+
+  uninstall_meta: true,
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
+    const dir = path.join(pluginsdir(), packageName);
+    await fs.rmdir(dir, { recursive: true });
+    socket.emitChanged(`installed-plugins-changed`);
+    // this.removedPlugins.push(packageName);
+    // await this.saveRemovePlugins();
+    return true;
+  },
+
+  upgrade_meta: true,
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
+    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
+    if (await fs.exists(dir)) {
+      await fs.rmdir(dir, { recursive: true });
+      await downloadPackage(packageName, dir);
+    }
+
+    socket.emitChanged(`installed-plugins-changed`);
+    return true;
+  },
+
+  command_meta: true,
+  async command({ packageName, command, args }) {
+    const content = requirePlugin(packageName);
+    return content.commands[command](args);
+  },
+
+  authTypes_meta: true,
+  async authTypes({ engine }) {
+    const packageName = extractPackageName(engine);
+    if (!packageName) return null;
+    const content = requirePlugin(packageName);
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
+  },
+
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }
+
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }
+
+  //       continue;
+  //     }
+
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }
+
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
+};

package/src/controllers/queryHistory.js

@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: true,
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: true,
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emit('query-history-changed');
+    return 'OK';
+  },
+};

package/src/controllers/runners.js

@@ -0,0 +1,234 @@
+const crypto = require('crypto');
+const _ = require('lodash');
+const path = require('path');
+const fs = require('fs-extra');
+const byline = require('byline');
+const socket = require('../utility/socket');
+const { fork } = require('child_process');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+  pinoLogRecordToMessageRecord,
+} = require('dbgate-tools');
+const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');
+
+function extractPlugins(script) {
+  const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
+  const matches = [...script.matchAll(requireRegex)];
+  return matches.map(x => x[1]);
+}
+
+const requirePluginsTemplate = (plugins, isExport) =>
+  plugins
+    .map(
+      packageName =>
+        `const ${_.camelCase(packageName)} = require(${
+          isExport ? `'${packageName}'` : `process.env.PLUGIN_${_.camelCase(packageName)}`
+        });\n`
+    )
+    .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
+
+const scriptTemplate = (script, isExport) => `
+const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
+dbgateApi.initializeApiEnvironment();
+${requirePluginsTemplate(extractPlugins(script), isExport)}
+require=null;
+async function run() {
+${script}
+await dbgateApi.finalizer.run();
+logger.info('Finished job script');
+}
+dbgateApi.runScript(run);
+`;
+
+const loaderScriptTemplate = (functionName, props, runid) => `
+const dbgateApi = require(process.env.DBGATE_API);
+dbgateApi.initializeApiEnvironment();
+${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
+require=null;
+async function run() {
+const reader=await ${extractShellApiFunctionName(functionName)}(${JSON.stringify(props)});
+const writer=await dbgateApi.collectorWriter({runid: '${runid}'});
+await dbgateApi.copyStream(reader, writer);
+}
+dbgateApi.runScript(run);
+`;
+
+module.exports = {
+  /** @type {import('dbgate-types').OpenedRunner[]} */
+  opened: [],
+  requests: {},
+
+  dispatchMessage(runid, message) {
+    if (message) {
+      if (_.isPlainObject(message)) logger.log(message);
+      else logger.info(message);
+
+      const toEmit = _.isPlainObject(message)
+        ? {
+            time: new Date(),
+            ...message,
+          }
+        : {
+            message,
+            time: new Date(),
+          };
+
+      if (toEmit.level >= 50) {
+        toEmit.severity = 'error';
+      }
+
+      socket.emit(`runner-info-${runid}`, toEmit);
+    }
+  },
+
+  handle_ping() {},
+
+  handle_freeData(runid, { freeData }) {
+    const [resolve, reject] = this.requests[runid];
+    resolve(freeData);
+    delete this.requests[runid];
+  },
+
+  rejectRequest(runid, error) {
+    if (this.requests[runid]) {
+      const [resolve, reject] = this.requests[runid];
+      reject(error);
+      delete this.requests[runid];
+    }
+  },
+
+  startCore(runid, scriptText) {
+    const directory = path.join(rundir(), runid);
+    const scriptFile = path.join(uploadsdir(), runid + '.js');
+    fs.writeFileSync(`${scriptFile}`, scriptText);
+    fs.mkdirSync(directory);
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
+    logger.info({ scriptFile }, 'Running script');
+    // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        '--is-forked-api',
+        '--process-display-name',
+        'script',
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
+    const pipeDispatcher = severity => data => {
+      const json = safeJsonParse(data, null);
+
+      if (json) {
+        return this.dispatchMessage(runid, pinoLogRecordToMessageRecord(json));
+      } else {
+        return this.dispatchMessage(runid, {
+          message: json == null ? data.toString().trim() : null,
+          severity,
+        });
+      }
+    };
+
+    byline(subprocess.stdout).on('data', pipeDispatcher('info'));
+    byline(subprocess.stderr).on('data', pipeDispatcher('error'));
+    subprocess.on('exit', code => {
+      this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
+      socket.emit(`runner-done-${runid}`, code);
+    });
+    subprocess.on('error', error => {
+      this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
+      console.error('... ERROR subprocess', error);
+      this.dispatchMessage({
+        severity: 'error',
+        message: error.toString(),
+      });
+    });
+    const newOpened = {
+      runid,
+      subprocess,
+    };
+    this.opened.push(newOpened);
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
+      this[`handle_${msgtype}`](runid, message);
+    });
+    return _.pick(newOpened, ['runid']);
+  },
+
+  start_meta: true,
+  async start({ script }) {
+    const runid = crypto.randomUUID();
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
+    return this.startCore(runid, scriptTemplate(script, false));
+  },
+
+  getNodeScript_meta: true,
+  async getNodeScript({ script }) {
+    return scriptTemplate(script, true);
+  },
+
+  cancel_meta: true,
+  async cancel({ runid }) {
+    const runner = this.opened.find(x => x.runid == runid);
+    if (!runner) {
+      throw new Error('Invalid runner');
+    }
+    runner.subprocess.kill();
+    return { state: 'ok' };
+  },
+
+  files_meta: true,
+  async files({ runid }) {
+    const directory = path.join(rundir(), runid);
+    const files = await fs.readdir(directory);
+    const res = [];
+    for (const file of files) {
+      const stat = await fs.stat(path.join(directory, file));
+      res.push({
+        name: file,
+        size: stat.size,
+        path: path.join(directory, file),
+      });
+    }
+    return res;
+  },
+
+  loadReader_meta: true,
+  async loadReader({ functionName, props }) {
+    const promise = new Promise((resolve, reject) => {
+      const runid = crypto.randomUUID();
+      this.requests[runid] = [resolve, reject];
+      this.startCore(runid, loaderScriptTemplate(functionName, props, runid));
+    });
+    return promise;
+  },
+};

package/src/controllers/scheduler.js

@@ -0,0 +1,46 @@
+const { filesdir } = require('../utility/directories');
+const fs = require('fs-extra');
+const path = require('path');
+const cron = require('node-cron');
+const runners = require('./runners');
+const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');
+
+const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
+
+module.exports = {
+  tasks: [],
+
+  async unload() {
+    this.tasks.forEach(x => x.destroy());
+    this.tasks = [];
+  },
+
+  async processFile(file) {
+    const text = await fs.readFile(file, { encoding: 'utf-8' });
+    const match = text.match(scheduleRegex);
+    if (!match) return;
+    const pattern = match[1];
+    if (!cron.validate(pattern)) return;
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
+    const task = cron.schedule(pattern, () => runners.start({ script: text }));
+    this.tasks.push(task);
+  },
+
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
+    const shellDir = path.join(filesdir(), 'shell');
+    await this.unload();
+    if (!(await fs.exists(shellDir))) return;
+    const files = await fs.readdir(shellDir);
+    for (const file of files) {
+      await this.processFile(path.join(shellDir, file));
+    }
+  },
+
+  async _init() {
+    this.reload();
+  },
+};