dbgate-api 4.4.1 → 4.4.5-alpha.1

package/src/controllers/plugins.js

@@ -12,7 +12,7 @@ const _ = require('lodash');
 const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
-  script_meta: 'get',
+  script_meta: true,
   async script({ packageName }) {
     const packagedContent = packagedPluginsContent();
 
@@ -30,7 +30,7 @@ module.exports = {
     return data;
   },
 
-  search_meta: 'get',
+  search_meta: true,
   async search({ filter }) {
     // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
     const resp = await axios.default.get(
@@ -40,7 +40,7 @@ module.exports = {
     return (objects || []).map(x => x.package);
   },
 
-  info_meta: 'get',
+  info_meta: true,
   async info({ packageName }) {
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
@@ -63,7 +63,7 @@ module.exports = {
     }
   },
 
-  installed_meta: 'get',
+  installed_meta: true,
   async installed() {
     const packagedContent = packagedPluginsContent();
 
@@ -107,7 +107,7 @@ module.exports = {
   //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
   // },
 
-  install_meta: 'post',
+  install_meta: true,
   async install({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -120,7 +120,7 @@ module.exports = {
     // await this.saveRemovePlugins();
   },
 
-  uninstall_meta: 'post',
+  uninstall_meta: true,
   async uninstall({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -130,7 +130,7 @@ module.exports = {
     await this.saveRemovePlugins();
   },
 
-  upgrade_meta: 'post',
+  upgrade_meta: true,
   async upgrade({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
@@ -143,13 +143,13 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
   },
 
-  command_meta: 'post',
+  command_meta: true,
   async command({ packageName, command, args }) {
     const content = requirePlugin(packageName);
     return content.commands[command](args);
   },
 
-  authTypes_meta: 'get',
+  authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
     const content = requirePlugin(packageName);

package/src/controllers/queryHistory.js

@@ -34,7 +34,7 @@ function readCore(reader, skip, limit, filter) {
 }
 
 module.exports = {
-  read_meta: 'get',
+  read_meta: true,
   async read({ skip, limit, filter }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     // @ts-ignore
@@ -44,7 +44,7 @@ module.exports = {
     return res;
   },
 
-  write_meta: 'post',
+  write_meta: true,
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');

package/src/controllers/runners.js

@@ -8,6 +8,7 @@ const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -98,15 +99,22 @@ module.exports = {
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
-      },
-    });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || global['dbgateApiModulePath'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
     const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
@@ -136,21 +144,21 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
-  start_meta: 'post',
+  start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
     return this.startCore(runid, scriptTemplate(script, false));
   },
 
-  getNodeScript_meta: 'post',
+  getNodeScript_meta: true,
   async getNodeScript({ script }) {
     return scriptTemplate(script, true);
   },
 
-  cancel_meta: 'post',
+  cancel_meta: true,
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
@@ -160,7 +168,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ runid }) {
     const directory = path.join(rundir(), runid);
     const files = await fs.readdir(directory);
@@ -176,7 +184,7 @@ module.exports = {
     return res;
   },
 
-  loadReader_meta: 'post',
+  loadReader_meta: true,
   async loadReader({ functionName, props }) {
     const promise = new Promise((resolve, reject) => {
       const runid = uuidv1();

package/src/controllers/serverConnections.js

@@ -6,6 +6,7 @@ const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
 const config = require('./config');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   opened: [],
@@ -37,10 +38,12 @@ module.exports = {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], [
+      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
         '--start-process',
         'serverConnectionProcess',
-        ...process.argv.slice(3),
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
       ]);
       const newOpened = {
         conid,
@@ -86,25 +89,25 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  disconnect_meta: true,
   async disconnect({ conid }) {
     await this.close(conid, true);
     return { status: 'ok' };
   },
 
-  listDatabases_meta: 'get',
+  listDatabases_meta: true,
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
-  version_meta: 'get',
+  version_meta: true,
   async version({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.version;
   },
 
-  serverStatus_meta: 'get',
+  serverStatus_meta: true,
   async serverStatus() {
     return {
       ...this.closed,
@@ -112,7 +115,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ connections }) {
     await Promise.all(
       _.uniq(connections).map(async conid => {
@@ -128,7 +131,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, keepOpen }) {
     if (!keepOpen) this.close(conid);
 
@@ -136,7 +139,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  createDatabase_meta: 'post',
+  createDatabase_meta: true,
   async createDatabase({ conid, name }) {
     const opened = await this.ensureOpened(conid);
     opened.subprocess.send({ msgtype: 'createDatabase', name });

package/src/controllers/sessions.js

@@ -5,6 +5,7 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -61,11 +62,17 @@ module.exports = {
 
   handle_ping() {},
 
-  create_meta: 'post',
+  create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'sessionProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
     const newOpened = {
       conid,
       database,
@@ -81,10 +88,10 @@ module.exports = {
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
-  executeQuery_meta: 'post',
+  executeQuery_meta: true,
   async executeQuery({ sesid, sql }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -98,7 +105,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // cancel_meta: 'post',
+  // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
   //   if (!session) {
@@ -108,7 +115,7 @@ module.exports = {
   //   return { state: 'ok' };
   // },
 
-  kill_meta: 'post',
+  kill_meta: true,
   async kill({ sesid }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -119,7 +126,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // runCommand_meta: 'post',
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.4.1',
-  buildTime: '2021-11-22T18:42:53.218Z'
+  version: '4.4.5-alpha.1',
+  buildTime: '2021-12-30T08:13:02.622Z'
 };

package/src/index.js

@@ -8,7 +8,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !processArgs.checkParent) {
+} else if (!processArgs.checkParent && !global['API_PACKAGE'] && !global['dbgateApiModulePath']) {
   const main = require('./main');
 
   main.start();

package/src/main.js

@@ -4,12 +4,8 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const io = require('socket.io');
-const fs = require('fs');
 const getPort = require('get-port');
-const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
-const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -31,10 +27,7 @@ const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
-const processArgs = require('./utility/processArgs');
-const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
 
-let authorization = null;
 let checkLocalhostOrigin = null;
 
 function start() {
@@ -43,7 +36,6 @@ function start() {
   const app = express();
 
   const server = http.createServer(app);
-  socket.set(io(server));
 
   if (process.env.LOGIN && process.env.PASSWORD) {
     app.use(
@@ -58,9 +50,6 @@ function start() {
   }
 
   app.use(function (req, res, next) {
-    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
-      return res.status(403).json({ error: 'Not authorized!' });
-    }
     if (checkLocalhostOrigin) {
       if (
         req.headers.origin &&
@@ -81,6 +70,20 @@ function start() {
   });
 
   app.use(cors());
+
+  app.get('/stream', async function (req, res) {
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.setSseResponse(res);
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
@@ -90,20 +93,7 @@ function start() {
     })
   );
 
-  useController(app, '/connections', connections);
-  useController(app, '/server-connections', serverConnections);
-  useController(app, '/database-connections', databaseConnections);
-  useController(app, '/metadata', metadata);
-  useController(app, '/sessions', sessions);
-  useController(app, '/runners', runners);
-  useController(app, '/jsldata', jsldata);
-  useController(app, '/config', config);
-  useController(app, '/archive', archive);
-  useController(app, '/uploads', uploads);
-  useController(app, '/plugins', plugins);
-  useController(app, '/files', files);
-  useController(app, '/scheduler', scheduler);
-  useController(app, '/query-history', queryHistory);
+  useAllControllers(app, null);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
@@ -122,19 +112,7 @@ function start() {
     }
   }
 
-  if (processArgs.dynport) {
-    childProcessChecker();
-
-    authorization = crypto.randomBytes(32).toString('hex');
-
-    getPort().then(port => {
-      checkLocalhostOrigin = `localhost:${port}`;
-      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port, authorization });
-      });
-    });
-  } else if (platformInfo.isNpmDist) {
+  if (platformInfo.isNpmDist) {
     app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
@@ -142,8 +120,47 @@ function start() {
       });
     });
   } else {
-    server.listen(3000);
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port', port);
+    server.listen(port);
+  }
+
+  function shutdown() {
+    console.log('\nShutting down DbGate API server');
+    server.close(() => {
+      console.log('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      console.log('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
   }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
+}
+
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+}
+
+function initializeElectronSender(electronSender) {
+  socket.setElectronSender(electronSender);
 }
 
-module.exports = { start };
+module.exports = { start, useAllControllers, initializeElectronSender };

package/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

package/src/nativeModulesContent.js

@@ -2,6 +2,6 @@
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
 const content = {};
 
-content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');
+content['better-sqlite3'] = () => require('better-sqlite3');
 
 module.exports = content;

package/src/proc/databaseConnectionProcess.js

@@ -6,10 +6,12 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
+const generateDeploySql = require('../shell/generateDeploySql');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
+let afterAnalyseCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
@@ -42,14 +44,18 @@ async function handleFullRefresh() {
   process.send({ msgtype: 'structure', structure: analysedStructure });
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
 async function handleIncrementalRefresh(forceSend) {
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('checkStructure');
-  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure, serverVersion));
+  const newStructure = await checkedAsyncCall(
+    driver.analyseIncremental(systemConnection, analysedStructure, serverVersion)
+  );
   analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
@@ -62,6 +68,7 @@ async function handleIncrementalRefresh(forceSend) {
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
 function handleSyncModel() {
@@ -123,6 +130,20 @@ function waitConnected() {
   });
 }
 
+function waitStructure() {
+  if (analysedStructure) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterAnalyseCallbacks.push([resolve, reject]);
+  });
+}
+
+function resolveAnalysedPromises() {
+  for (const [resolve] of afterAnalyseCallbacks) {
+    resolve();
+  }
+  afterAnalyseCallbacks = [];
+}
+
 async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -168,7 +189,7 @@ async function handleUpdateCollection({ msgid, changeSet }) {
 }
 
 async function handleSqlPreview({ msgid, objects, options }) {
-  await waitConnected();
+  await waitStructure();
   const driver = requireEngineDriver(storedConnection);
 
   try {
@@ -188,6 +209,22 @@ async function handleSqlPreview({ msgid, objects, options }) {
   }
 }
 
+async function handleGenerateDeploySql({ msgid, modelFolder }) {
+  await waitStructure();
+
+  try {
+    const res = await generateDeploySql({
+      systemConnection,
+      connection: storedConnection,
+      analysedStructure,
+      modelFolder,
+    });
+    process.send({ ...res, msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -208,6 +245,7 @@ const messageHandlers = {
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
+  generateDeploySql: handleGenerateDeploySql,
   // runCommand: handleRunCommand,
 };
 
@@ -232,6 +270,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (e) {
+      console.error('Error in DB connection', e);
       process.send({ msgtype: 'error', error: e.message });
     }
   });

package/src/shell/generateDeploySql.js

@@ -5,6 +5,7 @@ const {
   databaseInfoFromYamlModel,
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -19,8 +20,9 @@ async function generateDeploySql({
   loadedDbModel = undefined,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
+
+  const pool = systemConnection || (await connectUtility(driver, connection));
   if (!analysedStructure) {
-    const pool = systemConnection || (await connectUtility(driver, connection));
     analysedStructure = await driver.analyseFull(pool);
   }
 
@@ -39,10 +41,20 @@ async function generateDeploySql({
     noRenameColumn: true,
   };
   const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+  const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, pool, driver);
+
+  // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
   // console.log('deployedModel', deployedModel.tables[0]);
   // console.log('currentModel', currentModel.tables[0]);
   // console.log('currentModelPaired', currentModelPaired.tables[0]);
-  const res = getAlterDatabaseScript(currentModelPaired, deployedModel, opts, deployedModel, driver);
+  const res = getAlterDatabaseScript(
+    currentModelPairedPreloaded,
+    deployedModel,
+    opts,
+    currentModelPairedPreloaded,
+    deployedModel,
+    driver
+  );
   return res;
 }