dbgate-api 5.5.5 → 5.5.7-alpha.16

package/.env

@@ -1,6 +1,8 @@
 DEVMODE=1
 SHELL_SCRIPTING=1
 
+CLOUD_UPGRADE_FILE=c:\test\upg\upgrade.zip
+
 # PERMISSIONS=~widgets/app,~widgets/plugins
 # DISABLE_SHELL=1
 # HIDE_APP_EDITOR=1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.5.5",
+  "version": "5.5.7-alpha.16",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -27,10 +27,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.5.5",
+    "dbgate-datalib": "^5.5.7-alpha.16",
     "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.5.5",
-    "dbgate-tools": "^5.5.5",
+    "dbgate-sqltree": "^5.5.7-alpha.16",
+    "dbgate-tools": "^5.5.7-alpha.16",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -56,6 +56,7 @@
     "pinomin": "^1.0.4",
     "portfinder": "^1.0.28",
     "rimraf": "^3.0.0",
+    "semver": "^7.6.3",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
     "stream-json": "^1.8.0",
@@ -77,7 +78,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.5.5",
+    "dbgate-types": "^5.5.7-alpha.16",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/auth.js

@@ -22,7 +22,8 @@ function unauthorizedResponse(req, res, text) {
   // if (req.path == getExpressPath('/connections/list')) {
   //   return res.json([]);
   // }
-  return res.sendStatus(401).send(text);
+
+  return res.status(401).send(text);
 }
 
 function authMiddleware(req, res, next) {
@@ -35,8 +36,9 @@ function authMiddleware(req, res, next) {
     '/auth/login',
     '/auth/redirect',
     '/stream',
-    'storage/get-connections-for-login-page',
-    'auth/get-providers',
+    '/storage/get-connections-for-login-page',
+    '/storage/set-admin-password',
+    '/auth/get-providers',
     '/connections/dblogin-web',
     '/connections/dblogin-app',
     '/connections/dblogin-auth',
@@ -68,6 +70,7 @@ function authMiddleware(req, res, next) {
     return next();
   } catch (err) {
     if (skipAuth) {
+      req.isInvalidToken = true;
       return next();
     }
 
@@ -88,7 +91,12 @@ module.exports = {
     const { amoid, login, password, isAdminPage } = params;
 
     if (isAdminPage) {
-      if (process.env.ADMIN_PASSWORD && process.env.ADMIN_PASSWORD == password) {
+      let adminPassword = process.env.ADMIN_PASSWORD;
+      if (!adminPassword) {
+        const adminConfig = await storage.readConfig({ group: 'admin' });
+        adminPassword = adminConfig?.adminPassword;
+      }
+      if (adminPassword && adminPassword == password) {
         return {
           accessToken: jwt.sign(
             {

package/src/controllers/config.js

@@ -17,6 +17,7 @@ const { checkLicense, checkLicenseKey } = require('../utility/checkLicense');
 const storage = require('./storage');
 const { getAuthProxyUrl } = require('../utility/authProxy');
 const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
+const { extractErrorMessage } = require('dbgate-tools');
 
 const lock = new AsyncLock();
 
@@ -39,10 +40,12 @@ module.exports = {
     const isUserLoggedIn = authProvider.isUserLoggedIn(req);
 
     const singleConid = authProvider.getSingleConnectionId(req);
+    const storageConnectionError = storage.getStorageConnectionError();
 
-    const singleConnection = singleConid
-      ? await connections.getCore({ conid: singleConid })
-      : connections.singleConnection;
+    const singleConnection =
+      singleConid && !storageConnectionError
+        ? await connections.getCore({ conid: singleConid })
+        : connections.singleConnection;
 
     let configurationError = null;
     if (process.env.STORAGE_DATABASE && process.env.BASIC_AUTH) {
@@ -50,8 +53,21 @@ module.exports = {
         'Basic authentization is not allowed, when using storage. Cannot use both STORAGE_DATABASE and BASIC_AUTH';
     }
 
-    const checkedLicense = await checkLicense();
+    if (storageConnectionError && !configurationError) {
+      configurationError = extractErrorMessage(storageConnectionError);
+    }
+
+    const checkedLicense = storageConnectionError ? null : await checkLicense();
     const isLicenseValid = checkedLicense?.status == 'ok';
+    const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl();
+    const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' });
+
+    const isAdminPasswordMissing = !!(
+      process.env.STORAGE_DATABASE &&
+      !process.env.ADMIN_PASSWORD &&
+      !process.env.BASIC_AUTH &&
+      !adminConfig?.adminPasswordState
+    );
 
     return {
       runAsPortal: !!connections.portalConnections,
@@ -68,17 +84,19 @@ module.exports = {
       trialDaysLeft: checkedLicense?.isGeneratedTrial && !checkedLicense?.isExpired ? checkedLicense?.daysLeft : null,
       checkedLicense,
       configurationError,
-      logoutUrl: await authProvider.getLogoutUrl(),
+      logoutUrl,
       permissions,
       login,
       // ...additionalConfigProps,
       isBasicAuth: !!process.env.BASIC_AUTH,
       isAdminLoginForm: !!(
         process.env.STORAGE_DATABASE &&
-        process.env.ADMIN_PASSWORD &&
-        !process.env.BASIC_AUTH &&
-        checkedLicense?.type == 'premium'
+        (process.env.ADMIN_PASSWORD || adminConfig?.adminPasswordState == 'set') &&
+        !process.env.BASIC_AUTH
       ),
+      isAdminPasswordMissing,
+      isInvalidToken: req.isInvalidToken,
+      adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
       connectionsFilePath: path.join(datadir(), 'connections.jsonl'),

package/src/controllers/storage.js

@@ -17,4 +17,13 @@ module.exports = {
   async getConnectionsForLoginPage() {
     return null;
   },
+
+  getStorageConnectionError() {
+    return null;
+  },
+
+  readConfig_meta: true,
+  async readConfig({ group }) {
+    return {};
+  },
 };

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.5.5',
-  buildTime: '2024-10-11T08:23:34.200Z'
+  version: '5.5.7-alpha.16',
+  buildTime: '2024-11-01T11:41:22.229Z'
 };

package/src/index.js

@@ -4,11 +4,13 @@ const fs = require('fs');
 const moment = require('moment');
 const path = require('path');
 const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const currentVersion = require('./currentVersion');
 
 const logger = getLogger('apiIndex');
 
 process.on('uncaughtException', err => {
-  logger.fatal(extractErrorLogData(err), 'Uncaught exception');
+  logger.fatal(extractErrorLogData(err), 'Uncaught exception, exiting process');
+  process.exit(1);
 });
 
 if (processArgs.startProcess) {
@@ -99,10 +101,17 @@ function configureLogger() {
 
 if (processArgs.listenApi) {
   configureLogger();
+  logger.info(`Starting API process version ${currentVersion.version}`);
+
+  if (process.env.DEBUG_PRINT_ENV_VARIABLES) {
+    logger.info('Debug print environment variables:');
+    for (const key of Object.keys(process.env)) {
+      logger.info(`  ${key}: ${JSON.stringify(process.env[key])}`);
+    }
+  }
 }
 
 const shell = require('./shell/index');
-const currentVersion = require('./currentVersion');
 
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),

package/src/main.js

@@ -35,6 +35,7 @@ const getExpressPath = require('./utility/getExpressPath');
 const _ = require('lodash');
 const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
+const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 
 const logger = getLogger('main');
 
@@ -73,6 +74,8 @@ function start() {
   if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
   } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
   } else if (process.env.DEVWEB) {
@@ -126,6 +129,10 @@ function start() {
     const port = process.env.PORT || 3000;
     logger.info(`DbGate API listening on port ${port} (docker build)`);
     server.listen(port);
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (AWS AMI build)`);
+    server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
       port: parseInt(
@@ -162,6 +169,10 @@ function start() {
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);
   process.on('SIGBREAK', shutdown);
+
+  if (process.env.CLOUD_UPGRADE_FILE) {
+    startCloudUpgradeTimer();
+  }
 }
 
 function useAllControllers(app, electron) {

package/src/proc/connectProcess.js

@@ -20,9 +20,10 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection, 'app');
-      const res = await driver.getVersion(conn);
+      const dbhan = await connectUtility(driver, connection, 'app');
+      const res = await driver.getVersion(dbhan);
       process.send({ msgtype: 'connected', ...res });
+      await driver.close(dbhan);
     } catch (e) {
       console.error(e);
       process.send({

package/src/proc/databaseConnectionProcess.js

@@ -329,8 +329,9 @@ async function handleSqlPreview({ msgid, objects, options }) {
     await generator.dump();
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
-      setTimeout(() => {
+      setTimeout(async () => {
         logger.error('Exiting because of unhandled exception');
+        await driver.close(dbhan);
         process.exit(0);
       }, 500);
     }
@@ -406,10 +407,12 @@ async function handleMessage({ msgtype, ...other }) {
 function start() {
   childProcessChecker();
 
-  setInterval(() => {
+  setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
       logger.info('Database connection not alive, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
       process.exit(0);
     }
   }, 10 * 1000);

package/src/proc/serverConnectionProcess.js

@@ -1,12 +1,12 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger, extractErrorLogData } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const logger = getLogger('srvconnProcess');
 
-let systemConnection;
+let dbhan;
 let storedConnection;
 let lastDatabases = null;
 let lastStatus = null;
@@ -16,7 +16,7 @@ let afterConnectCallbacks = [];
 async function handleRefresh() {
   const driver = requireEngineDriver(storedConnection);
   try {
-    let databases = await driver.listDatabases(systemConnection);
+    let databases = await driver.listDatabases(dbhan);
     if (storedConnection?.allowedDatabases?.trim()) {
       const allowedDatabaseList = storedConnection.allowedDatabases
         .split('\n')
@@ -39,14 +39,14 @@ async function handleRefresh() {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error refreshing server databases');
     setTimeout(() => process.exit(1), 1000);
   }
 }
 
 async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
-  const version = await driver.getVersion(systemConnection);
+  const version = await driver.getVersion(dbhan);
   process.send({ msgtype: 'version', version });
 }
 
@@ -70,7 +70,7 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection, 'app');
+    dbhan = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
@@ -84,7 +84,7 @@ async function handleConnect(connection) {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error connecting to server');
     setTimeout(() => process.exit(1), 1000);
   }
 
@@ -95,7 +95,7 @@ async function handleConnect(connection) {
 }
 
 function waitConnected() {
-  if (systemConnection) return Promise.resolve();
+  if (dbhan) return Promise.resolve();
   return new Promise((resolve, reject) => {
     afterConnectCallbacks.push([resolve, reject]);
   });
@@ -108,14 +108,14 @@ function handlePing() {
 async function handleDatabaseOp(op, { msgid, name }) {
   try {
     const driver = requireEngineDriver(storedConnection);
-    systemConnection = await connectUtility(driver, storedConnection, 'app');
+    dbhan = await connectUtility(driver, storedConnection, 'app');
     if (driver[op]) {
-      await driver[op](systemConnection, name);
+      await driver[op](dbhan, name);
     } else {
       const dmp = driver.createDumper();
       dmp[op](name);
       logger.info({ sql: dmp.s }, 'Running script');
-      await driver.query(systemConnection, dmp.s, { discardResult: true });
+      await driver.query(dbhan, dmp.s, { discardResult: true });
     }
     await handleRefresh();
 
@@ -137,11 +137,11 @@ async function handleDriverDataCore(msgid, callMethod) {
 }
 
 async function handleServerSummary({ msgid }) {
-  return handleDriverDataCore(msgid, driver => driver.serverSummary(systemConnection));
+  return handleDriverDataCore(msgid, driver => driver.serverSummary(dbhan));
 }
 
 async function handleSummaryCommand({ msgid, command, row }) {
-  return handleDriverDataCore(msgid, driver => driver.summaryCommand(systemConnection, command, row));
+  return handleDriverDataCore(msgid, driver => driver.summaryCommand(dbhan, command, row));
 }
 
 const messageHandlers = {
@@ -161,10 +161,12 @@ async function handleMessage({ msgtype, ...other }) {
 function start() {
   childProcessChecker();
 
-  setInterval(() => {
+  setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
       logger.info('Server connection not alive, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
       process.exit(0);
     }
   }, 10 * 1000);
@@ -178,6 +180,7 @@ function start() {
         name: 'error',
         message: err.message,
       });
+      logger.error(extractErrorLogData(err), `Error processing message ${message?.['msgtype']}`);
     }
   });
 }

package/src/proc/sessionProcess.js

@@ -14,7 +14,7 @@ const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require
 
 const logger = getLogger('sessionProcess');
 
-let systemConnection;
+let dbhan;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
@@ -177,7 +177,7 @@ function handleStream(driver, resultIndexHolder, sqlItem) {
   return new Promise((resolve, reject) => {
     const start = sqlItem.trimStart || sqlItem.start;
     const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line);
-    driver.stream(systemConnection, sqlItem.text, handler);
+    driver.stream(dbhan, sqlItem.text, handler);
   });
 }
 
@@ -196,7 +196,7 @@ async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection, 'app');
+  dbhan = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -210,7 +210,7 @@ async function handleConnect(connection) {
 // }
 
 function waitConnected() {
-  if (systemConnection) return Promise.resolve();
+  if (dbhan) return Promise.resolve();
   return new Promise((resolve, reject) => {
     afterConnectCallbacks.push([resolve, reject]);
   });
@@ -230,7 +230,7 @@ async function handleStartProfiler({ jslid }) {
   const writer = new TableWriter();
   writer.initializeFromReader(jslid);
 
-  currentProfiler = await driver.startProfiler(systemConnection, {
+  currentProfiler = await driver.startProfiler(dbhan, {
     row: data => writer.rowFromReader(data),
   });
   currentProfiler.writer = writer;
@@ -241,7 +241,7 @@ async function handleStopProfiler({ jslid }) {
 
   const driver = requireEngineDriver(storedConnection);
   currentProfiler.writer.close();
-  driver.stopProfiler(systemConnection, currentProfiler);
+  driver.stopProfiler(dbhan, currentProfiler);
   currentProfiler = null;
 }
 
@@ -304,7 +304,7 @@ async function handleExecuteReader({ jslid, sql, fileName }) {
   const writer = new TableWriter();
   writer.initializeFromReader(jslid);
 
-  const reader = await driver.readQuery(systemConnection, sql);
+  const reader = await driver.readQuery(dbhan, sql);
 
   reader.on('data', data => {
     writer.rowFromReader(data);
@@ -340,10 +340,12 @@ function start() {
 
   lastPing = new Date().getTime();
 
-  setInterval(() => {
+  setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 25 * 1000) {
       logger.info('Session not alive, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
       process.exit(0);
     }
 
@@ -362,6 +364,8 @@ function start() {
       executingScripts == 0
     ) {
       logger.info('Session not active, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
       process.exit(0);
     }
   }, 10 * 1000);

package/src/proc/sshForwardProcess.js

@@ -3,7 +3,7 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, extractErrorMessage } = require('dbgate-tools');
 
 const logger = getLogger('sshProcess');
 
@@ -46,7 +46,7 @@ async function handleStart({ connection, tunnelConfig }) {
       msgtype: 'error',
       connection,
       tunnelConfig,
-      errorMessage: err.message,
+      errorMessage: extractErrorMessage(err.message),
     });
   }
 }

package/src/shell/autoIndexForeignKeysTransform.js

@@ -0,0 +1,19 @@
+const autoIndexForeignKeysTransform = () => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        indexes: [
+          ...(table.indexes || []),
+          ...table.foreignKeys.map(fk => ({
+            constraintName: `IX_${fk.constraintName}`,
+            columns: fk.columns,
+          })),
+        ],
+      };
+    }),
+  };
+};
+
+module.exports = autoIndexForeignKeysTransform;

package/src/shell/dataTypeMapperTransform.js

@@ -0,0 +1,21 @@
+const dataTypeMapperTransform = (oldType, newType) => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        columns: table.columns.map(column => {
+          if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
+            return {
+              ...column,
+              dataType: newType,
+            };
+          }
+          return column;
+        }),
+      };
+    }),
+  };
+};
+
+module.exports = dataTypeMapperTransform;

package/src/shell/deployDb.js

@@ -1,7 +1,16 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
 
-async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+async function deployDb({
+  connection,
+  systemConnection,
+  driver,
+  analysedStructure,
+  modelFolder,
+  loadedDbModel,
+  modelTransforms,
+  dbdiffOptionsExtra,
+}) {
   const { sql } = await generateDeploySql({
     connection,
     systemConnection,
@@ -9,9 +18,11 @@ async function deployDb({ connection, systemConnection, driver, analysedStructur
     analysedStructure,
     modelFolder,
     loadedDbModel,
+    modelTransforms,
+    dbdiffOptionsExtra,
   });
   // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql });
+  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
 }
 
 module.exports = deployDb;

package/src/shell/executeQuery.js

@@ -1,11 +1,19 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');
 
-async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  logger.info({ sql }, `Execute query`);
+async function executeQuery({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  sql,
+  logScriptItems = false,
+}) {
+  if (!logScriptItems) {
+    logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
+  }
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
@@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   try {
     logger.info(`Connected.`);
 
-    await driver.script(dbhan, sql);
+    await driver.script(dbhan, sql, { logScriptItems });
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

package/src/shell/generateDeploySql.js

@@ -18,6 +18,8 @@ async function generateDeploySql({
   analysedStructure = undefined,
   modelFolder = undefined,
   loadedDbModel = undefined,
+  modelTransforms = undefined,
+  dbdiffOptionsExtra = {},
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -28,9 +30,15 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
-    const deployedModel = generateDbPairingId(
-      extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
-    );
+    let deployedModelSource = loadedDbModel
+      ? databaseInfoFromYamlModel(loadedDbModel)
+      : await importDbModel(modelFolder);
+
+    for (const transform of modelTransforms || []) {
+      deployedModelSource = transform(deployedModelSource);
+    }
+
+    const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {
       ...modelCompareDbDiffOptions,
@@ -41,6 +49,8 @@ async function generateDeploySql({
       noDropSqlObject: true,
       noRenameTable: true,
       noRenameColumn: true,
+
+      ...dbdiffOptionsExtra,
     };
     const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
     const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
@@ -57,7 +67,7 @@ async function generateDeploySql({
       deployedModel,
       driver
     );
-    
+
     return res;
   } finally {
     if (!systemConnection) {

package/src/shell/index.js

@@ -30,6 +30,10 @@ const dataDuplicator = require('./dataDuplicator');
 const dbModelToJson = require('./dbModelToJson');
 const jsonToDbModel = require('./jsonToDbModel');
 const jsonReader = require('./jsonReader');
+const dataTypeMapperTransform = require('./dataTypeMapperTransform');
+const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
+const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
+const generateDeploySql = require('./generateDeploySql');
 
 const dbgateApi = {
   queryReader,
@@ -63,6 +67,10 @@ const dbgateApi = {
   dataDuplicator,
   dbModelToJson,
   jsonToDbModel,
+  dataTypeMapperTransform,
+  sqlTextReplacementTransform,
+  autoIndexForeignKeysTransform,
+  generateDeploySql,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/sqlTextReplacementTransform.js

@@ -0,0 +1,32 @@
+function replaceInText(text, replacements) {
+  let result = text;
+  for (const key of Object.keys(replacements)) {
+    result = result.split(key).join(replacements[key]);
+  }
+  return result;
+}
+
+function replaceInCollection(collection, replacements) {
+  if (!collection) return collection;
+  return collection.map(item => {
+    if (item.createSql) {
+      return {
+        ...item,
+        createSql: replaceInText(item.createSql, replacements),
+      };
+    }
+    return item;
+  });
+}
+
+const sqlTextReplacementTransform = replacements => database => {
+  return {
+    ...database,
+    views: replaceInCollection(database.views, replacements),
+    matviews: replaceInCollection(database.matviews, replacements),
+    procedures: replaceInCollection(database.procedures, replacements),
+    functions: replaceInCollection(database.functions, replacements),
+  };
+};
+
+module.exports = sqlTextReplacementTransform;

package/src/utility/cloudUpgrade.js

@@ -0,0 +1,61 @@
+const axios = require('axios');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const semver = require('semver');
+const currentVersion = require('../currentVersion');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+
+const logger = getLogger('cloudUpgrade');
+
+async function checkCloudUpgrade() {
+  try {
+    const resp = await axios.default.get('https://api.github.com/repos/dbgate/dbgate/releases/latest');
+    const json = resp.data;
+    const version = json.name.substring(1);
+    let cloudDownloadedVersion = null;
+    try {
+      cloudDownloadedVersion = await fsp.readFile(process.env.CLOUD_UPGRADE_FILE + '.version', 'utf-8');
+    } catch (err) {
+      cloudDownloadedVersion = null;
+    }
+    if (
+      semver.gt(version, currentVersion.version) &&
+      (!cloudDownloadedVersion || semver.gt(version, cloudDownloadedVersion))
+    ) {
+      logger.info(`New version available: ${version}`);
+      const zipUrl = json.assets.find(x => x.name == 'cloud-build.zip').browser_download_url;
+
+      const writer = fs.createWriteStream(process.env.CLOUD_UPGRADE_FILE);
+
+      const response = await axios.default({
+        url: zipUrl,
+        method: 'GET',
+        responseType: 'stream',
+      });
+
+      response.data.pipe(writer);
+
+      await new Promise((resolve, reject) => {
+        writer.on('finish', resolve);
+        writer.on('error', reject);
+      });
+      await fsp.writeFile(process.env.CLOUD_UPGRADE_FILE + '.version', version);
+
+      logger.info(`Downloaded new version from ${zipUrl}`);
+    } else {
+      logger.info(`Checked version ${version} is not newer than ${cloudDownloadedVersion ?? currentVersion.version}, upgrade skippped`);
+    }
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');
+  }
+}
+
+function startCloudUpgradeTimer() {
+  // at first in 5 seconds
+  setTimeout(checkCloudUpgrade, 5000);
+
+  // hourly
+  setInterval(checkCloudUpgrade, 60 * 60 * 1000);
+}
+
+module.exports = startCloudUpgradeTimer;

package/src/utility/directories.js

@@ -77,6 +77,9 @@ function packagedPluginsDir() {
   if (platformInfo.isDocker) {
     return '/home/dbgate-docker/plugins';
   }
+  if (platformInfo.isAwsUbuntuLayout) {
+    return '/home/ubuntu/build/plugins';
+  }
   if (platformInfo.isNpmDist) {
     // node_modules
     return global['PLUGINS_DIR'];

package/src/utility/hardwareFingerprint.js

@@ -72,6 +72,7 @@ async function getPublicHardwareFingerprint() {
       country: fingerprint.country,
       region: fingerprint.region,
       isDocker: platformInfo.isDocker,
+      isAwsUbuntuLayout: platformInfo.isAwsUbuntuLayout,
       isElectron: platformInfo.isElectron,
     },
   };

package/src/utility/platformInfo.js

@@ -14,6 +14,7 @@ const isBuiltWebMode = process.env.BUILTWEBMODE == '1';
 const isNpmDist = !!global['IS_NPM_DIST'];
 const isDbModel = !!global['IS_DB_MODEL'];
 const isForkedApi = processArgs.isForkedApi;
+const isAwsUbuntuLayout = fs.existsSync('/home/ubuntu/build/public');
 
 // function moduleAvailable(name) {
 //   try {
@@ -47,6 +48,7 @@ const platformInfo = {
     (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron() || !!isDbModel,
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
+  isAwsUbuntuLayout,
 };
 
 module.exports = platformInfo;

package/src/utility/sshTunnel.js

@@ -50,7 +50,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
         resolve(subprocess);
       }
       if (msgtype == 'error') {
-        reject(errorMessage);
+        reject(new Error(errorMessage));
       }
     });
     subprocess.on('exit', code => {
@@ -91,6 +91,7 @@ async function getSshTunnel(connection) {
       };
       return sshTunnelCache[tunnelCacheKey];
     } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error creating SSH tunnel:');
       // error is not cached
       return {
         state: 'error',