dbgate-api 5.5.6 → 5.5.7-alpha.25

package/.env

@@ -1,6 +1,8 @@
 DEVMODE=1
 SHELL_SCRIPTING=1
 
+CLOUD_UPGRADE_FILE=c:\test\upg\upgrade.zip
+
 # PERMISSIONS=~widgets/app,~widgets/plugins
 # DISABLE_SHELL=1
 # HIDE_APP_EDITOR=1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.5.6",
+  "version": "5.5.7-alpha.25",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -27,10 +27,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.5.6",
+    "dbgate-datalib": "^5.5.7-alpha.25",
     "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.5.6",
-    "dbgate-tools": "^5.5.6",
+    "dbgate-sqltree": "^5.5.7-alpha.25",
+    "dbgate-tools": "^5.5.7-alpha.25",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -56,6 +56,7 @@
     "pinomin": "^1.0.4",
     "portfinder": "^1.0.28",
     "rimraf": "^3.0.0",
+    "semver": "^7.6.3",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
     "stream-json": "^1.8.0",
@@ -77,7 +78,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.5.6",
+    "dbgate-types": "^5.5.7-alpha.25",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/auth.js

@@ -36,8 +36,9 @@ function authMiddleware(req, res, next) {
     '/auth/login',
     '/auth/redirect',
     '/stream',
-    'storage/get-connections-for-login-page',
-    'auth/get-providers',
+    '/storage/get-connections-for-login-page',
+    '/storage/set-admin-password',
+    '/auth/get-providers',
     '/connections/dblogin-web',
     '/connections/dblogin-app',
     '/connections/dblogin-auth',
@@ -69,6 +70,7 @@ function authMiddleware(req, res, next) {
     return next();
   } catch (err) {
     if (skipAuth) {
+      req.isInvalidToken = true;
       return next();
     }
 
@@ -89,7 +91,12 @@ module.exports = {
     const { amoid, login, password, isAdminPage } = params;
 
     if (isAdminPage) {
-      if (process.env.ADMIN_PASSWORD && process.env.ADMIN_PASSWORD == password) {
+      let adminPassword = process.env.ADMIN_PASSWORD;
+      if (!adminPassword) {
+        const adminConfig = await storage.readConfig({ group: 'admin' });
+        adminPassword = adminConfig?.adminPassword;
+      }
+      if (adminPassword && adminPassword == password) {
         return {
           accessToken: jwt.sign(
             {

package/src/controllers/config.js

@@ -60,8 +60,16 @@ module.exports = {
     const checkedLicense = storageConnectionError ? null : await checkLicense();
     const isLicenseValid = checkedLicense?.status == 'ok';
     const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl();
+    const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' });
 
-    return {
+    const isAdminPasswordMissing = !!(
+      process.env.STORAGE_DATABASE &&
+      !process.env.ADMIN_PASSWORD &&
+      !process.env.BASIC_AUTH &&
+      !adminConfig?.adminPasswordState
+    );
+
+    const configResult = {
       runAsPortal: !!connections.portalConnections,
       singleDbConnection: connections.singleDbConnection,
       singleConnection: singleConnection,
@@ -83,15 +91,19 @@ module.exports = {
       isBasicAuth: !!process.env.BASIC_AUTH,
       isAdminLoginForm: !!(
         process.env.STORAGE_DATABASE &&
-        process.env.ADMIN_PASSWORD &&
-        !process.env.BASIC_AUTH &&
-        checkedLicense?.type == 'premium'
+        (process.env.ADMIN_PASSWORD || adminConfig?.adminPasswordState == 'set') &&
+        !process.env.BASIC_AUTH
       ),
+      isAdminPasswordMissing,
+      isInvalidToken: req?.isInvalidToken,
+      adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
       connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
+
+    return configResult;
   },
 
   logout_meta: {

package/src/controllers/connections.js

@@ -368,6 +368,11 @@ module.exports = {
 
   get_meta: true,
   async get({ conid }, req) {
+    if (conid == '__model') {
+      return {
+        _id: '__model',
+      };
+    }
     testConnectionPermission(conid, req);
     return this.getCore({ conid, mask: true });
   },

package/src/controllers/databaseConnections.js

@@ -13,6 +13,7 @@ const {
   modelCompareDbDiffOptions,
   getLogger,
   extractErrorLogData,
+  filterStructureBySchema,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -31,6 +32,8 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const crypto = require('crypto');
+const loadModelTransform = require('../utility/loadModelTransform');
+const exportDbModelSql = require('../utility/exportDbModelSql');
 
 const logger = getLogger('databaseConnections');
 
@@ -349,6 +352,11 @@ module.exports = {
 
   syncModel_meta: true,
   async syncModel({ conid, database, isFullRefresh }, req) {
+    if (conid == '__model') {
+      socket.emitChanged('database-structure-changed', { conid, database });
+      return { status: 'ok' };
+    }
+
     testConnectionPermission(conid, req);
     const conn = await this.ensureOpened(conid, database);
     conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
@@ -392,11 +400,12 @@ module.exports = {
   },
 
   structure_meta: true,
-  async structure({ conid, database }, req) {
+  async structure({ conid, database, modelTransFile = null }, req) {
     testConnectionPermission(conid, req);
     if (conid == '__model') {
       const model = await importDbModel(database);
-      return model;
+      const trans = await loadModelTransform(modelTransFile);
+      return trans ? trans(model) : model;
     }
 
     const opened = await this.ensureOpened(conid, database);
@@ -432,14 +441,35 @@ module.exports = {
   },
 
   exportModel_meta: true,
-  async exportModel({ conid, database }, req) {
+  async exportModel({ conid, database, outputFolder, schema }, req) {
+    testConnectionPermission(conid, req);
+
+    const realFolder = outputFolder.startsWith('archive:')
+      ? resolveArchiveFolder(outputFolder.substring('archive:'.length))
+      : outputFolder;
+
+    const model = await this.structure({ conid, database });
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModel(extendDatabaseInfo(filteredModel), realFolder);
+
+    if (outputFolder.startsWith('archive:')) {
+      socket.emitChanged(`archive-files-changed`, { folder: outputFolder.substring('archive:'.length) });
+    }
+    return { status: 'ok' };
+  },
+
+  exportModelSql_meta: true,
+  async exportModelSql({ conid, database, outputFolder, outputFile, schema }, req) {
     testConnectionPermission(conid, req);
-    const archiveFolder = await archive.getNewArchiveFolder({ database });
-    await fs.mkdir(path.join(archivedir(), archiveFolder));
+
+    const connection = await connections.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+
     const model = await this.structure({ conid, database });
-    await exportDbModel(model, path.join(archivedir(), archiveFolder));
-    socket.emitChanged(`archive-folders-changed`);
-    return { archiveFolder };
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModelSql(extendDatabaseInfo(filteredModel), driver, outputFolder, outputFile);
+
+    return { status: 'ok' };
   },
 
   generateDeploySql_meta: true,

package/src/controllers/runners.js

@@ -12,6 +12,7 @@ const {
   jsonScriptToJavascript,
   getLogger,
   safeJsonParse,
+  pinoLogRecordToMessageRecord,
 } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
@@ -68,18 +69,20 @@ module.exports = {
 
   dispatchMessage(runid, message) {
     if (message) {
-      const json = safeJsonParse(message.message);
-
-      if (json) logger.log(json);
-      else logger.info(message.message);
-
-      const toEmit = {
-        time: new Date(),
-        ...message,
-        message: json ? json.msg : message.message,
-      };
-
-      if (json && json.level >= 50) {
+      if (_.isPlainObject(message)) logger.log(message);
+      else logger.info(message);
+
+      const toEmit = _.isPlainObject(message)
+        ? {
+            time: new Date(),
+            ...message,
+          }
+        : {
+            message,
+            time: new Date(),
+          };
+
+      if (toEmit.level >= 50) {
         toEmit.severity = 'error';
       }
 
@@ -131,7 +134,16 @@ module.exports = {
       }
     );
     const pipeDispatcher = severity => data => {
-      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+      const json = safeJsonParse(data, null);
+
+      if (json) {
+        return this.dispatchMessage(runid, pinoLogRecordToMessageRecord(json));
+      } else {
+        return this.dispatchMessage(runid, {
+          message: json == null ? data.toString().trim() : null,
+          severity,
+        });
+      }
     };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
@@ -165,7 +177,7 @@ module.exports = {
 
   start_meta: true,
   async start({ script }) {
-    const runid = crypto.randomUUID()
+    const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
       const js = jsonScriptToJavascript(script);

package/src/controllers/serverConnections.js

@@ -134,6 +134,7 @@ module.exports = {
   listDatabases_meta: true,
   async listDatabases({ conid }, req) {
     if (!conid) return [];
+    if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
@@ -172,7 +173,7 @@ module.exports = {
         }
       })
     );
-    socket.setStreamIdFilter(strmid, { conid: conidArray });
+    socket.setStreamIdFilter(strmid, { conid: [...(conidArray ?? []), '__model'] });
     return { status: 'ok' };
   },
 

package/src/controllers/storage.js

@@ -20,5 +20,10 @@ module.exports = {
 
   getStorageConnectionError() {
     return null;
-  }
+  },
+
+  readConfig_meta: true,
+  async readConfig({ group }) {
+    return {};
+  },
 };

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.5.6',
-  buildTime: '2024-10-17T11:26:28.099Z'
+  version: '5.5.7-alpha.25',
+  buildTime: '2024-11-12T07:05:33.638Z'
 };

package/src/main.js

@@ -35,6 +35,7 @@ const getExpressPath = require('./utility/getExpressPath');
 const _ = require('lodash');
 const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
+const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 
 const logger = getLogger('main');
 
@@ -73,6 +74,8 @@ function start() {
   if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
   } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
   } else if (process.env.DEVWEB) {
@@ -126,6 +129,10 @@ function start() {
     const port = process.env.PORT || 3000;
     logger.info(`DbGate API listening on port ${port} (docker build)`);
     server.listen(port);
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (AWS AMI build)`);
+    server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
       port: parseInt(
@@ -162,6 +169,10 @@ function start() {
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);
   process.on('SIGBREAK', shutdown);
+
+  if (process.env.CLOUD_UPGRADE_FILE) {
+    startCloudUpgradeTimer();
+  }
 }
 
 function useAllControllers(app, electron) {

package/src/proc/serverConnectionProcess.js

@@ -1,5 +1,5 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger, extractErrorLogData } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
@@ -39,7 +39,7 @@ async function handleRefresh() {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error refreshing server databases');
     setTimeout(() => process.exit(1), 1000);
   }
 }
@@ -84,7 +84,7 @@ async function handleConnect(connection) {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error connecting to server');
     setTimeout(() => process.exit(1), 1000);
   }
 
@@ -164,7 +164,6 @@ function start() {
   setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      
       logger.info('Server connection not alive, exiting');
       const driver = requireEngineDriver(storedConnection);
       await driver.close(dbhan);
@@ -181,6 +180,7 @@ function start() {
         name: 'error',
         message: err.message,
       });
+      logger.error(extractErrorLogData(err), `Error processing message ${message?.['msgtype']}`);
     }
   });
 }

package/src/proc/sshForwardProcess.js

@@ -3,7 +3,7 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, extractErrorMessage } = require('dbgate-tools');
 
 const logger = getLogger('sshProcess');
 
@@ -46,7 +46,7 @@ async function handleStart({ connection, tunnelConfig }) {
       msgtype: 'error',
       connection,
       tunnelConfig,
-      errorMessage: err.message,
+      errorMessage: extractErrorMessage(err.message),
     });
   }
 }

package/src/shell/autoIndexForeignKeysTransform.js

@@ -0,0 +1,19 @@
+const autoIndexForeignKeysTransform = () => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        indexes: [
+          ...(table.indexes || []),
+          ...table.foreignKeys.map(fk => ({
+            constraintName: `IX_${fk.constraintName}`,
+            columns: fk.columns.map(x => ({ columnName: x.columnName })),
+          })),
+        ],
+      };
+    }),
+  };
+};
+
+module.exports = autoIndexForeignKeysTransform;

package/src/shell/dataTypeMapperTransform.js

@@ -0,0 +1,21 @@
+const dataTypeMapperTransform = (oldType, newType) => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        columns: table.columns.map(column => {
+          if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
+            return {
+              ...column,
+              dataType: newType,
+            };
+          }
+          return column;
+        }),
+      };
+    }),
+  };
+};
+
+module.exports = dataTypeMapperTransform;

package/src/shell/deployDb.js

@@ -1,17 +1,54 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
+const { ScriptDrivedDeployer } = require('dbgate-datalib');
+const connectUtility = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const loadModelFolder = require('../utility/loadModelFolder');
 
-async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
-  const { sql } = await generateDeploySql({
-    connection,
-    systemConnection,
-    driver,
-    analysedStructure,
-    modelFolder,
-    loadedDbModel,
-  });
-  // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql });
+async function deployDb({
+  connection,
+  systemConnection,
+  driver,
+  analysedStructure,
+  modelFolder,
+  loadedDbModel,
+  modelTransforms,
+  dbdiffOptionsExtra,
+  ignoreNameRegex = '',
+  targetSchema = null,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    const scriptDeployer = new ScriptDrivedDeployer(
+      dbhan,
+      driver,
+      loadedDbModel ?? (await loadModelFolder(modelFolder))
+    );
+    await scriptDeployer.runPre();
+
+    const { sql } = await generateDeploySql({
+      connection,
+      systemConnection: dbhan,
+      driver,
+      analysedStructure,
+      modelFolder,
+      loadedDbModel,
+      modelTransforms,
+      dbdiffOptionsExtra,
+      ignoreNameRegex,
+      targetSchema,
+    });
+    // console.log('RUNNING DEPLOY SCRIPT:', sql);
+    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+    await scriptDeployer.runPost();
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
 }
 
 module.exports = deployDb;

package/src/shell/dropAllDbObjects.js

@@ -0,0 +1,42 @@
+const executeQuery = require('./executeQuery');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
+
+const logger = getLogger('dropAllDbObjects');
+
+async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(dbhan);
+  }
+
+  analysedStructure = extendDatabaseInfo(analysedStructure);
+
+  const dmp = driver.createDumper();
+
+  for (const table of analysedStructure.tables) {
+    for (const fk of table.foreignKeys) {
+      dmp.dropForeignKey(fk);
+    }
+  }
+  for (const table of analysedStructure.tables) {
+    dmp.dropTable(table);
+  }
+  for (const field of Object.keys(analysedStructure)) {
+    if (dmp.getSqlObjectSqlName(field)) {
+      for (const obj of analysedStructure[field]) {
+        dmp.dropSqlObject(obj);
+      }
+    }
+  }
+
+  await executeQuery({ connection, systemConnection, driver, sql: dmp.s, logScriptItems: true });
+}
+
+module.exports = dropAllDbObjects;

package/src/shell/executeQuery.js

@@ -1,11 +1,19 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');
 
-async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  logger.info({ sql }, `Execute query`);
+async function executeQuery({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  sql,
+  logScriptItems = false,
+}) {
+  if (!logScriptItems) {
+    logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
+  }
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
@@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   try {
     logger.info(`Connected.`);
 
-    await driver.script(dbhan, sql);
+    await driver.script(dbhan, sql, { logScriptItems });
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

package/src/shell/generateDeploySql.js

@@ -6,6 +6,10 @@ const {
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
   enrichWithPreloadedRows,
+  skipNamesInStructureByRegex,
+  replaceSchemaInStructure,
+  filterStructureBySchema,
+  skipDbGateInternalObjects,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -18,6 +22,10 @@ async function generateDeploySql({
   analysedStructure = undefined,
   modelFolder = undefined,
   loadedDbModel = undefined,
+  modelTransforms = undefined,
+  dbdiffOptionsExtra = {},
+  ignoreNameRegex = '',
+  targetSchema = null,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -28,9 +36,25 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
-    const deployedModel = generateDbPairingId(
-      extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
-    );
+    if (ignoreNameRegex) {
+      analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
+    }
+    analysedStructure = skipDbGateInternalObjects(analysedStructure);
+
+    let deployedModelSource = loadedDbModel
+      ? databaseInfoFromYamlModel(loadedDbModel)
+      : await importDbModel(modelFolder);
+
+    for (const transform of modelTransforms || []) {
+      deployedModelSource = transform(deployedModelSource);
+    }
+
+    if (targetSchema) {
+      deployedModelSource = replaceSchemaInStructure(deployedModelSource, targetSchema);
+      analysedStructure = filterStructureBySchema(analysedStructure, targetSchema);
+    }
+
+    const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {
       ...modelCompareDbDiffOptions,
@@ -41,6 +65,8 @@ async function generateDeploySql({
       noDropSqlObject: true,
       noRenameTable: true,
       noRenameColumn: true,
+
+      ...dbdiffOptionsExtra,
     };
     const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
     const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
@@ -57,7 +83,7 @@ async function generateDeploySql({
       deployedModel,
       driver
     );
-    
+
     return res;
   } finally {
     if (!systemConnection) {

package/src/shell/index.js

@@ -30,6 +30,12 @@ const dataDuplicator = require('./dataDuplicator');
 const dbModelToJson = require('./dbModelToJson');
 const jsonToDbModel = require('./jsonToDbModel');
 const jsonReader = require('./jsonReader');
+const dataTypeMapperTransform = require('./dataTypeMapperTransform');
+const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
+const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
+const generateDeploySql = require('./generateDeploySql');
+const dropAllDbObjects = require('./dropAllDbObjects');
+const scriptDrivedDeploy = require('./scriptDrivedDeploy');
 
 const dbgateApi = {
   queryReader,
@@ -63,6 +69,12 @@ const dbgateApi = {
   dataDuplicator,
   dbModelToJson,
   jsonToDbModel,
+  dataTypeMapperTransform,
+  sqlTextReplacementTransform,
+  autoIndexForeignKeysTransform,
+  generateDeploySql,
+  dropAllDbObjects,
+  scriptDrivedDeploy,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/sqlTextReplacementTransform.js

@@ -0,0 +1,32 @@
+function replaceInText(text, replacements) {
+  let result = text;
+  for (const key of Object.keys(replacements)) {
+    result = result.split(key).join(replacements[key]);
+  }
+  return result;
+}
+
+function replaceInCollection(collection, replacements) {
+  if (!collection) return collection;
+  return collection.map(item => {
+    if (item.createSql) {
+      return {
+        ...item,
+        createSql: replaceInText(item.createSql, replacements),
+      };
+    }
+    return item;
+  });
+}
+
+const sqlTextReplacementTransform = replacements => database => {
+  return {
+    ...database,
+    views: replaceInCollection(database.views, replacements),
+    matviews: replaceInCollection(database.matviews, replacements),
+    procedures: replaceInCollection(database.procedures, replacements),
+    functions: replaceInCollection(database.functions, replacements),
+  };
+};
+
+module.exports = sqlTextReplacementTransform;

package/src/utility/cloudUpgrade.js

@@ -0,0 +1,61 @@
+const axios = require('axios');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const semver = require('semver');
+const currentVersion = require('../currentVersion');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+
+const logger = getLogger('cloudUpgrade');
+
+async function checkCloudUpgrade() {
+  try {
+    const resp = await axios.default.get('https://api.github.com/repos/dbgate/dbgate/releases/latest');
+    const json = resp.data;
+    const version = json.name.substring(1);
+    let cloudDownloadedVersion = null;
+    try {
+      cloudDownloadedVersion = await fsp.readFile(process.env.CLOUD_UPGRADE_FILE + '.version', 'utf-8');
+    } catch (err) {
+      cloudDownloadedVersion = null;
+    }
+    if (
+      semver.gt(version, currentVersion.version) &&
+      (!cloudDownloadedVersion || semver.gt(version, cloudDownloadedVersion))
+    ) {
+      logger.info(`New version available: ${version}`);
+      const zipUrl = json.assets.find(x => x.name == 'cloud-build.zip').browser_download_url;
+
+      const writer = fs.createWriteStream(process.env.CLOUD_UPGRADE_FILE);
+
+      const response = await axios.default({
+        url: zipUrl,
+        method: 'GET',
+        responseType: 'stream',
+      });
+
+      response.data.pipe(writer);
+
+      await new Promise((resolve, reject) => {
+        writer.on('finish', resolve);
+        writer.on('error', reject);
+      });
+      await fsp.writeFile(process.env.CLOUD_UPGRADE_FILE + '.version', version);
+
+      logger.info(`Downloaded new version from ${zipUrl}`);
+    } else {
+      logger.info(`Checked version ${version} is not newer than ${cloudDownloadedVersion ?? currentVersion.version}, upgrade skippped`);
+    }
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');
+  }
+}
+
+function startCloudUpgradeTimer() {
+  // at first in 5 seconds
+  setTimeout(checkCloudUpgrade, 5000);
+
+  // hourly
+  setInterval(checkCloudUpgrade, 60 * 60 * 1000);
+}
+
+module.exports = startCloudUpgradeTimer;

package/src/utility/directories.js

@@ -77,6 +77,9 @@ function packagedPluginsDir() {
   if (platformInfo.isDocker) {
     return '/home/dbgate-docker/plugins';
   }
+  if (platformInfo.isAwsUbuntuLayout) {
+    return '/home/ubuntu/build/plugins';
+  }
   if (platformInfo.isNpmDist) {
     // node_modules
     return global['PLUGINS_DIR'];

package/src/utility/exportDbModelSql.js

@@ -0,0 +1,80 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { getSchemasUsedByStructure } = require('dbgate-tools');
+
+async function exportDbModelSql(dbModel, driver, outputDir, outputFile) {
+  const { tables, views, procedures, functions, triggers, matviews } = dbModel;
+
+  const usedSchemas = getSchemasUsedByStructure(dbModel);
+  const useSchemaDir = usedSchemas.length > 1;
+
+  const createdDirs = new Set();
+  async function ensureDir(dir) {
+    if (!createdDirs.has(dir)) {
+      await fs.mkdir(dir, { recursive: true });
+      createdDirs.add(dir);
+    }
+  }
+
+  async function writeLists(writeList) {
+    await writeList(views, 'views');
+    await writeList(procedures, 'procedures');
+    await writeList(functions, 'functions');
+    await writeList(triggers, 'triggers');
+    await writeList(matviews, 'matviews');
+  }
+
+  if (outputFile) {
+    const dmp = driver.createDumper();
+    for (const table of tables || []) {
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+    }
+    for (const table of tables || []) {
+      for (const fk of table.foreignKeys || []) {
+        dmp.createForeignKey(fk);
+      }
+    }
+    writeLists((list, folder) => {
+      for (const obj of list || []) {
+        dmp.createSqlObject(obj);
+      }
+    });
+
+    const script = dmp.s;
+    await fs.writeFile(outputFile, script);
+  }
+
+  if (outputDir) {
+    for (const table of tables || []) {
+      const tablesDir = useSchemaDir
+        ? path.join(outputDir, table.schemaName ?? 'default', 'tables')
+        : path.join(outputDir, 'tables');
+      await ensureDir(tablesDir);
+      const dmp = driver.createDumper();
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+      await fs.writeFile(path.join(tablesDir, `${table.pureName}.sql`), dmp.s);
+    }
+
+    await writeLists(async (list, folder) => {
+      for (const obj of list || []) {
+        const objdir = useSchemaDir
+          ? path.join(outputDir, obj.schemaName ?? 'default', folder)
+          : path.join(outputDir, folder);
+        await ensureDir(objdir);
+        const dmp = driver.createDumper();
+        dmp.createSqlObject(obj);
+        await fs.writeFile(path.join(objdir, `${obj.pureName}.sql`), dmp.s);
+      }
+    });
+  }
+}
+
+module.exports = exportDbModelSql;

package/src/utility/hardwareFingerprint.js

@@ -72,6 +72,7 @@ async function getPublicHardwareFingerprint() {
       country: fingerprint.country,
       region: fingerprint.region,
       isDocker: platformInfo.isDocker,
+      isAwsUbuntuLayout: platformInfo.isAwsUbuntuLayout,
       isElectron: platformInfo.isElectron,
     },
   };

package/src/utility/importDbModel.js

@@ -1,28 +1,8 @@
-const fs = require('fs-extra');
-const path = require('path');
-const yaml = require('js-yaml');
 const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools');
-const { startsWith } = require('lodash');
-const { archivedir, resolveArchiveFolder } = require('./directories');
-const loadFilesRecursive = require('./loadFilesRecursive');
+const loadModelFolder = require('./loadModelFolder');
 
 async function importDbModel(inputDir) {
-  const files = [];
-
-  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
-
-  for (const name of await loadFilesRecursive(dir)) {
-    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
-      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
-
-      files.push({
-        name: path.parse(name).base,
-        text,
-        json: name.endsWith('.yaml') ? yaml.load(text) : null,
-      });
-    }
-  }
-
+  const files = await loadModelFolder(inputDir);
   return databaseInfoFromYamlModel(files);
 }
 

package/src/utility/loadModelFolder.js

@@ -0,0 +1,27 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { resolveArchiveFolder } = require('./directories');
+const loadFilesRecursive = require('./loadFilesRecursive');
+
+async function loadModelFolder(inputDir) {
+  const files = [];
+
+  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
+
+  for (const name of await loadFilesRecursive(dir)) {
+    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
+      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
+
+      files.push({
+        name: path.parse(name).base,
+        text,
+        json: name.endsWith('.yaml') ? yaml.load(text) : null,
+      });
+    }
+  }
+
+  return files;
+}
+
+module.exports = loadModelFolder;

package/src/utility/loadModelTransform.js

@@ -0,0 +1,36 @@
+const { filesdir } = require('./directories');
+const path = require('path');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const dbgateApi = require('../shell');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('loadModelTransform');
+
+function modelTransformFromJson(json) {
+  if (!dbgateApi[json.transform]) return null;
+  const creator = dbgateApi[json.transform];
+  return creator(...json.arguments);
+}
+
+async function loadModelTransform(file) {
+  if (!file) return null;
+  try {
+    const dir = filesdir();
+    const fullPath = path.join(dir, 'modtrans', file);
+    const text = await fs.readFile(fullPath, { encoding: 'utf-8' });
+    const json = JSON.parse(text);
+    if (_.isArray(json)) {
+      const array = _.compact(json.map(x => modelTransformFromJson(x)));
+      return array.length ? structure => array.reduce((acc, val) => val(acc), structure) : null;
+    }
+    if (_.isPlainObject(json)) {
+      return modelTransformFromJson(json);
+    }
+    return null;
+  } catch (err) {
+    logger.error(extractErrorLogData(err), `Error loading model transform ${file}`);
+    return null;
+  }
+}
+
+module.exports = loadModelTransform;

package/src/utility/platformInfo.js

@@ -14,6 +14,7 @@ const isBuiltWebMode = process.env.BUILTWEBMODE == '1';
 const isNpmDist = !!global['IS_NPM_DIST'];
 const isDbModel = !!global['IS_DB_MODEL'];
 const isForkedApi = processArgs.isForkedApi;
+const isAwsUbuntuLayout = fs.existsSync('/home/ubuntu/build/public');
 
 // function moduleAvailable(name) {
 //   try {
@@ -42,11 +43,20 @@ const platformInfo = {
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
   allowShellConnection:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_CONNECTION || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_CONNECTION ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowShellScripting:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_SCRIPTING ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
+  isAwsUbuntuLayout,
 };
 
 module.exports = platformInfo;

package/src/utility/sshTunnel.js

@@ -50,7 +50,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
         resolve(subprocess);
       }
       if (msgtype == 'error') {
-        reject(errorMessage);
+        reject(new Error(errorMessage));
       }
     });
     subprocess.on('exit', code => {
@@ -91,6 +91,7 @@ async function getSshTunnel(connection) {
       };
       return sshTunnelCache[tunnelCacheKey];
     } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error creating SSH tunnel:');
       // error is not cached
       return {
         state: 'error',