dbgate-api 5.5.6 → 5.5.7-alpha.16

package/.env

@@ -1,6 +1,8 @@
 DEVMODE=1
 SHELL_SCRIPTING=1
 
+CLOUD_UPGRADE_FILE=c:\test\upg\upgrade.zip
+
 # PERMISSIONS=~widgets/app,~widgets/plugins
 # DISABLE_SHELL=1
 # HIDE_APP_EDITOR=1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.5.6",
+  "version": "5.5.7-alpha.16",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -27,10 +27,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.5.6",
+    "dbgate-datalib": "^5.5.7-alpha.16",
     "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.5.6",
-    "dbgate-tools": "^5.5.6",
+    "dbgate-sqltree": "^5.5.7-alpha.16",
+    "dbgate-tools": "^5.5.7-alpha.16",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -56,6 +56,7 @@
     "pinomin": "^1.0.4",
     "portfinder": "^1.0.28",
     "rimraf": "^3.0.0",
+    "semver": "^7.6.3",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
     "stream-json": "^1.8.0",
@@ -77,7 +78,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.5.6",
+    "dbgate-types": "^5.5.7-alpha.16",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/auth.js

@@ -36,8 +36,9 @@ function authMiddleware(req, res, next) {
     '/auth/login',
     '/auth/redirect',
     '/stream',
-    'storage/get-connections-for-login-page',
-    'auth/get-providers',
+    '/storage/get-connections-for-login-page',
+    '/storage/set-admin-password',
+    '/auth/get-providers',
     '/connections/dblogin-web',
     '/connections/dblogin-app',
     '/connections/dblogin-auth',
@@ -69,6 +70,7 @@ function authMiddleware(req, res, next) {
     return next();
   } catch (err) {
     if (skipAuth) {
+      req.isInvalidToken = true;
       return next();
     }
 
@@ -89,7 +91,12 @@ module.exports = {
     const { amoid, login, password, isAdminPage } = params;
 
     if (isAdminPage) {
-      if (process.env.ADMIN_PASSWORD && process.env.ADMIN_PASSWORD == password) {
+      let adminPassword = process.env.ADMIN_PASSWORD;
+      if (!adminPassword) {
+        const adminConfig = await storage.readConfig({ group: 'admin' });
+        adminPassword = adminConfig?.adminPassword;
+      }
+      if (adminPassword && adminPassword == password) {
         return {
           accessToken: jwt.sign(
             {

package/src/controllers/config.js

@@ -60,6 +60,14 @@ module.exports = {
     const checkedLicense = storageConnectionError ? null : await checkLicense();
     const isLicenseValid = checkedLicense?.status == 'ok';
     const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl();
+    const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' });
+
+    const isAdminPasswordMissing = !!(
+      process.env.STORAGE_DATABASE &&
+      !process.env.ADMIN_PASSWORD &&
+      !process.env.BASIC_AUTH &&
+      !adminConfig?.adminPasswordState
+    );
 
     return {
       runAsPortal: !!connections.portalConnections,
@@ -83,10 +91,12 @@ module.exports = {
       isBasicAuth: !!process.env.BASIC_AUTH,
       isAdminLoginForm: !!(
         process.env.STORAGE_DATABASE &&
-        process.env.ADMIN_PASSWORD &&
-        !process.env.BASIC_AUTH &&
-        checkedLicense?.type == 'premium'
+        (process.env.ADMIN_PASSWORD || adminConfig?.adminPasswordState == 'set') &&
+        !process.env.BASIC_AUTH
       ),
+      isAdminPasswordMissing,
+      isInvalidToken: req.isInvalidToken,
+      adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
       connectionsFilePath: path.join(datadir(), 'connections.jsonl'),

package/src/controllers/storage.js

@@ -20,5 +20,10 @@ module.exports = {
 
   getStorageConnectionError() {
     return null;
-  }
+  },
+
+  readConfig_meta: true,
+  async readConfig({ group }) {
+    return {};
+  },
 };

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.5.6',
-  buildTime: '2024-10-17T11:26:28.099Z'
+  version: '5.5.7-alpha.16',
+  buildTime: '2024-11-01T11:41:22.229Z'
 };

package/src/main.js

@@ -35,6 +35,7 @@ const getExpressPath = require('./utility/getExpressPath');
 const _ = require('lodash');
 const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
+const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 
 const logger = getLogger('main');
 
@@ -73,6 +74,8 @@ function start() {
   if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
   } else if (platformInfo.isNpmDist) {
     app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
   } else if (process.env.DEVWEB) {
@@ -126,6 +129,10 @@ function start() {
     const port = process.env.PORT || 3000;
     logger.info(`DbGate API listening on port ${port} (docker build)`);
     server.listen(port);
+  } else if (platformInfo.isAwsUbuntuLayout) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (AWS AMI build)`);
+    server.listen(port);
   } else if (platformInfo.isNpmDist) {
     getPort({
       port: parseInt(
@@ -162,6 +169,10 @@ function start() {
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);
   process.on('SIGBREAK', shutdown);
+
+  if (process.env.CLOUD_UPGRADE_FILE) {
+    startCloudUpgradeTimer();
+  }
 }
 
 function useAllControllers(app, electron) {

package/src/proc/serverConnectionProcess.js

@@ -1,5 +1,5 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger, extractErrorLogData } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
@@ -39,7 +39,7 @@ async function handleRefresh() {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error refreshing server databases');
     setTimeout(() => process.exit(1), 1000);
   }
 }
@@ -84,7 +84,7 @@ async function handleConnect(connection) {
       name: 'error',
       message: err.message,
     });
-    // console.error(err);
+    logger.error(extractErrorLogData(err), 'Error connecting to server');
     setTimeout(() => process.exit(1), 1000);
   }
 
@@ -164,7 +164,6 @@ function start() {
   setInterval(async () => {
     const time = new Date().getTime();
     if (time - lastPing > 40 * 1000) {
-      
       logger.info('Server connection not alive, exiting');
       const driver = requireEngineDriver(storedConnection);
       await driver.close(dbhan);
@@ -181,6 +180,7 @@ function start() {
         name: 'error',
         message: err.message,
       });
+      logger.error(extractErrorLogData(err), `Error processing message ${message?.['msgtype']}`);
     }
   });
 }

package/src/proc/sshForwardProcess.js

@@ -3,7 +3,7 @@ const platformInfo = require('../utility/platformInfo');
 const childProcessChecker = require('../utility/childProcessChecker');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SSHConnection } = require('../utility/SSHConnection');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { getLogger, extractErrorLogData, extractErrorMessage } = require('dbgate-tools');
 
 const logger = getLogger('sshProcess');
 
@@ -46,7 +46,7 @@ async function handleStart({ connection, tunnelConfig }) {
       msgtype: 'error',
       connection,
       tunnelConfig,
-      errorMessage: err.message,
+      errorMessage: extractErrorMessage(err.message),
     });
   }
 }

package/src/shell/autoIndexForeignKeysTransform.js

@@ -0,0 +1,19 @@
+const autoIndexForeignKeysTransform = () => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        indexes: [
+          ...(table.indexes || []),
+          ...table.foreignKeys.map(fk => ({
+            constraintName: `IX_${fk.constraintName}`,
+            columns: fk.columns,
+          })),
+        ],
+      };
+    }),
+  };
+};
+
+module.exports = autoIndexForeignKeysTransform;

package/src/shell/dataTypeMapperTransform.js

@@ -0,0 +1,21 @@
+const dataTypeMapperTransform = (oldType, newType) => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        columns: table.columns.map(column => {
+          if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
+            return {
+              ...column,
+              dataType: newType,
+            };
+          }
+          return column;
+        }),
+      };
+    }),
+  };
+};
+
+module.exports = dataTypeMapperTransform;

package/src/shell/deployDb.js

@@ -1,7 +1,16 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
 
-async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+async function deployDb({
+  connection,
+  systemConnection,
+  driver,
+  analysedStructure,
+  modelFolder,
+  loadedDbModel,
+  modelTransforms,
+  dbdiffOptionsExtra,
+}) {
   const { sql } = await generateDeploySql({
     connection,
     systemConnection,
@@ -9,9 +18,11 @@ async function deployDb({ connection, systemConnection, driver, analysedStructur
     analysedStructure,
     modelFolder,
     loadedDbModel,
+    modelTransforms,
+    dbdiffOptionsExtra,
   });
   // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql });
+  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
 }
 
 module.exports = deployDb;

package/src/shell/executeQuery.js

@@ -1,11 +1,19 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');
 
-async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  logger.info({ sql }, `Execute query`);
+async function executeQuery({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  sql,
+  logScriptItems = false,
+}) {
+  if (!logScriptItems) {
+    logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
+  }
 
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
@@ -13,7 +21,7 @@ async function executeQuery({ connection = undefined, systemConnection = undefin
   try {
     logger.info(`Connected.`);
 
-    await driver.script(dbhan, sql);
+    await driver.script(dbhan, sql, { logScriptItems });
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

package/src/shell/generateDeploySql.js

@@ -18,6 +18,8 @@ async function generateDeploySql({
   analysedStructure = undefined,
   modelFolder = undefined,
   loadedDbModel = undefined,
+  modelTransforms = undefined,
+  dbdiffOptionsExtra = {},
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -28,9 +30,15 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
-    const deployedModel = generateDbPairingId(
-      extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
-    );
+    let deployedModelSource = loadedDbModel
+      ? databaseInfoFromYamlModel(loadedDbModel)
+      : await importDbModel(modelFolder);
+
+    for (const transform of modelTransforms || []) {
+      deployedModelSource = transform(deployedModelSource);
+    }
+
+    const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {
       ...modelCompareDbDiffOptions,
@@ -41,6 +49,8 @@ async function generateDeploySql({
       noDropSqlObject: true,
       noRenameTable: true,
       noRenameColumn: true,
+
+      ...dbdiffOptionsExtra,
     };
     const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
     const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
@@ -57,7 +67,7 @@ async function generateDeploySql({
       deployedModel,
       driver
     );
-    
+
     return res;
   } finally {
     if (!systemConnection) {

package/src/shell/index.js

@@ -30,6 +30,10 @@ const dataDuplicator = require('./dataDuplicator');
 const dbModelToJson = require('./dbModelToJson');
 const jsonToDbModel = require('./jsonToDbModel');
 const jsonReader = require('./jsonReader');
+const dataTypeMapperTransform = require('./dataTypeMapperTransform');
+const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
+const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
+const generateDeploySql = require('./generateDeploySql');
 
 const dbgateApi = {
   queryReader,
@@ -63,6 +67,10 @@ const dbgateApi = {
   dataDuplicator,
   dbModelToJson,
   jsonToDbModel,
+  dataTypeMapperTransform,
+  sqlTextReplacementTransform,
+  autoIndexForeignKeysTransform,
+  generateDeploySql,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/shell/sqlTextReplacementTransform.js

@@ -0,0 +1,32 @@
+function replaceInText(text, replacements) {
+  let result = text;
+  for (const key of Object.keys(replacements)) {
+    result = result.split(key).join(replacements[key]);
+  }
+  return result;
+}
+
+function replaceInCollection(collection, replacements) {
+  if (!collection) return collection;
+  return collection.map(item => {
+    if (item.createSql) {
+      return {
+        ...item,
+        createSql: replaceInText(item.createSql, replacements),
+      };
+    }
+    return item;
+  });
+}
+
+const sqlTextReplacementTransform = replacements => database => {
+  return {
+    ...database,
+    views: replaceInCollection(database.views, replacements),
+    matviews: replaceInCollection(database.matviews, replacements),
+    procedures: replaceInCollection(database.procedures, replacements),
+    functions: replaceInCollection(database.functions, replacements),
+  };
+};
+
+module.exports = sqlTextReplacementTransform;

package/src/utility/cloudUpgrade.js

@@ -0,0 +1,61 @@
+const axios = require('axios');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const semver = require('semver');
+const currentVersion = require('../currentVersion');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+
+const logger = getLogger('cloudUpgrade');
+
+async function checkCloudUpgrade() {
+  try {
+    const resp = await axios.default.get('https://api.github.com/repos/dbgate/dbgate/releases/latest');
+    const json = resp.data;
+    const version = json.name.substring(1);
+    let cloudDownloadedVersion = null;
+    try {
+      cloudDownloadedVersion = await fsp.readFile(process.env.CLOUD_UPGRADE_FILE + '.version', 'utf-8');
+    } catch (err) {
+      cloudDownloadedVersion = null;
+    }
+    if (
+      semver.gt(version, currentVersion.version) &&
+      (!cloudDownloadedVersion || semver.gt(version, cloudDownloadedVersion))
+    ) {
+      logger.info(`New version available: ${version}`);
+      const zipUrl = json.assets.find(x => x.name == 'cloud-build.zip').browser_download_url;
+
+      const writer = fs.createWriteStream(process.env.CLOUD_UPGRADE_FILE);
+
+      const response = await axios.default({
+        url: zipUrl,
+        method: 'GET',
+        responseType: 'stream',
+      });
+
+      response.data.pipe(writer);
+
+      await new Promise((resolve, reject) => {
+        writer.on('finish', resolve);
+        writer.on('error', reject);
+      });
+      await fsp.writeFile(process.env.CLOUD_UPGRADE_FILE + '.version', version);
+
+      logger.info(`Downloaded new version from ${zipUrl}`);
+    } else {
+      logger.info(`Checked version ${version} is not newer than ${cloudDownloadedVersion ?? currentVersion.version}, upgrade skippped`);
+    }
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');
+  }
+}
+
+function startCloudUpgradeTimer() {
+  // at first in 5 seconds
+  setTimeout(checkCloudUpgrade, 5000);
+
+  // hourly
+  setInterval(checkCloudUpgrade, 60 * 60 * 1000);
+}
+
+module.exports = startCloudUpgradeTimer;

package/src/utility/directories.js

@@ -77,6 +77,9 @@ function packagedPluginsDir() {
   if (platformInfo.isDocker) {
     return '/home/dbgate-docker/plugins';
   }
+  if (platformInfo.isAwsUbuntuLayout) {
+    return '/home/ubuntu/build/plugins';
+  }
   if (platformInfo.isNpmDist) {
     // node_modules
     return global['PLUGINS_DIR'];

package/src/utility/hardwareFingerprint.js

@@ -72,6 +72,7 @@ async function getPublicHardwareFingerprint() {
       country: fingerprint.country,
       region: fingerprint.region,
       isDocker: platformInfo.isDocker,
+      isAwsUbuntuLayout: platformInfo.isAwsUbuntuLayout,
       isElectron: platformInfo.isElectron,
     },
   };

package/src/utility/platformInfo.js

@@ -14,6 +14,7 @@ const isBuiltWebMode = process.env.BUILTWEBMODE == '1';
 const isNpmDist = !!global['IS_NPM_DIST'];
 const isDbModel = !!global['IS_DB_MODEL'];
 const isForkedApi = processArgs.isForkedApi;
+const isAwsUbuntuLayout = fs.existsSync('/home/ubuntu/build/public');
 
 // function moduleAvailable(name) {
 //   try {
@@ -47,6 +48,7 @@ const platformInfo = {
     (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron() || !!isDbModel,
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
+  isAwsUbuntuLayout,
 };
 
 module.exports = platformInfo;

package/src/utility/sshTunnel.js

@@ -50,7 +50,7 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
         resolve(subprocess);
       }
       if (msgtype == 'error') {
-        reject(errorMessage);
+        reject(new Error(errorMessage));
       }
     });
     subprocess.on('exit', code => {
@@ -91,6 +91,7 @@ async function getSshTunnel(connection) {
       };
       return sshTunnelCache[tunnelCacheKey];
     } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error creating SSH tunnel:');
       // error is not cached
       return {
         state: 'error',