dbgate-api-premium 6.4.3-alpha.1 → 6.5.1

package/src/shell/deployDb.js

@@ -20,6 +20,7 @@ const crypto = require('crypto');
  * @param {string} options.ignoreNameRegex - regex for ignoring objects by name
  * @param {string} options.targetSchema - target schema for deployment
  * @param {number} options.maxMissingTablesRatio - maximum ratio of missing tables in database. Safety check, if missing ratio is highe, deploy is stopped (preventing accidental drop of all tables)
+ * @param {boolean} options.useTransaction - run deploy in transaction. If not provided, it will be set to true if driver supports transactions
  */
 async function deployDb({
   connection,
@@ -33,6 +34,7 @@ async function deployDb({
   ignoreNameRegex = '',
   targetSchema = null,
   maxMissingTablesRatio = undefined,
+  useTransaction,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
@@ -60,7 +62,14 @@ async function deployDb({
       maxMissingTablesRatio,
     });
     // console.log('RUNNING DEPLOY SCRIPT:', sql);
-    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+    await executeQuery({
+      connection,
+      systemConnection: dbhan,
+      driver,
+      sql,
+      logScriptItems: true,
+      useTransaction,
+    });
 
     await scriptDeployer.runPost();
   } finally {

package/src/shell/executeQuery.js

@@ -14,6 +14,7 @@ const logger = getLogger('execQuery');
  * @param {string} [options.sql] - SQL query
  * @param {string} [options.sqlFile] - SQL file
  * @param {boolean} [options.logScriptItems] - whether to log script items instead of whole script
+ * @param {boolean} [options.useTransaction] - run query in transaction
  * @param {boolean} [options.skipLogging] - whether to skip logging
  */
 async function executeQuery({
@@ -24,6 +25,7 @@ async function executeQuery({
   sqlFile = undefined,
   logScriptItems = false,
   skipLogging = false,
+  useTransaction,
 }) {
   if (!logScriptItems && !skipLogging) {
     logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
@@ -42,7 +44,7 @@ async function executeQuery({
       logger.debug(`Running SQL query, length: ${sql.length}`);
     }
 
-    await driver.script(dbhan, sql, { logScriptItems });
+    await driver.script(dbhan, sql, { logScriptItems, useTransaction });
   } finally {
     if (!systemConnection) {
       await driver.close(dbhan);

package/src/utility/authProxy.js

@@ -41,16 +41,16 @@ function getAxiosParamsWithLicense() {
   };
 }
 
-function getExternalParamsWithLicense() {
+function getLicenseHttpHeaders() {
+  const licenseValue = licenseKey ?? process.env.DBGATE_LICENSE;
+  if (!licenseValue) {
+    return {};
+  }
   return {
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${licenseKey ?? process.env.DBGATE_LICENSE}`,
-    },
+    'x-license': licenseValue,
   };
 }
 
-
 async function authProxyGetRedirectUrl({ client, type, state, redirectUri }) {
   const respSession = await axios.default.post(
     `${AUTH_PROXY_URL}/create-session`,
@@ -148,7 +148,11 @@ async function obtainRefreshedLicense() {
     return null;
   }
 
-  const decoded = jwt.decode(licenseKey);
+  const decoded = jwt.decode(licenseKey?.trim());
+  if (!decoded?.end) {
+    logger.info('Invalid license found');
+    return null;
+  }
 
   if (Date.now() > decoded.end * 1000) {
     logger.info('License expired, trying to obtain fresh license');
@@ -291,5 +295,5 @@ module.exports = {
   callTextToSqlApi,
   callCompleteOnCursorApi,
   callRefactorSqlQueryApi,
-  getExternalParamsWithLicense,
+  getLicenseHttpHeaders,
 };

package/src/utility/checkLicense.js

@@ -116,7 +116,7 @@ function checkLicenseKey(licenseKey) {
   }
 
   try {
-    const decoded = jwt.verify(licenseKey, publicKey, {
+    const decoded = jwt.verify(licenseKey?.trim(), publicKey, {
       algorithms: ['RS256'],
     });
 
@@ -144,7 +144,7 @@ function checkLicenseKey(licenseKey) {
   } catch (err) {
     try {
       // detect expired license
-      const decoded = jwt.decode(licenseKey);
+      const decoded = jwt.decode(licenseKey?.trim());
       if (decoded) {
         const { exp } = decoded;
         if (exp * 1000 < Date.now()) {
@@ -222,18 +222,16 @@ async function checkLicense() {
   }
 
   const datadir = path.join(os.homedir(), '.dbgate');
-  if (isElectron()) {
-    try {
-      const licenseKey = fs.readFileSync(path.join(datadir, 'license.key'), {
-        encoding: 'utf-8',
-      });
-      setAuthProxyLicense(licenseKey);
-      if (licenseKey) {
-        return checkLicenseKey(licenseKey);
-      }
-    } catch (err) {
-      logger.warn(extractErrorLogData(err), 'Error loading license key');
+  try {
+    const licenseKey = fs.readFileSync(path.join(datadir, 'license.key'), {
+      encoding: 'utf-8',
+    });
+    setAuthProxyLicense(licenseKey);
+    if (licenseKey) {
+      return checkLicenseKey(licenseKey);
     }
+  } catch (err) {
+    logger.warn(extractErrorLogData(err), 'Error loading license key');
   }
 
   if (isElectron()) {

package/src/utility/cloudIntf.js

@@ -0,0 +1,399 @@
+const axios = require('axios');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { getLicenseHttpHeaders } = require('./authProxy');
+const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
+const { datadir } = require('./directories');
+const platformInfo = require('./platformInfo');
+const connections = require('../controllers/connections');
+const { isProApp } = require('./checkLicense');
+const socket = require('./socket');
+const config = require('../controllers/config');
+const simpleEncryptor = require('simple-encryptor');
+const currentVersion = require('../currentVersion');
+const { getPublicIpInfo } = require('./hardwareFingerprint');
+
+const logger = getLogger('cloudIntf');
+
+let cloudFiles = null;
+
+const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY
+  ? 'http://localhost:3103'
+  : process.env.PROD_DBGATE_IDENTITY
+  ? 'https://identity.dbgate.io'
+  : process.env.DEVWEB || process.env.DEVMODE
+  ? 'https://identity.dbgate.udolni.net'
+  : 'https://identity.dbgate.io';
+
+const DBGATE_CLOUD_URL = process.env.LOCAL_DBGATE_CLOUD
+  ? 'http://localhost:3110'
+  : process.env.PROD_DBGATE_CLOUD
+  ? 'https://cloud.dbgate.io'
+  : process.env.DEVWEB || process.env.DEVMODE
+  ? 'https://cloud.dbgate.udolni.net'
+  : 'https://cloud.dbgate.io';
+
+async function createDbGateIdentitySession(client) {
+  const resp = await axios.default.post(
+    `${DBGATE_IDENTITY_URL}/api/create-session`,
+    {
+      client,
+    },
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        'Content-Type': 'application/json',
+      },
+    }
+  );
+  return {
+    sid: resp.data.sid,
+    url: `${DBGATE_IDENTITY_URL}/api/signin/${resp.data.sid}`,
+  };
+}
+
+function startCloudTokenChecking(sid, callback) {
+  const started = Date.now();
+  const interval = setInterval(async () => {
+    if (Date.now() - started > 60 * 1000) {
+      clearInterval(interval);
+      return;
+    }
+
+    try {
+      // console.log(`Checking cloud token for session: ${DBGATE_IDENTITY_URL}/api/get-token/${sid}`);
+      const resp = await axios.default.get(`${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, {
+        headers: {
+          ...getLicenseHttpHeaders(),
+        },
+      });
+      // console.log('CHECK RESP:', resp.data);
+
+      if (resp.data.email) {
+        clearInterval(interval);
+        callback(resp.data);
+      }
+    } catch (err) {
+      logger.error(extractErrorLogData(err), 'Error checking cloud token');
+    }
+  }, 500);
+}
+
+async function loadCloudFiles() {
+  try {
+    const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
+    const parsedJson = jsonLinesParse(fileContent);
+    cloudFiles = _.sortBy(parsedJson, x => `${x.folder}/${x.title}`);
+  } catch (err) {
+    cloudFiles = [];
+  }
+}
+
+async function getCloudUsedEngines() {
+  try {
+    const resp = await callCloudApiGet('content-engines');
+    return resp || [];
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error getting cloud content list');
+    return [];
+  }
+}
+
+async function collectCloudFilesSearchTags() {
+  const res = [];
+  if (platformInfo.isElectron) {
+    res.push('app');
+  } else {
+    res.push('web');
+  }
+  if (platformInfo.isWindows) {
+    res.push('windows');
+  }
+  if (platformInfo.isMac) {
+    res.push('mac');
+  }
+  if (platformInfo.isLinux) {
+    res.push('linux');
+  }
+  if (platformInfo.isAwsUbuntuLayout) {
+    res.push('aws');
+  }
+  if (platformInfo.isAzureUbuntuLayout) {
+    res.push('azure');
+  }
+  if (platformInfo.isSnap) {
+    res.push('snap');
+  }
+  if (platformInfo.isDocker) {
+    res.push('docker');
+  }
+  if (platformInfo.isNpmDist) {
+    res.push('npm');
+  }
+  const engines = await connections.getUsedEngines();
+  const engineTags = engines.map(engine => engine.split('@')[0]);
+  res.push(...engineTags);
+  const cloudEngines = await getCloudUsedEngines();
+  const cloudEngineTags = cloudEngines.map(engine => engine.split('@')[0]);
+  res.push(...cloudEngineTags);
+
+  // team-premium and trials will return the same cloud files as premium - no need to check
+  res.push(isProApp() ? 'premium' : 'community');
+
+  return _.uniq(res);
+}
+
+async function getCloudSigninHolder() {
+  const settingsValue = await config.getSettings();
+  const holder = settingsValue['cloudSigninTokenHolder'];
+  return holder;
+}
+
+async function getCloudSigninHeaders(holder = null) {
+  if (!holder) {
+    holder = await getCloudSigninHolder();
+  }
+  if (holder) {
+    return {
+      'x-cloud-login': holder.token,
+    };
+  }
+  return null;
+}
+
+async function updateCloudFiles(isRefresh) {
+  let lastCloudFilesTags;
+  try {
+    lastCloudFilesTags = await fs.readFile(path.join(datadir(), 'cloud-files-tags.txt'), 'utf-8');
+  } catch (err) {
+    lastCloudFilesTags = '';
+  }
+
+  const ipInfo = await getPublicIpInfo();
+
+  const tags = (await collectCloudFilesSearchTags()).join(',');
+  let lastCheckedTm = 0;
+  if (tags == lastCloudFilesTags && cloudFiles.length > 0) {
+    lastCheckedTm = _.max(cloudFiles.map(x => parseInt(x.modifiedTm)));
+  }
+
+  logger.info({ tags, lastCheckedTm }, 'Downloading cloud files');
+
+  const resp = await axios.default.get(
+    `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}&isRefresh=${
+      isRefresh ? 1 : 0
+    }&country=${ipInfo?.country || ''}`,
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+        ...(await getCloudSigninHeaders()),
+        'x-app-version': currentVersion.version,
+      },
+    }
+  );
+
+  logger.info(`Downloaded ${resp.data.length} cloud files`);
+
+  const filesByPath = lastCheckedTm == 0 ? {} : _.keyBy(cloudFiles, 'path');
+  for (const file of resp.data) {
+    if (file.isDeleted) {
+      delete filesByPath[file.path];
+    } else {
+      filesByPath[file.path] = file;
+    }
+  }
+
+  cloudFiles = Object.values(filesByPath);
+
+  await fs.writeFile(path.join(datadir(), 'cloud-files.jsonl'), cloudFiles.map(x => JSON.stringify(x)).join('\n'));
+  await fs.writeFile(path.join(datadir(), 'cloud-files-tags.txt'), tags);
+
+  socket.emitChanged(`public-cloud-changed`);
+}
+
+async function startCloudFiles() {
+  loadCloudFiles();
+}
+
+async function getPublicCloudFiles() {
+  if (!loadCloudFiles) {
+    await loadCloudFiles();
+  }
+  return cloudFiles;
+}
+
+async function getPublicFileData(path) {
+  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/public/${path}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+    },
+  });
+  return resp.data;
+}
+
+async function refreshPublicFiles(isRefresh) {
+  if (!cloudFiles) {
+    await loadCloudFiles();
+  }
+  try {
+    await updateCloudFiles(isRefresh);
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error updating cloud files');
+  }
+}
+
+async function callCloudApiGet(endpoint, signinHolder = null, additionalHeaders = {}) {
+  if (!signinHolder) {
+    signinHolder = await getCloudSigninHolder();
+  }
+  if (!signinHolder) {
+    return null;
+  }
+  const signinHeaders = await getCloudSigninHeaders(signinHolder);
+
+  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/${endpoint}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+      ...signinHeaders,
+      ...additionalHeaders,
+    },
+    validateStatus: status => status < 500,
+  });
+  const { errorMessage, isLicenseLimit, limitedLicenseLimits } = resp.data;
+  if (errorMessage) {
+    return {
+      apiErrorMessage: errorMessage,
+      apiErrorIsLicenseLimit: isLicenseLimit,
+      apiErrorLimitedLicenseLimits: limitedLicenseLimits,
+    };
+  }
+  return resp.data;
+}
+
+async function callCloudApiPost(endpoint, body, signinHolder = null) {
+  if (!signinHolder) {
+    signinHolder = await getCloudSigninHolder();
+  }
+  if (!signinHolder) {
+    return null;
+  }
+  const signinHeaders = await getCloudSigninHeaders(signinHolder);
+
+  const resp = await axios.default.post(`${DBGATE_CLOUD_URL}/${endpoint}`, body, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+      ...signinHeaders,
+    },
+    validateStatus: status => status < 500,
+  });
+  const { errorMessage, isLicenseLimit, limitedLicenseLimits } = resp.data;
+  if (errorMessage) {
+    return {
+      apiErrorMessage: errorMessage,
+      apiErrorIsLicenseLimit: isLicenseLimit,
+      apiErrorLimitedLicenseLimits: limitedLicenseLimits,
+    };
+  }
+  return resp.data;
+}
+
+async function getCloudFolderEncryptor(folid) {
+  const { encryptionKey } = await callCloudApiGet(`folder-key/${folid}`);
+  if (!encryptionKey) {
+    throw new Error('No encryption key for folder: ' + folid);
+  }
+  return simpleEncryptor.createEncryptor(encryptionKey);
+}
+
+async function getCloudContent(folid, cntid) {
+  const signinHolder = await getCloudSigninHolder();
+  if (!signinHolder) {
+    throw new Error('No signed in');
+  }
+
+  const encryptor = simpleEncryptor.createEncryptor(signinHolder.encryptionKey);
+
+  const { content, name, type, contentAttributes, apiErrorMessage } = await callCloudApiGet(
+    `content/${folid}/${cntid}`,
+    signinHolder,
+    {
+      'x-kehid': signinHolder.kehid,
+    }
+  );
+
+  if (apiErrorMessage) {
+    return { apiErrorMessage };
+  }
+
+  return {
+    content: encryptor.decrypt(content),
+    name,
+    type,
+    contentAttributes,
+  };
+}
+
+/**
+ *
+ * @returns Promise<{ cntid: string } | { apiErrorMessage: string }>
+ */
+async function putCloudContent(folid, cntid, content, name, type, contentAttributes) {
+  const signinHolder = await getCloudSigninHolder();
+  if (!signinHolder) {
+    throw new Error('No signed in');
+  }
+
+  const encryptor = simpleEncryptor.createEncryptor(signinHolder.encryptionKey);
+
+  const resp = await callCloudApiPost(
+    `put-content`,
+    {
+      folid,
+      cntid,
+      name,
+      type,
+      kehid: signinHolder.kehid,
+      content: encryptor.encrypt(content),
+      contentAttributes,
+    },
+    signinHolder
+  );
+  socket.emitChanged('cloud-content-changed');
+  socket.emit('cloud-content-updated');
+  return resp;
+}
+
+const cloudConnectionCache = {};
+async function loadCachedCloudConnection(folid, cntid) {
+  const cacheKey = `${folid}|${cntid}`;
+  if (!cloudConnectionCache[cacheKey]) {
+    const { content } = await getCloudContent(folid, cntid);
+    cloudConnectionCache[cacheKey] = {
+      ...JSON.parse(content),
+      _id: `cloud://${folid}/${cntid}`,
+    };
+  }
+  return cloudConnectionCache[cacheKey];
+}
+
+function removeCloudCachedConnection(folid, cntid) {
+  const cacheKey = `${folid}|${cntid}`;
+  delete cloudConnectionCache[cacheKey];
+}
+
+module.exports = {
+  createDbGateIdentitySession,
+  startCloudTokenChecking,
+  startCloudFiles,
+  getPublicCloudFiles,
+  getPublicFileData,
+  refreshPublicFiles,
+  callCloudApiGet,
+  callCloudApiPost,
+  getCloudFolderEncryptor,
+  getCloudContent,
+  loadCachedCloudConnection,
+  putCloudContent,
+  removeCloudCachedConnection,
+};

package/src/utility/crypting.js

@@ -81,11 +81,11 @@ function decryptPasswordString(password) {
   return password;
 }
 
-function encryptObjectPasswordField(obj, field) {
+function encryptObjectPasswordField(obj, field, encryptor = null) {
   if (obj && obj[field] && !obj[field].startsWith('crypt:')) {
     return {
       ...obj,
-      [field]: 'crypt:' + getInternalEncryptor().encrypt(obj[field]),
+      [field]: 'crypt:' + (encryptor || getInternalEncryptor()).encrypt(obj[field]),
     };
   }
   return obj;
@@ -101,11 +101,11 @@ function decryptObjectPasswordField(obj, field) {
   return obj;
 }
 
-function encryptConnection(connection) {
+function encryptConnection(connection, encryptor = null) {
   if (connection.passwordMode != 'saveRaw') {
-    connection = encryptObjectPasswordField(connection, 'password');
-    connection = encryptObjectPasswordField(connection, 'sshPassword');
-    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword');
+    connection = encryptObjectPasswordField(connection, 'password', encryptor);
+    connection = encryptObjectPasswordField(connection, 'sshPassword', encryptor);
+    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword', encryptor);
   }
   return connection;
 }

package/src/utility/handleQueryStream.js

@@ -5,6 +5,8 @@ const _ = require('lodash');
 
 const { jsldir } = require('../utility/directories');
 const { serializeJsTypesReplacer } = require('dbgate-tools');
+const { ChartProcessor } = require('dbgate-datalib');
+const { isProApp } = require('./checkLicense');
 
 class QueryStreamTableWriter {
   constructor(sesid = undefined) {
@@ -12,9 +14,13 @@ class QueryStreamTableWriter {
     this.currentChangeIndex = 1;
     this.initializedFile = false;
     this.sesid = sesid;
+    // if (isProApp()) {
+    //   this.chartProcessor = new ChartProcessor();
+    // }
+    this.chartProcessor = new ChartProcessor();
   }
 
-  initializeFromQuery(structure, resultIndex) {
+  initializeFromQuery(structure, resultIndex, chartDefinition) {
     this.jslid = crypto.randomUUID();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
@@ -28,6 +34,9 @@ class QueryStreamTableWriter {
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
     this.initializedFile = true;
+    if (isProApp() && chartDefinition) {
+      this.chartProcessor = new ChartProcessor([chartDefinition]);
+    }
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
   }
 
@@ -40,6 +49,15 @@ class QueryStreamTableWriter {
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row, serializeJsTypesReplacer) + '\n');
+    try {
+      if (this.chartProcessor) {
+        this.chartProcessor.addRow(row);
+      }
+    } catch (e) {
+      console.error('Error processing chart row', e);
+      this.chartProcessor = null;
+    }
+
     this.currentRowCount += 1;
 
     if (!this.plannedStats) {
@@ -87,6 +105,23 @@ class QueryStreamTableWriter {
         this.currentStream.end(() => {
           this.writeCurrentStats(true, true);
           if (afterClose) afterClose();
+          if (this.chartProcessor) {
+            try {
+              this.chartProcessor.finalize();
+              if (isProApp() && this.chartProcessor.charts.length > 0) {
+                process.send({
+                  msgtype: 'charts',
+                  sesid: this.sesid,
+                  jslid: this.jslid,
+                  charts: this.chartProcessor.charts,
+                  resultIndex: this.resultIndex,
+                });
+              }
+            } catch (e) {
+              console.error('Error finalizing chart processor', e);
+              this.chartProcessor = null;
+            }
+          }
           resolve();
         });
       } else {
@@ -97,10 +132,18 @@ class QueryStreamTableWriter {
 }
 
 class StreamHandler {
-  constructor(queryStreamInfoHolder, resolve, startLine, sesid = undefined, limitRows = undefined) {
+  constructor(
+    queryStreamInfoHolder,
+    resolve,
+    startLine,
+    sesid = undefined,
+    limitRows = undefined,
+    frontMatter = undefined
+  ) {
     this.recordset = this.recordset.bind(this);
     this.startLine = startLine;
     this.sesid = sesid;
+    this.frontMatter = frontMatter;
     this.limitRows = limitRows;
     this.rowsLimitOverflow = false;
     this.row = this.row.bind(this);
@@ -133,7 +176,8 @@ class StreamHandler {
     this.currentWriter = new QueryStreamTableWriter(this.sesid);
     this.currentWriter.initializeFromQuery(
       Array.isArray(columns) ? { columns } : columns,
-      this.queryStreamInfoHolder.resultIndex
+      this.queryStreamInfoHolder.resultIndex,
+      this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`]
     );
     this.queryStreamInfoHolder.resultIndex += 1;
     this.rowCounter = 0;
@@ -201,10 +245,25 @@ class StreamHandler {
   }
 }
 
-function handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid = undefined, limitRows = undefined) {
+function handleQueryStream(
+  dbhan,
+  driver,
+  queryStreamInfoHolder,
+  sqlItem,
+  sesid = undefined,
+  limitRows = undefined,
+  frontMatter = undefined
+) {
   return new Promise((resolve, reject) => {
     const start = sqlItem.trimStart || sqlItem.start;
-    const handler = new StreamHandler(queryStreamInfoHolder, resolve, start && start.line, sesid, limitRows);
+    const handler = new StreamHandler(
+      queryStreamInfoHolder,
+      resolve,
+      start && start.line,
+      sesid,
+      limitRows,
+      frontMatter
+    );
     driver.stream(dbhan, sqlItem.text, handler);
   });
 }