dbgate-api 6.3.2 → 6.4.0

package/src/utility/DatastoreProxy.js

@@ -60,6 +60,10 @@ class DatastoreProxy {
         // if (this.disconnected) return;
         this.subprocess = null;
       });
+      this.subprocess.on('error', err => {
+        logger.error(extractErrorLogData(err), 'Error in data store subprocess');
+        this.subprocess = null;
+      });
       this.subprocess.send({ msgtype: 'open', file: this.file });
     }
     return this.subprocess;

package/src/utility/cloudUpgrade.js

@@ -1,61 +1,3 @@
-const axios = require('axios');
-const fs = require('fs');
-const fsp = require('fs/promises');
-const semver = require('semver');
-const currentVersion = require('../currentVersion');
-const { getLogger, extractErrorLogData } = require('dbgate-tools');
-
-const logger = getLogger('cloudUpgrade');
-
-async function checkCloudUpgrade() {
-  try {
-    const resp = await axios.default.get('https://api.github.com/repos/dbgate/dbgate/releases/latest');
-    const json = resp.data;
-    const version = json.name.substring(1);
-    let cloudDownloadedVersion = null;
-    try {
-      cloudDownloadedVersion = await fsp.readFile(process.env.CLOUD_UPGRADE_FILE + '.version', 'utf-8');
-    } catch (err) {
-      cloudDownloadedVersion = null;
-    }
-    if (
-      semver.gt(version, currentVersion.version) &&
-      (!cloudDownloadedVersion || semver.gt(version, cloudDownloadedVersion))
-    ) {
-      logger.info(`New version available: ${version}`);
-      const zipUrl = json.assets.find(x => x.name == 'cloud-build.zip').browser_download_url;
-
-      const writer = fs.createWriteStream(process.env.CLOUD_UPGRADE_FILE);
-
-      const response = await axios.default({
-        url: zipUrl,
-        method: 'GET',
-        responseType: 'stream',
-      });
-
-      response.data.pipe(writer);
-
-      await new Promise((resolve, reject) => {
-        writer.on('finish', resolve);
-        writer.on('error', reject);
-      });
-      await fsp.writeFile(process.env.CLOUD_UPGRADE_FILE + '.version', version);
-
-      logger.info(`Downloaded new version from ${zipUrl}`);
-    } else {
-      logger.info(`Checked version ${version} is not newer than ${cloudDownloadedVersion ?? currentVersion.version}, upgrade skippped`);
-    }
-  } catch (err) {
-    logger.error(extractErrorLogData(err), 'Error checking cloud upgrade');
-  }
-}
-
-function startCloudUpgradeTimer() {
-  // at first in 5 seconds
-  setTimeout(checkCloudUpgrade, 5000);
-
-  // hourly
-  setInterval(checkCloudUpgrade, 60 * 60 * 1000);
-}
+function startCloudUpgradeTimer() {}
 
 module.exports = startCloudUpgradeTimer;

package/src/utility/connectUtility.js

@@ -96,7 +96,9 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
     ...decryptConnection(connectionLoaded),
   };
 
-  if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
+  if (!connection.port && driver.defaultPort) {
+    connection.port = driver.defaultPort.toString();
+  }
 
   if (connection.useSshTunnel) {
     const tunnel = await getSshTunnelProxy(connection);

package/src/utility/crypting.js

@@ -5,12 +5,16 @@ const path = require('path');
 const _ = require('lodash');
 
 const { datadir } = require('./directories');
+const { encryptionKeyArg } = require('./processArgs');
 
 const defaultEncryptionKey = 'mQAUaXhavRGJDxDTXSCg7Ej0xMmGCrx6OKA07DIMBiDcYYkvkaXjTAzPUEHEHEf9';
 
 let _encryptionKey = null;
 
 function loadEncryptionKey() {
+  if (encryptionKeyArg) {
+    return encryptionKeyArg;
+  }
   if (_encryptionKey) {
     return _encryptionKey;
   }
@@ -33,9 +37,29 @@ function loadEncryptionKey() {
   return _encryptionKey;
 }
 
+async function loadEncryptionKeyFromExternal(storedValue, setStoredValue) {
+  const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
+
+  if (!storedValue) {
+    const generatedKey = crypto.randomBytes(32);
+    const newKey = generatedKey.toString('hex');
+    const result = {
+      encryptionKey: newKey,
+    };
+    await setStoredValue(encryptor.encrypt(result));
+
+    setEncryptionKey(newKey);
+
+    return;
+  }
+
+  const data = encryptor.decrypt(storedValue);
+  setEncryptionKey(data['encryptionKey']);
+}
+
 let _encryptor = null;
 
-function getEncryptor() {
+function getInternalEncryptor() {
   if (_encryptor) {
     return _encryptor;
   }
@@ -43,35 +67,46 @@ function getEncryptor() {
   return _encryptor;
 }
 
-function encryptPasswordField(connection, field) {
-  if (
-    connection &&
-    connection[field] &&
-    !connection[field].startsWith('crypt:') &&
-    connection.passwordMode != 'saveRaw'
-  ) {
+function encryptPasswordString(password) {
+  if (password && !password.startsWith('crypt:')) {
+    return 'crypt:' + getInternalEncryptor().encrypt(password);
+  }
+  return password;
+}
+
+function decryptPasswordString(password) {
+  if (password && password.startsWith('crypt:')) {
+    return getInternalEncryptor().decrypt(password.substring('crypt:'.length));
+  }
+  return password;
+}
+
+function encryptObjectPasswordField(obj, field) {
+  if (obj && obj[field] && !obj[field].startsWith('crypt:')) {
     return {
-      ...connection,
-      [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
+      ...obj,
+      [field]: 'crypt:' + getInternalEncryptor().encrypt(obj[field]),
     };
   }
-  return connection;
+  return obj;
 }
 
-function decryptPasswordField(connection, field) {
-  if (connection && connection[field] && connection[field].startsWith('crypt:')) {
+function decryptObjectPasswordField(obj, field) {
+  if (obj && obj[field] && obj[field].startsWith('crypt:')) {
     return {
-      ...connection,
-      [field]: getEncryptor().decrypt(connection[field].substring('crypt:'.length)),
+      ...obj,
+      [field]: getInternalEncryptor().decrypt(obj[field].substring('crypt:'.length)),
     };
   }
-  return connection;
+  return obj;
 }
 
 function encryptConnection(connection) {
-  connection = encryptPasswordField(connection, 'password');
-  connection = encryptPasswordField(connection, 'sshPassword');
-  connection = encryptPasswordField(connection, 'sshKeyfilePassword');
+  if (connection.passwordMode != 'saveRaw') {
+    connection = encryptObjectPasswordField(connection, 'password');
+    connection = encryptObjectPasswordField(connection, 'sshPassword');
+    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword');
+  }
   return connection;
 }
 
@@ -81,12 +116,24 @@ function maskConnection(connection) {
 }
 
 function decryptConnection(connection) {
-  connection = decryptPasswordField(connection, 'password');
-  connection = decryptPasswordField(connection, 'sshPassword');
-  connection = decryptPasswordField(connection, 'sshKeyfilePassword');
+  connection = decryptObjectPasswordField(connection, 'password');
+  connection = decryptObjectPasswordField(connection, 'sshPassword');
+  connection = decryptObjectPasswordField(connection, 'sshKeyfilePassword');
   return connection;
 }
 
+function encryptUser(user) {
+  if (user.encryptPassword) {
+    user = encryptObjectPasswordField(user, 'password');
+  }
+  return user;
+}
+
+function decryptUser(user) {
+  user = decryptObjectPasswordField(user, 'password');
+  return user;
+}
+
 function pickSafeConnectionInfo(connection) {
   if (process.env.LOG_CONNECTION_SENSITIVE_VALUES) {
     return connection;
@@ -99,10 +146,78 @@ function pickSafeConnectionInfo(connection) {
   });
 }
 
+function setEncryptionKey(encryptionKey) {
+  _encryptionKey = encryptionKey;
+  _encryptor = null;
+  global.ENCRYPTION_KEY = encryptionKey;
+}
+
+function getEncryptionKey() {
+  return _encryptionKey;
+}
+
+function generateTransportEncryptionKey() {
+  const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
+  const result = {
+    encryptionKey: crypto.randomBytes(32).toString('hex'),
+  };
+  return encryptor.encrypt(result);
+}
+
+function createTransportEncryptor(encryptionData) {
+  const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
+  const data = encryptor.decrypt(encryptionData);
+  const res = simpleEncryptor.createEncryptor(data['encryptionKey']);
+  return res;
+}
+
+function recryptObjectPasswordField(obj, field, decryptEncryptor, encryptEncryptor) {
+  if (obj && obj[field] && obj[field].startsWith('crypt:')) {
+    return {
+      ...obj,
+      [field]: 'crypt:' + encryptEncryptor.encrypt(decryptEncryptor.decrypt(obj[field].substring('crypt:'.length))),
+    };
+  }
+  return obj;
+}
+
+function recryptObjectPasswordFieldInPlace(obj, field, decryptEncryptor, encryptEncryptor) {
+  if (obj && obj[field] && obj[field].startsWith('crypt:')) {
+    obj[field] = 'crypt:' + encryptEncryptor.encrypt(decryptEncryptor.decrypt(obj[field].substring('crypt:'.length)));
+  }
+}
+
+function recryptConnection(connection, decryptEncryptor, encryptEncryptor) {
+  connection = recryptObjectPasswordField(connection, 'password', decryptEncryptor, encryptEncryptor);
+  connection = recryptObjectPasswordField(connection, 'sshPassword', decryptEncryptor, encryptEncryptor);
+  connection = recryptObjectPasswordField(connection, 'sshKeyfilePassword', decryptEncryptor, encryptEncryptor);
+  return connection;
+}
+
+function recryptUser(user, decryptEncryptor, encryptEncryptor) {
+  user = recryptObjectPasswordField(user, 'password', decryptEncryptor, encryptEncryptor);
+  return user;
+}
+
 module.exports = {
   loadEncryptionKey,
   encryptConnection,
+  encryptUser,
+  decryptUser,
   decryptConnection,
   maskConnection,
   pickSafeConnectionInfo,
+  loadEncryptionKeyFromExternal,
+  getEncryptionKey,
+  setEncryptionKey,
+  encryptPasswordString,
+  decryptPasswordString,
+
+  getInternalEncryptor,
+  recryptConnection,
+  recryptUser,
+  generateTransportEncryptionKey,
+  createTransportEncryptor,
+  recryptObjectPasswordField,
+  recryptObjectPasswordFieldInPlace,
 };

package/src/utility/extractSingleFileFromZip.js

@@ -0,0 +1,77 @@
+const yauzl = require('yauzl');
+const fs = require('fs');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('extractSingleFileFromZip');
+/**
+ * Extracts a single file from a ZIP using yauzl.
+ * Stops reading the rest of the archive once the file is found.
+ *
+ * @param {string} zipPath - Path to the ZIP file on disk.
+ * @param {string} fileInZip - The file path *inside* the ZIP to extract.
+ * @param {string} outputPath - Where to write the extracted file on disk.
+ * @returns {Promise<boolean>} - Resolves with a success message or a "not found" message.
+ */
+function extractSingleFileFromZip(zipPath, fileInZip, outputPath) {
+  return new Promise((resolve, reject) => {
+    yauzl.open(zipPath, { lazyEntries: true }, (err, zipFile) => {
+      if (err) return reject(err);
+
+      let fileFound = false;
+
+      // Start reading the first entry
+      zipFile.readEntry();
+
+      zipFile.on('entry', entry => {
+        // Compare the entry name to the file we want
+        if (entry.fileName === fileInZip) {
+          fileFound = true;
+
+          // Open a read stream for this entry
+          zipFile.openReadStream(entry, (err, readStream) => {
+            if (err) return reject(err);
+
+            // Create a write stream to outputPath
+            const writeStream = fs.createWriteStream(outputPath);
+            readStream.pipe(writeStream);
+
+            // When the read stream ends, we can close the zipFile
+            readStream.on('end', () => {
+              // We won't read further entries
+              zipFile.close();
+            });
+
+            // When the file is finished writing, resolve
+            writeStream.on('finish', () => {
+              logger.info(`File "${fileInZip}" extracted to "${outputPath}".`);
+              resolve(true);
+            });
+
+            // Handle write errors
+            writeStream.on('error', writeErr => {
+              logger.error(extractErrorLogData(writeErr), `Error extracting "${fileInZip}" from "${zipPath}".`);
+              reject(writeErr);
+            });
+          });
+        } else {
+          // Not the file we want; skip to the next entry
+          zipFile.readEntry();
+        }
+      });
+
+      // If we reach the end without finding the file
+      zipFile.on('end', () => {
+        if (!fileFound) {
+          resolve(false);
+        }
+      });
+
+      // Handle general errors
+      zipFile.on('error', err => {
+        logger.error(extractErrorLogData(err), `ZIP file error in ${zipPath}.`);
+        reject(err);
+      });
+    });
+  });
+}
+
+module.exports = extractSingleFileFromZip;

package/src/utility/getMapExport.js

@@ -22,6 +22,8 @@ const getMapExport = (geoJson) => {
       })
       .addTo(map);
       
+	  leaflet.control.scale().addTo(map);
+	  
       const geoJsonObj = leaflet
       .geoJSON(${JSON.stringify(geoJson)}, {
         style: function () {

package/src/utility/handleQueryStream.js

@@ -0,0 +1,186 @@
+const crypto = require('crypto');
+const path = require('path');
+const fs = require('fs');
+const _ = require('lodash');
+
+const { jsldir } = require('../utility/directories');
+
+class QueryStreamTableWriter {
+  constructor(sesid = undefined) {
+    this.currentRowCount = 0;
+    this.currentChangeIndex = 1;
+    this.initializedFile = false;
+    this.sesid = sesid;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = crypto.randomUUID();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
+    this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+    this.writeCurrentStats(false, false);
+    this.resultIndex = resultIndex;
+    this.initializedFile = true;
+    process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
+  }
+
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
+  row(row) {
+    // console.log('ACCEPT ROW', row);
+    this.currentStream.write(JSON.stringify(row) + '\n');
+    this.currentRowCount += 1;
+
+    if (!this.plannedStats) {
+      this.plannedStats = true;
+      process.nextTick(() => {
+        if (this.currentStream) this.currentStream.uncork();
+        process.nextTick(() => this.writeCurrentStats(false, true));
+        this.plannedStats = false;
+      });
+    }
+  }
+
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid, sesid: this.sesid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
+  writeCurrentStats(isFinished = false, emitEvent = false) {
+    const stats = {
+      rowCount: this.currentRowCount,
+      changeIndex: this.currentChangeIndex,
+      isFinished,
+      jslid: this.jslid,
+    };
+    fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats));
+    this.currentChangeIndex += 1;
+    if (emitEvent) {
+      process.send({ msgtype: 'stats', sesid: this.sesid, ...stats });
+    }
+  }
+
+  close(afterClose) {
+    if (this.currentStream) {
+      this.currentStream.end(() => {
+        this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
+      });
+    }
+  }
+}
+
+class StreamHandler {
+  constructor(queryStreamInfoHolder, resolve, startLine, sesid = undefined) {
+    this.recordset = this.recordset.bind(this);
+    this.startLine = startLine;
+    this.sesid = sesid;
+    this.row = this.row.bind(this);
+    // this.error = this.error.bind(this);
+    this.done = this.done.bind(this);
+    this.info = this.info.bind(this);
+
+    // use this for cancelling - not implemented
+    // this.stream = null;
+
+    this.plannedStats = false;
+    this.queryStreamInfoHolder = queryStreamInfoHolder;
+    this.resolve = resolve;
+    // currentHandlers = [...currentHandlers, this];
+  }
+
+  closeCurrentWriter() {
+    if (this.currentWriter) {
+      this.currentWriter.close();
+      this.currentWriter = null;
+    }
+  }
+
+  recordset(columns) {
+    this.closeCurrentWriter();
+    this.currentWriter = new QueryStreamTableWriter(this.sesid);
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.queryStreamInfoHolder.resultIndex
+    );
+    this.queryStreamInfoHolder.resultIndex += 1;
+
+    // this.writeCurrentStats();
+
+    // this.onRow = _.throttle((jslid) => {
+    //   if (jslid == this.jslid) {
+    //     this.writeCurrentStats(false, true);
+    //   }
+    // }, 500);
+  }
+  row(row) {
+    if (this.currentWriter) this.currentWriter.row(row);
+    else if (row.message) process.send({ msgtype: 'info', info: { message: row.message }, sesid: this.sesid });
+    // this.onRow(this.jslid);
+  }
+  // error(error) {
+  //   process.send({ msgtype: 'error', error });
+  // }
+  done(result) {
+    this.closeCurrentWriter();
+    // currentHandlers = currentHandlers.filter((x) => x != this);
+    this.resolve();
+  }
+  info(info) {
+    if (info && info.line != null) {
+      info = {
+        ...info,
+        line: this.startLine + info.line,
+      };
+    }
+    if (info.severity == 'error') {
+      this.queryStreamInfoHolder.canceled = true;
+    }
+    process.send({ msgtype: 'info', info, sesid: this.sesid });
+  }
+}
+
+function handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid = undefined) {
+  return new Promise((resolve, reject) => {
+    const start = sqlItem.trimStart || sqlItem.start;
+    const handler = new StreamHandler(queryStreamInfoHolder, resolve, start && start.line, sesid);
+    driver.stream(dbhan, sqlItem.text, handler);
+  });
+}
+
+function allowExecuteCustomScript(storedConnection, driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
+module.exports = {
+  handleQueryStream,
+  QueryStreamTableWriter,
+  allowExecuteCustomScript,
+};

package/src/utility/healthStatus.js

@@ -24,4 +24,15 @@ async function getHealthStatus() {
   };
 }
 
-module.exports = getHealthStatus;
+async function getHealthStatusSprinx() {
+  return {
+    overallStatus: 'OK',
+    timeStamp: new Date().toISOString(),
+    timeStampUnix: Math.floor(Date.now() / 1000),
+  };
+}
+
+module.exports = {
+  getHealthStatus,
+  getHealthStatusSprinx,
+};

package/src/utility/listZipEntries.js

@@ -0,0 +1,41 @@
+const yauzl = require('yauzl');
+const path = require('path');
+
+/**
+ * Lists the files in a ZIP archive using yauzl,
+ * returning an array of { fileName, uncompressedSize } objects.
+ *
+ * @param {string} zipPath - The path to the ZIP file.
+ * @returns {Promise<Array<{fileName: string, uncompressedSize: number}>>}
+ */
+function listZipEntries(zipPath) {
+  return new Promise((resolve, reject) => {
+    yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
+      if (err) return reject(err);
+
+      const entries = [];
+
+      // Start reading entries
+      zipfile.readEntry();
+
+      // Handle each entry
+      zipfile.on('entry', entry => {
+        entries.push({
+          fileName: entry.fileName,
+          uncompressedSize: entry.uncompressedSize,
+        });
+
+        // Move on to the next entry (we’re only listing, not reading file data)
+        zipfile.readEntry();
+      });
+
+      // Finished reading all entries
+      zipfile.on('end', () => resolve(entries));
+
+      // Handle errors
+      zipfile.on('error', err => reject(err));
+    });
+  });
+}
+
+module.exports = listZipEntries;

package/src/utility/processArgs.js

@@ -17,6 +17,7 @@ const processDisplayName = getNamedArg('--process-display-name');
 const listenApi = process.argv.includes('--listen-api');
 const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;
 const runE2eTests = process.argv.includes('--run-e2e-tests');
+const encryptionKeyArg = getNamedArg('--encryption-key');
 
 function getPassArgs() {
   const res = [];
@@ -31,6 +32,9 @@ function getPassArgs() {
   if (runE2eTests) {
     res.push('--run-e2e-tests');
   }
+  if (global['ENCRYPTION_KEY']) {
+    res.push('--encryption-key', global['ENCRYPTION_KEY']);
+  }
   return res;
 }
 
@@ -45,4 +49,5 @@ module.exports = {
   listenApiChild,
   processDisplayName,
   runE2eTests,
+  encryptionKeyArg,
 };

package/src/utility/sshTunnel.js

@@ -57,10 +57,21 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
       }
     });
     subprocess.on('exit', code => {
-      logger.info('SSH forward process exited');
+      logger.info(`SSH forward process exited with code ${code}`);
       delete sshTunnelCache[tunnelCacheKey];
       if (!promiseHandled) {
-        reject(new Error('SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'));
+        reject(
+          new Error(
+            'SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'
+          )
+        );
+      }
+    });
+    subprocess.on('error', error => {
+      logger.error(extractErrorLogData(error), 'SSH forward process error');
+      delete sshTunnelCache[tunnelCacheKey];
+      if (!promiseHandled) {
+        reject(error);
       }
     });
   });