dbgate-api-premium 6.3.2 → 6.4.0

package/src/controllers/storageDb.js

@@ -1,7 +1,13 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const storageModel = require('../storageModel');
 const dbgateApi = require('../shell');
-const { getPredefinedPermissions, getLogger, extractErrorLogData } = require('dbgate-tools');
+const {
+  getPredefinedPermissions,
+  getLogger,
+  extractErrorLogData,
+  runQueryFmt,
+  runQueryOnDriver,
+} = require('dbgate-tools');
 const _ = require('lodash');
 const logger = getLogger('storageDb');
 const { extractConnectionSslParams } = require('../utility/connectUtility');
@@ -238,10 +244,60 @@ async function storageWriteConfig(group, config) {
   }
 }
 
+async function storageSaveRelationDiff(table, idColumn, valueColumn, idValue, newValueSet) {
+  const [conn, driver] = await getStorageConnection();
+  if (!conn) {
+    return null;
+  }
+
+  const oldValueSet = await storageSelectFmt(`select %i from %i where %i = %v`, valueColumn, table, idColumn, idValue);
+
+  const oldValues = oldValueSet.map(x => x[valueColumn]);
+
+  for (const added of _.difference(newValueSet, oldValues)) {
+    await runQueryFmt(
+      driver,
+      conn,
+      'insert into %i (%i, %i) values (%v, %v)',
+      table,
+      idColumn,
+      valueColumn,
+      idValue,
+      added
+    );
+  }
+
+  for (const removed of _.difference(oldValues, newValueSet)) {
+    await runQueryFmt(
+      driver,
+      conn,
+      'delete from %i where %i = %v and %i = %v',
+      table,
+      idColumn,
+      idValue,
+      valueColumn,
+      removed
+    );
+  }
+}
+
 function getStorageConnectionError() {
   return storageConnectionError;
 }
 
+async function selectStorageIdentity(tableName) {
+  const [conn, driver] = await getStorageConnection();
+
+  const resp = await runQueryOnDriver(conn, driver, dmp =>
+    dmp.selectScopeIdentity(
+      // @ts-ignore
+      { pureName: tableName }
+    )
+  );
+
+  return Object.entries(resp.rows[0])[0][1];
+}
+
 module.exports = {
   getStorageConnection,
   storageSelectFmt,
@@ -254,4 +310,6 @@ module.exports = {
   storageWriteConfig,
   getStorageConnectionError,
   storageSqlCommandFmt,
+  storageSaveRelationDiff,
+  selectStorageIdentity,
 };

package/src/controllers/uploads.js

@@ -39,52 +39,6 @@ module.exports = {
     });
   },
 
-  uploadDataFile_meta: {
-    method: 'post',
-    raw: true,
-  },
-  uploadDataFile(req, res) {
-    const { data } = req.files || {};
-
-    if (!data) {
-      res.json(null);
-      return;
-    }
-
-    if (data.name.toLowerCase().endsWith('.sql')) {
-      logger.info(`Uploading SQL file ${data.name}, size=${data.size}`);
-      data.mv(path.join(filesdir(), 'sql', data.name), () => {
-        res.json({
-          name: data.name,
-          folder: 'sql',
-        });
-
-        socket.emitChanged(`files-changed`, { folder: 'sql' });
-        socket.emitChanged(`all-files-changed`);
-      });
-      return;
-    }
-
-    res.json(null);
-  },
-
-  saveDataFile_meta: true,
-  async saveDataFile({ filePath }) {
-    if (filePath.toLowerCase().endsWith('.sql')) {
-      logger.info(`Saving SQL file ${filePath}`);
-      await fs.copyFile(filePath, path.join(filesdir(), 'sql', path.basename(filePath)));
-
-      socket.emitChanged(`files-changed`, { folder: 'sql' });
-      socket.emitChanged(`all-files-changed`);
-      return {
-        name: path.basename(filePath),
-        folder: 'sql',
-      };
-    }
-
-    return null;
-  },
-
   get_meta: {
     method: 'get',
     raw: true,

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '6.3.2',
-  buildTime: '2025-04-02T12:10:04.222Z'
+  version: '6.4.0',
+  buildTime: '2025-04-30T07:59:56.792Z'
 };

package/src/main.js

@@ -38,7 +38,7 @@ const { getLogger } = require('dbgate-tools');
 const { getDefaultAuthProvider } = require('./auth/authProvider');
 const startCloudUpgradeTimer = require('./utility/cloudUpgrade');
 const { isProApp } = require('./utility/checkLicense');
-const getHealthStatus = require('./utility/healthStatus');
+const { getHealthStatus, getHealthStatusSprinx } = require('./utility/healthStatus');
 
 const logger = getLogger('main');
 
@@ -124,6 +124,12 @@ function start() {
     res.end(JSON.stringify(health, null, 2));
   });
 
+  app.get(getExpressPath('/__health'), async function (req, res) {
+    res.setHeader('Content-Type', 'application/json');
+    const health = await getHealthStatusSprinx();
+    res.end(JSON.stringify(health, null, 2));
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(

package/src/proc/connectProcess.js

@@ -4,6 +4,8 @@ const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
 const _ = require('lodash');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('connectProcess');
 
 const formatErrorDetail = (e, connection) => `${e.stack}
 
@@ -23,12 +25,22 @@ function start() {
     try {
       const driver = requireEngineDriver(connection);
       const dbhan = await connectUtility(driver, connection, 'app');
-      const res = await driver.getVersion(dbhan);
+      let version = {
+        version: 'Unknown',
+      };
+      try {
+        version = await driver.getVersion(dbhan);
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'Error getting DB server version');
+        version = {
+          version: 'Unknown',
+        };
+      }
       let databases = undefined;
       if (requestDbList) {
         databases = await driver.listDatabases(dbhan);
       }
-      process.send({ msgtype: 'connected', ...res, databases });
+      process.send({ msgtype: 'connected', ...version, databases });
       await driver.close(dbhan);
     } catch (e) {
       console.error(e);

package/src/proc/databaseConnectionProcess.js

@@ -9,13 +9,21 @@ const {
   dbNameLogCategory,
   extractErrorMessage,
   extractErrorLogData,
+  ScriptWriterEval,
+  SqlGenerator,
+  playJsonScriptWriter,
 } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
-const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { dumpSqlSelect } = require('dbgate-sqltree');
+const { allowExecuteCustomScript, handleQueryStream } = require('../utility/handleQueryStream');
+const dbgateApi = require('../shell');
+const requirePlugin = require('../shell/requirePlugin');
+const path = require('path');
+const { rundir } = require('../utility/directories');
+const fs = require('fs-extra');
 
 const logger = getLogger('dbconnProcess');
 
@@ -120,10 +128,15 @@ function setStatusName(name) {
 
 async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
-  const version = await driver.getVersion(dbhan);
-  logger.debug(`Got server version: ${version.version}`);
-  process.send({ msgtype: 'version', version });
-  serverVersion = version;
+  try {
+    const version = await driver.getVersion(dbhan);
+    logger.debug(`Got server version: ${version.version}`);
+    serverVersion = version;
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error getting DB server version');
+    serverVersion = { version: 'Unknown' };
+  }
+  process.send({ msgtype: 'version', version: serverVersion });
 }
 
 async function handleConnect({ connection, structure, globalSettings }) {
@@ -370,6 +383,56 @@ async function handleGenerateDeploySql({ msgid, modelFolder }) {
   }
 }
 
+async function handleExecuteSessionQuery({ sesid, sql }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(storedConnection, driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+      sesid,
+    });
+    process.send({ msgtype: 'done', sesid, skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
+  const queryStreamInfoHolder = {
+    resultIndex: 0,
+    canceled: false,
+  };
+  for (const sqlItem of splitQuery(sql, {
+    ...driver.getQuerySplitterOptions('stream'),
+    returnRichInfo: true,
+  })) {
+    await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, sesid);
+    if (queryStreamInfoHolder.canceled) {
+      break;
+    }
+  }
+  process.send({ msgtype: 'done', sesid });
+}
+
+async function handleEvalJsonScript({ script, runid }) {
+  const directory = path.join(rundir(), runid);
+  fs.mkdirSync(directory);
+  const originalCwd = process.cwd();
+
+  try {
+    process.chdir(directory);
+
+    const evalWriter = new ScriptWriterEval(dbgateApi, requirePlugin, dbhan, runid);
+    await playJsonScriptWriter(script, evalWriter);
+    process.send({ msgtype: 'runnerDone', runid });
+  } finally {
+    process.chdir(originalCwd);
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -400,6 +463,8 @@ const messageHandlers = {
   sqlSelect: handleSqlSelect,
   exportKeys: handleExportKeys,
   schemaList: handleSchemaList,
+  executeSessionQuery: handleExecuteSessionQuery,
+  evalJsonScript: handleEvalJsonScript,
   // runCommand: handleRunCommand,
 };
 

package/src/proc/serverConnectionProcess.js

@@ -46,7 +46,13 @@ async function handleRefresh() {
 
 async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
-  const version = await driver.getVersion(dbhan);
+  let version;
+  try {
+    version = await driver.getVersion(dbhan);
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error getting DB server version');
+    version = { version: 'Unknown' };
+  }
   process.send({ msgtype: 'version', version });
 }
 

package/src/proc/sessionProcess.js

@@ -11,6 +11,7 @@ const { decryptConnection } = require('../utility/crypting');
 const { connectUtility } = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+const { handleQueryStream, QueryStreamTableWriter, allowExecuteCustomScript } = require('../utility/handleQueryStream');
 
 const logger = getLogger('sessionProcess');
 
@@ -23,175 +24,6 @@ let lastActivity = null;
 let currentProfiler = null;
 let executingScripts = 0;
 
-class TableWriter {
-  constructor() {
-    this.currentRowCount = 0;
-    this.currentChangeIndex = 1;
-    this.initializedFile = false;
-  }
-
-  initializeFromQuery(structure, resultIndex) {
-    this.jslid = crypto.randomUUID();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
-    fs.writeFileSync(
-      this.currentFile,
-      JSON.stringify({
-        ...structure,
-        __isStreamHeader: true,
-      }) + '\n'
-    );
-    this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
-    this.writeCurrentStats(false, false);
-    this.resultIndex = resultIndex;
-    this.initializedFile = true;
-    process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
-  }
-
-  initializeFromReader(jslid) {
-    this.jslid = jslid;
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
-    this.writeCurrentStats(false, false);
-  }
-
-  row(row) {
-    // console.log('ACCEPT ROW', row);
-    this.currentStream.write(JSON.stringify(row) + '\n');
-    this.currentRowCount += 1;
-
-    if (!this.plannedStats) {
-      this.plannedStats = true;
-      process.nextTick(() => {
-        if (this.currentStream) this.currentStream.uncork();
-        process.nextTick(() => this.writeCurrentStats(false, true));
-        this.plannedStats = false;
-      });
-    }
-  }
-
-  rowFromReader(row) {
-    if (!this.initializedFile) {
-      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
-      this.initializedFile = true;
-
-      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
-      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
-      this.writeCurrentStats(false, false);
-      this.initializedFile = true;
-      return;
-    }
-
-    this.row(row);
-  }
-
-  writeCurrentStats(isFinished = false, emitEvent = false) {
-    const stats = {
-      rowCount: this.currentRowCount,
-      changeIndex: this.currentChangeIndex,
-      isFinished,
-      jslid: this.jslid,
-    };
-    fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats));
-    this.currentChangeIndex += 1;
-    if (emitEvent) {
-      process.send({ msgtype: 'stats', ...stats });
-    }
-  }
-
-  close(afterClose) {
-    if (this.currentStream) {
-      this.currentStream.end(() => {
-        this.writeCurrentStats(true, true);
-        if (afterClose) afterClose();
-      });
-    }
-  }
-}
-
-class StreamHandler {
-  constructor(resultIndexHolder, resolve, startLine) {
-    this.recordset = this.recordset.bind(this);
-    this.startLine = startLine;
-    this.row = this.row.bind(this);
-    // this.error = this.error.bind(this);
-    this.done = this.done.bind(this);
-    this.info = this.info.bind(this);
-
-    // use this for cancelling - not implemented
-    // this.stream = null;
-
-    this.plannedStats = false;
-    this.resultIndexHolder = resultIndexHolder;
-    this.resolve = resolve;
-    // currentHandlers = [...currentHandlers, this];
-  }
-
-  closeCurrentWriter() {
-    if (this.currentWriter) {
-      this.currentWriter.close();
-      this.currentWriter = null;
-    }
-  }
-
-  recordset(columns) {
-    this.closeCurrentWriter();
-    this.currentWriter = new TableWriter();
-    this.currentWriter.initializeFromQuery(
-      Array.isArray(columns) ? { columns } : columns,
-      this.resultIndexHolder.value
-    );
-    this.resultIndexHolder.value += 1;
-
-    // this.writeCurrentStats();
-
-    // this.onRow = _.throttle((jslid) => {
-    //   if (jslid == this.jslid) {
-    //     this.writeCurrentStats(false, true);
-    //   }
-    // }, 500);
-  }
-  row(row) {
-    if (this.currentWriter) this.currentWriter.row(row);
-    else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
-    // this.onRow(this.jslid);
-  }
-  // error(error) {
-  //   process.send({ msgtype: 'error', error });
-  // }
-  done(result) {
-    this.closeCurrentWriter();
-    // currentHandlers = currentHandlers.filter((x) => x != this);
-    this.resolve();
-  }
-  info(info) {
-    if (info && info.line != null) {
-      info = {
-        ...info,
-        line: this.startLine + info.line,
-      };
-    }
-    process.send({ msgtype: 'info', info });
-  }
-}
-
-function handleStream(driver, resultIndexHolder, sqlItem) {
-  return new Promise((resolve, reject) => {
-    const start = sqlItem.trimStart || sqlItem.start;
-    const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line);
-    driver.stream(dbhan, sqlItem.text, handler);
-  });
-}
-
-function allowExecuteCustomScript(driver) {
-  if (driver.readOnlySessions) {
-    return true;
-  }
-  if (storedConnection.isReadOnly) {
-    return false;
-    // throw new Error('Connection is read only');
-  }
-  return true;
-}
-
 async function handleConnect(connection) {
   storedConnection = connection;
 
@@ -222,12 +54,12 @@ async function handleStartProfiler({ jslid }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
-  if (!allowExecuteCustomScript(driver)) {
+  if (!allowExecuteCustomScript(storedConnection, driver)) {
     process.send({ msgtype: 'done' });
     return;
   }
 
-  const writer = new TableWriter();
+  const writer = new QueryStreamTableWriter();
   writer.initializeFromReader(jslid);
 
   currentProfiler = await driver.startProfiler(dbhan, {
@@ -251,7 +83,7 @@ async function handleExecuteControlCommand({ command }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
-  if (command == 'commitTransaction' && !allowExecuteCustomScript(driver)) {
+  if (command == 'commitTransaction' && !allowExecuteCustomScript(storedConnection, driver)) {
     process.send({
       msgtype: 'info',
       info: {
@@ -291,7 +123,7 @@ async function handleExecuteQuery({ sql, autoCommit }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
-  if (!allowExecuteCustomScript(driver)) {
+  if (!allowExecuteCustomScript(storedConnection, driver)) {
     process.send({
       msgtype: 'info',
       info: {
@@ -306,18 +138,23 @@ async function handleExecuteQuery({ sql, autoCommit }) {
 
   executingScripts++;
   try {
-    const resultIndexHolder = {
-      value: 0,
+    const queryStreamInfoHolder = {
+      resultIndex: 0,
+      canceled: false,
     };
     for (const sqlItem of splitQuery(sql, {
       ...driver.getQuerySplitterOptions('stream'),
       returnRichInfo: true,
     })) {
-      await handleStream(driver, resultIndexHolder, sqlItem);
+      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem);
       // const handler = new StreamHandler(resultIndex);
       // const stream = await driver.stream(systemConnection, sqlItem, handler);
       // handler.stream = stream;
       // resultIndex = handler.resultIndex;
+
+      if (queryStreamInfoHolder.canceled) {
+        break;
+      }
     }
     process.send({ msgtype: 'done', autoCommit });
   } finally {
@@ -335,13 +172,13 @@ async function handleExecuteReader({ jslid, sql, fileName }) {
   if (fileName) {
     sql = fs.readFileSync(fileName, 'utf-8');
   } else {
-    if (!allowExecuteCustomScript(driver)) {
+    if (!allowExecuteCustomScript(storedConnection, driver)) {
       process.send({ msgtype: 'done' });
       return;
     }
   }
 
-  const writer = new TableWriter();
+  const writer = new QueryStreamTableWriter();
   writer.initializeFromReader(jslid);
 
   const reader = await driver.readQuery(dbhan, sql);

package/src/shell/archiveReader.js

@@ -3,7 +3,9 @@ const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
 function archiveReader({ folderName, fileName, ...other }) {
-  const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
+  const jsonlFile = folderName.endsWith('.zip')
+    ? `zip://archive:${folderName}//${fileName}.jsonl`
+    : path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
   const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }

package/src/shell/collectorWriter.js

@@ -15,9 +15,9 @@ class CollectorWriterStream extends stream.Writable {
 
   _final(callback) {
     process.send({
-      msgtype: 'freeData',
+      msgtype: 'dataResult',
       runid: this.runid,
-      freeData: { rows: this.rows, structure: this.structure },
+      dataResult: { rows: this.rows, structure: this.structure },
     });
     callback();
   }

package/src/shell/copyStream.js

@@ -69,6 +69,7 @@ async function copyStream(input, output, options) {
       msgtype: 'copyStreamError',
       copyStreamError: {
         message: extractErrorMessage(err),
+        progressName,
         ...err,
       },
     });