dbgate-api-premium 5.5.7-alpha.45

package/src/proc/sessionProcess.js

@@ -0,0 +1,390 @@
+const crypto = require('crypto');
+const path = require('path');
+const fs = require('fs');
+const _ = require('lodash');
+const childProcessChecker = require('../utility/childProcessChecker');
+const { splitQuery } = require('dbgate-query-splitter');
+
+const { jsldir } = require('../utility/directories');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+
+const logger = getLogger('sessionProcess');
+
+let dbhan;
+let storedConnection;
+let afterConnectCallbacks = [];
+// let currentHandlers = [];
+let lastPing = null;
+let lastActivity = null;
+let currentProfiler = null;
+let executingScripts = 0;
+
+class TableWriter {
+  constructor() {
+    this.currentRowCount = 0;
+    this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = crypto.randomUUID();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
+    this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+    this.writeCurrentStats(false, false);
+    this.resultIndex = resultIndex;
+    this.initializedFile = true;
+    process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
+  }
+
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
+  row(row) {
+    // console.log('ACCEPT ROW', row);
+    this.currentStream.write(JSON.stringify(row) + '\n');
+    this.currentRowCount += 1;
+
+    if (!this.plannedStats) {
+      this.plannedStats = true;
+      process.nextTick(() => {
+        if (this.currentStream) this.currentStream.uncork();
+        process.nextTick(() => this.writeCurrentStats(false, true));
+        this.plannedStats = false;
+      });
+    }
+  }
+
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
+  writeCurrentStats(isFinished = false, emitEvent = false) {
+    const stats = {
+      rowCount: this.currentRowCount,
+      changeIndex: this.currentChangeIndex,
+      isFinished,
+      jslid: this.jslid,
+    };
+    fs.writeFileSync(`${this.currentFile}.stats`, JSON.stringify(stats));
+    this.currentChangeIndex += 1;
+    if (emitEvent) {
+      process.send({ msgtype: 'stats', ...stats });
+    }
+  }
+
+  close(afterClose) {
+    if (this.currentStream) {
+      this.currentStream.end(() => {
+        this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
+      });
+    }
+  }
+}
+
+class StreamHandler {
+  constructor(resultIndexHolder, resolve, startLine) {
+    this.recordset = this.recordset.bind(this);
+    this.startLine = startLine;
+    this.row = this.row.bind(this);
+    // this.error = this.error.bind(this);
+    this.done = this.done.bind(this);
+    this.info = this.info.bind(this);
+
+    // use this for cancelling - not implemented
+    // this.stream = null;
+
+    this.plannedStats = false;
+    this.resultIndexHolder = resultIndexHolder;
+    this.resolve = resolve;
+    // currentHandlers = [...currentHandlers, this];
+  }
+
+  closeCurrentWriter() {
+    if (this.currentWriter) {
+      this.currentWriter.close();
+      this.currentWriter = null;
+    }
+  }
+
+  recordset(columns) {
+    this.closeCurrentWriter();
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
+    this.resultIndexHolder.value += 1;
+
+    // this.writeCurrentStats();
+
+    // this.onRow = _.throttle((jslid) => {
+    //   if (jslid == this.jslid) {
+    //     this.writeCurrentStats(false, true);
+    //   }
+    // }, 500);
+  }
+  row(row) {
+    if (this.currentWriter) this.currentWriter.row(row);
+    else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
+    // this.onRow(this.jslid);
+  }
+  // error(error) {
+  //   process.send({ msgtype: 'error', error });
+  // }
+  done(result) {
+    this.closeCurrentWriter();
+    // currentHandlers = currentHandlers.filter((x) => x != this);
+    this.resolve();
+  }
+  info(info) {
+    if (info && info.line != null) {
+      info = {
+        ...info,
+        line: this.startLine + info.line,
+      };
+    }
+    process.send({ msgtype: 'info', info });
+  }
+}
+
+function handleStream(driver, resultIndexHolder, sqlItem) {
+  return new Promise((resolve, reject) => {
+    const start = sqlItem.trimStart || sqlItem.start;
+    const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line);
+    driver.stream(dbhan, sqlItem.text, handler);
+  });
+}
+
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
+async function handleConnect(connection) {
+  storedConnection = connection;
+
+  const driver = requireEngineDriver(storedConnection);
+  dbhan = await connectUtility(driver, storedConnection, 'app');
+  for (const [resolve] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+// function handleCancel() {
+//   for (const handler of currentHandlers) {
+//     if (handler.stream) handler.stream.cancel();
+//   }
+// }
+
+function waitConnected() {
+  if (dbhan) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
+}
+
+async function handleStartProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({ msgtype: 'done' });
+    return;
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  currentProfiler = await driver.startProfiler(dbhan, {
+    row: data => writer.rowFromReader(data),
+  });
+  currentProfiler.writer = writer;
+}
+
+async function handleStopProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
+  const driver = requireEngineDriver(storedConnection);
+  currentProfiler.writer.close();
+  driver.stopProfiler(dbhan, currentProfiler);
+  currentProfiler = null;
+}
+
+async function handleExecuteQuery({ sql }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
+  executingScripts++;
+  try {
+    const resultIndexHolder = {
+      value: 0,
+    };
+    for (const sqlItem of splitQuery(sql, {
+      ...driver.getQuerySplitterOptions('stream'),
+      returnRichInfo: true,
+    })) {
+      await handleStream(driver, resultIndexHolder, sqlItem);
+      // const handler = new StreamHandler(resultIndex);
+      // const stream = await driver.stream(systemConnection, sqlItem, handler);
+      // handler.stream = stream;
+      // resultIndex = handler.resultIndex;
+    }
+    process.send({ msgtype: 'done' });
+  } finally {
+    executingScripts--;
+  }
+}
+
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(dbhan, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
+function handlePing() {
+  lastPing = new Date().getTime();
+}
+
+const messageHandlers = {
+  connect: handleConnect,
+  executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
+  startProfiler: handleStartProfiler,
+  stopProfiler: handleStopProfiler,
+  ping: handlePing,
+  // cancel: handleCancel,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+function start() {
+  childProcessChecker();
+
+  lastPing = new Date().getTime();
+
+  setInterval(async () => {
+    const time = new Date().getTime();
+    if (time - lastPing > 25 * 1000) {
+      logger.info('Session not alive, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
+      process.exit(0);
+    }
+
+    const useSessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractBoolSettingsValue(storedConnection.globalSettings, 'session.autoClose', true)
+        : false;
+    const sessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractIntSettingsValue(storedConnection.globalSettings, 'session.autoCloseTimeout', 15, 1, 120)
+        : 15;
+    if (
+      useSessionTimeout &&
+      time - lastActivity > sessionTimeout * 60 * 1000 &&
+      !currentProfiler &&
+      executingScripts == 0
+    ) {
+      logger.info('Session not active, exiting');
+      const driver = requireEngineDriver(storedConnection);
+      await driver.close(dbhan);
+      process.exit(0);
+    }
+  }, 10 * 1000);
+
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
+    try {
+      await handleMessage(message);
+    } catch (e) {
+      process.send({
+        msgtype: 'info',
+        info: {
+          message: e.message,
+          severity: 'error',
+        },
+      });
+      //process.send({ msgtype: 'error', error: e.message });
+    }
+  });
+}
+
+module.exports = { start };

package/src/proc/sshForwardProcess.js

@@ -0,0 +1,75 @@
+const fs = require('fs-extra');
+const platformInfo = require('../utility/platformInfo');
+const childProcessChecker = require('../utility/childProcessChecker');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { SSHConnection } = require('../utility/SSHConnection');
+const { getLogger, extractErrorLogData, extractErrorMessage } = require('dbgate-tools');
+
+const logger = getLogger('sshProcess');
+
+async function getSshConnection(connection) {
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
+    username: connection.sshLogin,
+    password: (connection.sshMode || 'userPassword') == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && (connection.sshKeyfile || platformInfo?.defaultKeyfile)
+        ? await fs.readFile(connection.sshKeyfile || platformInfo?.defaultKeyfile)
+        : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  return sshConn;
+}
+
+async function handleStart({ connection, tunnelConfig }) {
+  try {
+    const sshConn = await getSshConnection(connection);
+    await sshConn.forward(tunnelConfig);
+
+    process.send({
+      msgtype: 'connected',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    logger.error(extractErrorLogData(err), 'Error creating SSH tunnel connection:');
+
+    process.send({
+      msgtype: 'error',
+      connection,
+      tunnelConfig,
+      errorMessage: extractErrorMessage(err.message),
+    });
+  }
+}
+
+const messageHandlers = {
+  connect: handleStart,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+function start() {
+  childProcessChecker();
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
+    try {
+      await handleMessage(message);
+    } catch (e) {
+      console.error('sshForwardProcess - unhandled error', e);
+    }
+  });
+}
+
+module.exports = { start };

package/src/shell/archiveReader.js

@@ -0,0 +1,11 @@
+const path = require('path');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
+const jsonLinesReader = require('./jsonLinesReader');
+
+function archiveReader({ folderName, fileName, ...other }) {
+  const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
+  const res = jsonLinesReader({ fileName: jsonlFile, ...other });
+  return res;
+}
+
+module.exports = archiveReader;

package/src/shell/archiveWriter.js

@@ -0,0 +1,22 @@
+const path = require('path');
+const fs = require('fs');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
+// const socket = require('../utility/socket');
+const jsonLinesWriter = require('./jsonLinesWriter');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger();
+
+function archiveWriter({ folderName, fileName }) {
+  const dir = resolveArchiveFolder(folderName);
+  if (!fs.existsSync(dir)) {
+    logger.info(`Creating directory ${dir}`);
+    fs.mkdirSync(dir);
+  }
+  const jsonlFile = path.join(dir, `${fileName}.jsonl`);
+  const res = jsonLinesWriter({ fileName: jsonlFile });
+  // socket.emitChanged(`archive-files-changed-${folderName}`);
+  return res;
+}
+
+module.exports = archiveWriter;

package/src/shell/autoIndexForeignKeysTransform.js

@@ -0,0 +1,19 @@
+const autoIndexForeignKeysTransform = () => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        indexes: [
+          ...(table.indexes || []),
+          ...table.foreignKeys.map(fk => ({
+            constraintName: `IX_${fk.constraintName}`,
+            columns: fk.columns.map(x => ({ columnName: x.columnName })),
+          })),
+        ],
+      };
+    }),
+  };
+};
+
+module.exports = autoIndexForeignKeysTransform;

package/src/shell/collectorWriter.js

@@ -0,0 +1,33 @@
+const stream = require('stream');
+
+class CollectorWriterStream extends stream.Writable {
+  constructor(options) {
+    super(options);
+    this.rows = [];
+    this.structure = null;
+    this.runid = options.runid;
+  }
+  _write(chunk, enc, next) {
+    if (!this.structure) this.structure = chunk;
+    else this.rows.push(chunk);
+    next();
+  }
+
+  _final(callback) {
+    process.send({
+      msgtype: 'freeData',
+      runid: this.runid,
+      freeData: { rows: this.rows, structure: this.structure },
+    });
+    callback();
+  }
+}
+
+async function collectorWriter({ runid }) {
+  return new CollectorWriterStream({
+    objectMode: true,
+    runid,
+  });
+}
+
+module.exports = collectorWriter;

package/src/shell/consoleObjectWriter.js

@@ -0,0 +1,16 @@
+const stream = require('stream');
+
+class ObjectWriterStream extends stream.Writable {
+  _write(chunk, enc, next) {
+    console.log(JSON.stringify(chunk));
+    next();
+  }
+}
+
+async function consoleObjectWriter() {
+  return new ObjectWriterStream({
+    objectMode: true,
+  });
+}
+
+module.exports = consoleObjectWriter;

package/src/shell/copyStream.js

@@ -0,0 +1,48 @@
+const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+const Stream = require('stream');
+const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+
+function copyStream(input, output, options) {
+  const { columns } = options || {};
+
+  const transforms = [];
+  if (columns) {
+    transforms.push(new ColumnMapTransformStream(columns));
+  }
+  if (output.requireFixedStructure) {
+    transforms.push(new EnsureStreamHeaderStream());
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   Stream.pipeline(input, ...transforms, output, err => {
+  //     if (err) {
+  //       reject(err);
+  //     } else {
+  //       resolve();
+  //     }
+  //   });
+  // });
+
+  return new Promise((resolve, reject) => {
+    const finisher = output['finisher'] || output;
+    finisher.on('finish', resolve);
+    finisher.on('error', reject);
+
+    let lastStream = input;
+    for (const tran of transforms) {
+      lastStream.pipe(tran);
+      lastStream = tran;
+    }
+    lastStream.pipe(output);
+
+    //  if (output.requireFixedStructure) {
+    //   const ensureHeader = new EnsureStreamHeaderStream();
+    //   input.pipe(ensureHeader);
+    //   ensureHeader.pipe(output);
+    // } else {
+    //   input.pipe(output);
+    // }
+  });
+}
+
+module.exports = copyStream;

package/src/shell/dataDuplicator.js

@@ -0,0 +1,63 @@
+const stream = require('stream');
+const path = require('path');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('dataDuplicator');
+const { DataDuplicator } = require('dbgate-datalib');
+const copyStream = require('./copyStream');
+const jsonLinesReader = require('./jsonLinesReader');
+const { resolveArchiveFolder } = require('../utility/directories');
+
+async function dataDuplicator({
+  connection,
+  archive,
+  folder,
+  items,
+  options,
+  analysedStructure = null,
+  driver,
+  systemConnection,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  try {
+    logger.info(`Connected.`);
+
+    if (!analysedStructure) {
+      analysedStructure = await driver.analyseFull(dbhan);
+    }
+
+    const sourceDir = archive
+      ? resolveArchiveFolder(archive)
+      : folder?.startsWith('archive:')
+      ? resolveArchiveFolder(folder.substring('archive:'.length))
+      : folder;
+
+    const dupl = new DataDuplicator(
+      dbhan,
+      driver,
+      analysedStructure,
+      items.map(item => ({
+        name: item.name,
+        operation: item.operation,
+        matchColumns: item.matchColumns,
+        openStream:
+          item.openStream || (() => jsonLinesReader({ fileName: path.join(sourceDir, `${item.name}.jsonl`) })),
+      })),
+      stream,
+      copyStream,
+      options
+    );
+
+    await dupl.run();
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = dataDuplicator;

package/src/shell/dataTypeMapperTransform.js

@@ -0,0 +1,21 @@
+const dataTypeMapperTransform = (oldType, newType) => database => {
+  return {
+    ...database,
+    tables: database.tables.map(table => {
+      return {
+        ...table,
+        columns: table.columns.map(column => {
+          if (column.dataType?.toLowerCase() === oldType?.toLowerCase()) {
+            return {
+              ...column,
+              dataType: newType,
+            };
+          }
+          return column;
+        }),
+      };
+    }),
+  };
+};
+
+module.exports = dataTypeMapperTransform;