dbgate-api-premium 5.5.7-alpha.45

package/src/shell/jsonReader.js

@@ -0,0 +1,84 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const { parser } = require('stream-json');
+const { pick } = require('stream-json/filters/Pick');
+const { streamArray } = require('stream-json/streamers/StreamArray');
+const { streamObject } = require('stream-json/streamers/StreamObject');
+const download = require('./download');
+
+const logger = getLogger('jsonReader');
+
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, jsonStyle, keyField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rowsWritten = 0;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      this.push({
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      });
+
+      this.wasHeader = true;
+    }
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
+      if (this.jsonStyle === 'object') {
+        this.push({
+          ...chunk.value,
+          [this.keyField]: chunk.key,
+        });
+      } else {
+        this.push(chunk.value);
+      }
+
+      this.rowsWritten += 1;
+    }
+    done();
+  }
+}
+
+async function jsonReader({
+  fileName,
+  jsonStyle,
+  keyField = '_key',
+  rootField = null,
+  encoding = 'utf-8',
+  limitRows = undefined,
+}) {
+  logger.info(`Reading file ${fileName}`);
+
+  const downloadedFile = await download(fileName);
+  const fileStream = fs.createReadStream(
+    downloadedFile,
+    // @ts-ignore
+    encoding
+  );
+  const parseJsonStream = parser();
+  fileStream.pipe(parseJsonStream);
+
+  const parseStream = new ParseStream({ limitRows, jsonStyle, keyField });
+
+  const tramsformer = jsonStyle === 'object' ? streamObject() : streamArray();
+
+  if (rootField) {
+    const filterStream = pick({ filter: rootField });
+    parseJsonStream.pipe(filterStream);
+    filterStream.pipe(tramsformer);
+  } else {
+    parseJsonStream.pipe(tramsformer);
+  }
+
+  tramsformer.pipe(parseStream);
+
+  return parseStream;
+}
+
+module.exports = jsonReader;

package/src/shell/jsonToDbModel.js

@@ -0,0 +1,9 @@
+const exportDbModel = require('../utility/exportDbModel');
+const fs = require('fs');
+
+async function jsonToDbModel({ modelFile, outputDir }) {
+  const dbInfo = JSON.parse(fs.readFileSync(modelFile, 'utf-8'));
+  await exportDbModel(dbInfo, outputDir);
+}
+
+module.exports = jsonToDbModel;

package/src/shell/jsonWriter.js

@@ -0,0 +1,97 @@
+const { getLogger } = require('dbgate-tools');
+const fs = require('fs');
+const stream = require('stream');
+const _ = require('lodash');
+
+const logger = getLogger('jsonArrayWriter');
+
+class StringifyStream extends stream.Transform {
+  constructor({ jsonStyle, keyField, rootField }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+    this.jsonStyle = jsonStyle;
+    this.keyField = keyField || '_key';
+    this.rootField = rootField;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip = chunk.__isStreamHeader;
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        if (this.rootField) {
+          if (this.jsonStyle === 'object') {
+            this.push(`{"${this.rootField}": {\n`);
+          } else {
+            this.push(`{"${this.rootField}": [\n`);
+          }
+        } else {
+          if (this.jsonStyle === 'object') {
+            this.push('{\n');
+          } else {
+            this.push('[\n');
+          }
+        }
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      if (this.jsonStyle === 'object') {
+        const key = chunk[this.keyField] ?? chunk[Object.keys(chunk)[0]];
+        this.push(`"${key}": ${JSON.stringify(_.omit(chunk, [this.keyField]))}`);
+      } else {
+        this.push(JSON.stringify(chunk));
+      }
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push(`{"${this.rootField}": {}}\n`);
+        } else {
+          this.push(`{"${this.rootField}": []}\n`);
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('{}\n');
+        } else {
+          this.push('[]\n');
+        }
+      }
+    } else {
+      if (this.rootField) {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}}\n');
+        } else {
+          this.push('\n]}\n');
+        }
+      } else {
+        if (this.jsonStyle === 'object') {
+          this.push('\n}\n');
+        } else {
+          this.push('\n]\n');
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
+  logger.info(`Writing file ${fileName}`);
+  const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonWriter;

package/src/shell/loadDatabase.js

@@ -0,0 +1,27 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const exportDbModel = require('../utility/exportDbModel');
+
+const logger = getLogger('analyseDb');
+
+async function loadDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, outputDir }) {
+  logger.info(`Analysing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  try {
+    logger.info(`Connected.`);
+
+    const dbInfo = await driver.analyseFull(dbhan);
+    logger.info(`Analyse finished`);
+
+    await exportDbModel(dbInfo, outputDir);
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = loadDatabase;

package/src/shell/loadFile.js

@@ -0,0 +1,10 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { filesdir } = require('../utility/directories');
+
+async function loadFile(file) {
+  const text = await fs.readFile(path.join(filesdir(), file), { encoding: 'utf-8' });
+  return text;
+}
+
+module.exports = loadFile;

package/src/shell/modifyJsonLinesReader.js

@@ -0,0 +1,148 @@
+const fs = require('fs');
+const _ = require('lodash');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger, processJsonDataUpdateCommands, removeTablePairingId } = require('dbgate-tools');
+const logger = getLogger('modifyJsonLinesReader');
+const stableStringify = require('json-stable-stringify');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows, changeSet, mergedRows, mergeKey, mergeMode }) {
+    super({ objectMode: true });
+    this.limitRows = limitRows;
+    this.changeSet = changeSet;
+    this.wasHeader = false;
+    this.currentRowIndex = 0;
+    if (mergeMode == 'merge') {
+      if (mergedRows && mergeKey) {
+        this.mergedRowsDict = {};
+        for (const row of mergedRows) {
+          const key = stableStringify(_.pick(row, mergeKey));
+          this.mergedRowsDict[key] = row;
+        }
+      }
+    }
+    this.mergedRowsArray = mergedRows;
+    this.mergeKey = mergeKey;
+    this.mergeMode = mergeMode;
+  }
+  _transform(chunk, encoding, done) {
+    let obj = JSON.parse(chunk);
+    if (obj.__isStreamHeader) {
+      if (this.changeSet && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+      } else {
+        this.push(obj);
+      }
+      this.wasHeader = true;
+      done();
+      return;
+    }
+
+    if (this.changeSet) {
+      if (!this.wasHeader && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+        this.wasHeader = true;
+      }
+
+      if (!this.limitRows || this.currentRowIndex < this.limitRows) {
+        if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
+          obj = null;
+        }
+
+        const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
+        if (update) {
+          if (update.document) {
+            obj = update.document;
+          } else {
+            obj = _.omitBy(
+              {
+                ...obj,
+                ...update.fields,
+              },
+              (v, k) => v?.$$undefined$$
+            );
+          }
+        }
+
+        if (obj) {
+          if (this.changeSet.dataUpdateCommands) {
+            obj = processJsonDataUpdateCommands(obj, this.changeSet.dataUpdateCommands);
+          }
+          this.push(obj);
+        }
+        this.currentRowIndex += 1;
+      }
+    } else if (this.mergedRowsArray && this.mergeKey && this.mergeMode) {
+      if (this.mergeMode == 'merge') {
+        const key = stableStringify(_.pick(obj, this.mergeKey));
+        if (this.mergedRowsDict[key]) {
+          this.push({ ...obj, ...this.mergedRowsDict[key] });
+          delete this.mergedRowsDict[key];
+        } else {
+          this.push(obj);
+        }
+      } else if (this.mergeMode == 'append') {
+        this.push(obj);
+      }
+    } else {
+      this.push(obj);
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (this.changeSet) {
+      for (const insert of this.changeSet.inserts) {
+        this.push({
+          ...insert.document,
+          ...insert.fields,
+        });
+      }
+    } else if (this.mergedRowsArray && this.mergeKey) {
+      if (this.mergeMode == 'merge') {
+        for (const row of this.mergedRowsArray) {
+          const key = stableStringify(_.pick(row, this.mergeKey));
+          if (this.mergedRowsDict[key]) {
+            this.push(row);
+          }
+        }
+      } else {
+        for (const row of this.mergedRowsArray) {
+          this.push(row);
+        }
+      }
+    }
+    done();
+  }
+}
+
+async function modifyJsonLinesReader({
+  fileName,
+  encoding = 'utf-8',
+  limitRows = undefined,
+  changeSet = null,
+  mergedRows = null,
+  mergeKey = null,
+  mergeMode = 'merge',
+}) {
+  logger.info(`Reading file ${fileName} with change set`);
+
+  const fileStream = fs.createReadStream(
+    fileName,
+    // @ts-ignore
+    encoding
+  );
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows, changeSet, mergedRows, mergeKey, mergeMode });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = modifyJsonLinesReader;

package/src/shell/queryReader.js

@@ -0,0 +1,30 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('queryReader');
+
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  logger.info({ sql: query || sql }, `Reading query`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
+
+  const driver = requireEngineDriver(connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
+  logger.info(`Connected.`);
+  const reader =
+    queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
+  return reader;
+}
+
+module.exports = queryReader;

package/src/shell/registerPlugins.js

@@ -0,0 +1,9 @@
+const requirePlugin = require('./requirePlugin');
+
+function registerPlugins(...plugins) {
+  for (const plugin of plugins) {
+    requirePlugin(plugin.packageName, plugin);
+  }
+}
+
+module.exports = registerPlugins;

package/src/shell/requirePlugin.js

@@ -0,0 +1,43 @@
+const path = require('path');
+const fs = require('fs');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
+const platformInfo = require('../utility/platformInfo');
+const authProxy = require('../utility/authProxy');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('requirePlugin');
+
+const loadedPlugins = {};
+
+const dbgateEnv = {
+  dbgateApi: null,
+  platformInfo,
+  authProxy,
+};
+function requirePlugin(packageName, requiredPlugin = null) {
+  if (!packageName) throw new Error('Missing packageName in plugin');
+  if (loadedPlugins[packageName]) return loadedPlugins[packageName];
+
+  if (requiredPlugin == null) {
+    let module;
+    const modulePath = getPluginBackendPath(packageName);
+    logger.info(`Loading module ${packageName} from ${modulePath}`);
+    try {
+      // @ts-ignore
+      module = __non_webpack_require__(modulePath);
+    } catch (err) {
+      // console.log('Failed load webpacked module', err.message);
+      module = require(modulePath);
+    }
+    requiredPlugin = module.__esModule ? module.default : module;
+  }
+  loadedPlugins[packageName] = requiredPlugin;
+  if (requiredPlugin.initialize) requiredPlugin.initialize(dbgateEnv);
+
+  return requiredPlugin;
+}
+
+requirePlugin.initializeDbgateApi = value => {
+  dbgateEnv.dbgateApi = value;
+};
+
+module.exports = requirePlugin;

package/src/shell/runScript.js

@@ -0,0 +1,19 @@
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');
+const logger = getLogger();
+
+async function runScript(func) {
+  if (processArgs.checkParent) {
+    childProcessChecker();
+  }
+  try {
+    await func();
+    process.exit(0);
+  } catch (err) {
+    logger.error(extractErrorLogData(err), `Error running script`);
+    process.exit(1);
+  }
+}
+
+module.exports = runScript;

package/src/shell/sqlDataWriter.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const stream = require('stream');
+const path = require('path');
+const { driverBase, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const logger = getLogger('sqlDataWriter');
+
+class SqlizeStream extends stream.Transform {
+  constructor({ fileName, dataName }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.tableName = path.parse(fileName).name;
+    this.dataName = dataName;
+    this.driver = driverBase;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+    if (!this.wasHeader) {
+      if (chunk.__isStreamHeader) {
+        skip = true;
+        this.tableName = chunk.pureName;
+        if (chunk.engine) {
+          // @ts-ignore
+          this.driver = requireEngineDriver(chunk.engine) || driverBase;
+        }
+      }
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      const dmp = this.driver.createDumper();
+      dmp.put(
+        '^insert ^into %f (%,i) ^values (%,v);\n',
+        { pureName: this.dataName || this.tableName },
+        Object.keys(chunk),
+        Object.values(chunk)
+      );
+      this.push(dmp.s);
+    }
+    done();
+  }
+}
+
+async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) {
+  logger.info(`Writing file ${fileName}`);
+  const stringify = new SqlizeStream({ fileName, dataName });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = sqlDataWriter;

package/src/shell/sqlTextReplacementTransform.js

@@ -0,0 +1,32 @@
+function replaceInText(text, replacements) {
+  let result = text;
+  for (const key of Object.keys(replacements)) {
+    result = result.split(key).join(replacements[key]);
+  }
+  return result;
+}
+
+function replaceInCollection(collection, replacements) {
+  if (!collection) return collection;
+  return collection.map(item => {
+    if (item.createSql) {
+      return {
+        ...item,
+        createSql: replaceInText(item.createSql, replacements),
+      };
+    }
+    return item;
+  });
+}
+
+const sqlTextReplacementTransform = replacements => database => {
+  return {
+    ...database,
+    views: replaceInCollection(database.views, replacements),
+    matviews: replaceInCollection(database.matviews, replacements),
+    procedures: replaceInCollection(database.procedures, replacements),
+    functions: replaceInCollection(database.functions, replacements),
+  };
+};
+
+module.exports = sqlTextReplacementTransform;

package/src/shell/tableReader.js

@@ -0,0 +1,39 @@
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableReader');
+
+async function tableReader({ connection, systemConnection, pureName, schemaName }) {
+  const driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+  logger.info(`Connected.`);
+
+  const fullName = { pureName, schemaName };
+
+  if (driver.databaseEngineTypes.includes('document')) {
+    // @ts-ignore
+    logger.info(`Reading collection ${fullNameToString(fullName)}`);
+    // @ts-ignore
+    return await driver.readQuery(dbhan, JSON.stringify(fullName));
+  }
+
+  const table = await driver.analyseSingleObject(dbhan, fullName, 'tables');
+  const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
+  if (table) {
+    // @ts-ignore
+    logger.info(`Reading table ${fullNameToString(table)}`);
+    // @ts-ignore
+    return await driver.readQuery(dbhan, query, table);
+  }
+  const view = await driver.analyseSingleObject(dbhan, fullName, 'views');
+  if (view) {
+    // @ts-ignore
+    logger.info(`Reading view ${fullNameToString(view)}`);
+    // @ts-ignore
+    return await driver.readQuery(dbhan, query, view);
+  }
+
+  return await driver.readQuery(dbhan, query);
+}
+
+module.exports = tableReader;

package/src/shell/tableWriter.js

@@ -0,0 +1,18 @@
+const { fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('tableWriter');
+
+async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
+  logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`);
+
+  if (!driver) {
+    driver = requireEngineDriver(connection);
+  }
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+  return await driver.writeTable(dbhan, { schemaName, pureName }, options);
+}
+
+module.exports = tableWriter;