dbgate-api-premium 5.5.7-alpha.45

package/src/shell/dbModelToJson.js

@@ -0,0 +1,16 @@
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function dbModelToJson({ modelFolder, outputFile, commonjs }) {
+  const dbInfo = await importDbModel(modelFolder);
+
+  const json = JSON.stringify(dbInfo, null, 2);
+  if (commonjs) {
+    fs.writeFileSync(outputFile, `module.exports = ${json};`);
+    return;
+  } else {
+    fs.writeFileSync(outputFile, json);
+  }
+}
+
+module.exports = dbModelToJson;

package/src/shell/deployDb.js

@@ -0,0 +1,56 @@
+const generateDeploySql = require('./generateDeploySql');
+const executeQuery = require('./executeQuery');
+const { ScriptDrivedDeployer } = require('dbgate-datalib');
+const connectUtility = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const loadModelFolder = require('../utility/loadModelFolder');
+const crypto = require('crypto');
+
+async function deployDb({
+  connection,
+  systemConnection,
+  driver,
+  analysedStructure,
+  modelFolder,
+  loadedDbModel,
+  modelTransforms,
+  dbdiffOptionsExtra,
+  ignoreNameRegex = '',
+  targetSchema = null,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    const scriptDeployer = new ScriptDrivedDeployer(
+      dbhan,
+      driver,
+      Array.isArray(loadedDbModel) ? loadedDbModel : modelFolder ? await loadModelFolder(modelFolder) : [],
+      crypto
+    );
+    await scriptDeployer.runPre();
+
+    const { sql } = await generateDeploySql({
+      connection,
+      systemConnection: dbhan,
+      driver,
+      analysedStructure,
+      modelFolder,
+      loadedDbModel,
+      modelTransforms,
+      dbdiffOptionsExtra,
+      ignoreNameRegex,
+      targetSchema,
+    });
+    // console.log('RUNNING DEPLOY SCRIPT:', sql);
+    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+    await scriptDeployer.runPost();
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = deployDb;

package/src/shell/download.js

@@ -0,0 +1,15 @@
+const crypto = require('crypto');
+const path = require('path');
+const { uploadsdir } = require('../utility/directories');
+const { downloadFile } = require('../utility/downloader');
+
+async function download(url) {
+  if (url && url.match(/(^http:\/\/)|(^https:\/\/)/)) {
+    const tmpFile = path.join(uploadsdir(), crypto.randomUUID());
+    await downloadFile(url, tmpFile);
+    return tmpFile;
+  }
+  return url;
+}
+
+module.exports = download;

package/src/shell/dropAllDbObjects.js

@@ -0,0 +1,42 @@
+const executeQuery = require('./executeQuery');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
+
+const logger = getLogger('dropAllDbObjects');
+
+async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(dbhan);
+  }
+
+  analysedStructure = extendDatabaseInfo(analysedStructure);
+
+  const dmp = driver.createDumper();
+
+  for (const table of analysedStructure.tables) {
+    for (const fk of table.foreignKeys) {
+      dmp.dropForeignKey(fk);
+    }
+  }
+  for (const table of analysedStructure.tables) {
+    dmp.dropTable(table);
+  }
+  for (const field of Object.keys(analysedStructure)) {
+    if (dmp.getSqlObjectSqlName(field)) {
+      for (const obj of analysedStructure[field]) {
+        dmp.dropSqlObject(obj);
+      }
+    }
+  }
+
+  await executeQuery({ connection, systemConnection, driver, sql: dmp.s, logScriptItems: true });
+}
+
+module.exports = dropAllDbObjects;

package/src/shell/dumpDatabase.js

@@ -0,0 +1,49 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('dumpDb');
+
+function doDump(dumper) {
+  return new Promise((resolve, reject) => {
+    dumper.once('end', () => {
+      resolve(true);
+    });
+    dumper.once('error', err => {
+      reject(err);
+    });
+    dumper.run();
+  });
+}
+
+async function dumpDatabase({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  outputFile,
+  databaseName,
+  schemaName,
+}) {
+  logger.info(`Dumping database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+
+  try {
+    logger.info(`Connected.`);
+
+    const dumper = await driver.createBackupDumper(dbhan, {
+      outputFile,
+      databaseName,
+      schemaName,
+    });
+    await doDump(dumper);
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = dumpDatabase;

package/src/shell/executeQuery.js

@@ -0,0 +1,39 @@
+const fs = require('fs-extra');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger, getLimitedQuery } = require('dbgate-tools');
+
+const logger = getLogger('execQuery');
+
+async function executeQuery({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  sql,
+  sqlFile = undefined,
+  logScriptItems = false,
+}) {
+  if (!logScriptItems) {
+    logger.info({ sql: getLimitedQuery(sql) }, `Execute query`);
+  }
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'script'));
+
+  if (sqlFile) {
+    logger.debug(`Loading SQL file ${sqlFile}`);
+    sql = await fs.readFile(sqlFile, { encoding: 'utf-8' });
+  }
+
+  try {
+    logger.info(`Connected.`);
+
+    await driver.script(dbhan, sql, { logScriptItems });
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = executeQuery;

package/src/shell/fakeObjectReader.js

@@ -0,0 +1,35 @@
+const stream = require('stream');
+
+async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  function doWrite() {
+    if (dynamicData) {
+      pass.write({ __isStreamHeader: true, __isDynamicStructure: true });
+      for (const item of dynamicData) {
+        pass.write(item);
+      }
+      pass.end();
+    } else {
+      pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+      pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ id: 2, country: 'Austria' });
+      pass.write({ country: 'Germany', id: 3 });
+      pass.write({ country: 'Romania', id: 4 });
+      pass.write({ country: 'Great Britain', id: 5 });
+      pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+      pass.end();
+    }
+  }
+
+  if (delay) {
+    setTimeout(doWrite, delay);
+  } else {
+    doWrite();
+  }
+
+  return pass;
+}
+
+module.exports = fakeObjectReader;

package/src/shell/finalizer.js

@@ -0,0 +1,12 @@
+const finalizers = [];
+
+module.exports = {
+  async run() {
+    for (const func of finalizers) {
+      await func();
+    }
+  },
+  register(func) {
+    finalizers.push(func);
+  },
+};

package/src/shell/generateDeploySql.js

@@ -0,0 +1,95 @@
+const {
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  databaseInfoFromYamlModel,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
+  skipNamesInStructureByRegex,
+  replaceSchemaInStructure,
+  filterStructureBySchema,
+  skipDbGateInternalObjects,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+async function generateDeploySql({
+  connection,
+  systemConnection = undefined,
+  driver = undefined,
+  analysedStructure = undefined,
+  modelFolder = undefined,
+  loadedDbModel = undefined,
+  modelTransforms = undefined,
+  dbdiffOptionsExtra = {},
+  ignoreNameRegex = '',
+  targetSchema = null,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    if (!analysedStructure) {
+      analysedStructure = await driver.analyseFull(dbhan);
+    }
+
+    if (ignoreNameRegex) {
+      analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
+    }
+    analysedStructure = skipDbGateInternalObjects(analysedStructure);
+
+    let deployedModelSource = loadedDbModel
+      ? databaseInfoFromYamlModel(loadedDbModel)
+      : await importDbModel(modelFolder);
+
+    for (const transform of modelTransforms || []) {
+      deployedModelSource = transform(deployedModelSource);
+    }
+
+    if (targetSchema) {
+      deployedModelSource = replaceSchemaInStructure(deployedModelSource, targetSchema);
+      analysedStructure = filterStructureBySchema(analysedStructure, targetSchema);
+    }
+
+    const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
+    const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
+    const opts = {
+      ...modelCompareDbDiffOptions,
+
+      noDropTable: true,
+      noDropColumn: true,
+      noDropConstraint: true,
+      noDropSqlObject: true,
+      noRenameTable: true,
+      noRenameColumn: true,
+
+      ...dbdiffOptionsExtra,
+    };
+    const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+    const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
+
+    // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
+    // console.log('deployedModel', deployedModel.tables[0]);
+    // console.log('currentModel', currentModel.tables[0]);
+    // console.log('currentModelPaired', currentModelPaired.tables[0]);
+    const res = getAlterDatabaseScript(
+      currentModelPairedPreloaded,
+      deployedModel,
+      opts,
+      currentModelPairedPreloaded,
+      deployedModel,
+      driver
+    );
+
+    return res;
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = generateDeploySql;

package/src/shell/generateModelSql.js

@@ -0,0 +1,30 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const {
+  extendDatabaseInfo,
+  databaseInfoFromYamlModel,
+  getAlterDatabaseScript,
+  DatabaseAnalyser,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function generateModelSql({ engine, driver, modelFolder, loadedDbModel, outputFile }) {
+  if (!driver) driver = requireEngineDriver(engine);
+
+  const dbInfo = extendDatabaseInfo(
+    loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder)
+  );
+
+  const { sql } = getAlterDatabaseScript(
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    {},
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    driver
+  );
+
+  fs.writeFileSync(outputFile, sql);
+}
+
+module.exports = generateModelSql;

package/src/shell/importDatabase.js

@@ -0,0 +1,85 @@
+const fs = require('fs');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
+const download = require('./download');
+const stream = require('stream');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('importDb');
+
+class ImportStream extends stream.Transform {
+  constructor(dbhan, driver) {
+    super({ objectMode: true });
+    this.dbhan = dbhan;
+    this.driver = driver;
+    this.writeQueryStream = null;
+  }
+  async _transform(chunk, encoding, cb) {
+    try {
+      if (chunk.specialMarker == 'copy_stdin_start') {
+        this.writeQueryStream = await this.driver.writeQueryFromStream(this.dbhan, chunk.text);
+      } else if (chunk.specialMarker == 'copy_stdin_line') {
+        this.writeQueryStream.write(chunk.text);
+      } else if (chunk.specialMarker == 'copy_stdin_end') {
+        this.writeQueryStream.end();
+        await new Promise((resolve, reject) => {
+          this.writeQueryStream.on('finish', resolve);
+          this.writeQueryStream.on('error', reject);
+        });
+        this.writeQueryStream = null;
+      } else {
+        await this.driver.script(this.dbhan, chunk.text, { queryOptions: { importSqlDump: true } });
+      }
+    } catch (err) {
+      this.emit('error', err.message);
+    }
+    cb();
+  }
+  _flush(cb) {
+    this.push('finish');
+    cb();
+    this.emit('end');
+  }
+}
+
+function awaitStreamEnd(stream) {
+  return new Promise((resolve, reject) => {
+    stream.once('end', () => {
+      resolve(true);
+    });
+    stream.once('error', err => {
+      reject(err);
+    });
+  });
+}
+
+async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
+  logger.info(`Importing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+  try {
+    logger.info(`Connected.`);
+
+    logger.info(`Input file: ${inputFile}`);
+    const downloadedFile = await download(inputFile);
+    logger.info(`Downloaded file: ${downloadedFile}`);
+
+    const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
+    const splittedStream = splitQueryStream(fileStream, {
+      ...driver.getQuerySplitterOptions('import'),
+      returnRichInfo: true,
+    });
+    const importStream = new ImportStream(dbhan, driver);
+    // @ts-ignore
+    splittedStream.pipe(importStream);
+    await awaitStreamEnd(importStream);
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
+}
+
+module.exports = importDatabase;

package/src/shell/index.js

@@ -0,0 +1,80 @@
+const queryReader = require('./queryReader');
+const runScript = require('./runScript');
+const tableWriter = require('./tableWriter');
+const tableReader = require('./tableReader');
+const copyStream = require('./copyStream');
+const fakeObjectReader = require('./fakeObjectReader');
+const consoleObjectWriter = require('./consoleObjectWriter');
+const jsonLinesWriter = require('./jsonLinesWriter');
+const jsonWriter = require('./jsonWriter');
+const jsonLinesReader = require('./jsonLinesReader');
+const sqlDataWriter = require('./sqlDataWriter');
+const jslDataReader = require('./jslDataReader');
+const archiveWriter = require('./archiveWriter');
+const archiveReader = require('./archiveReader');
+const collectorWriter = require('./collectorWriter');
+const finalizer = require('./finalizer');
+const registerPlugins = require('./registerPlugins');
+const requirePlugin = require('./requirePlugin');
+const download = require('./download');
+const executeQuery = require('./executeQuery');
+const loadFile = require('./loadFile');
+const deployDb = require('./deployDb');
+const initializeApiEnvironment = require('./initializeApiEnvironment');
+const dumpDatabase = require('./dumpDatabase');
+const importDatabase = require('./importDatabase');
+const loadDatabase = require('./loadDatabase');
+const generateModelSql = require('./generateModelSql');
+const modifyJsonLinesReader = require('./modifyJsonLinesReader');
+const dataDuplicator = require('./dataDuplicator');
+const dbModelToJson = require('./dbModelToJson');
+const jsonToDbModel = require('./jsonToDbModel');
+const jsonReader = require('./jsonReader');
+const dataTypeMapperTransform = require('./dataTypeMapperTransform');
+const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
+const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
+const generateDeploySql = require('./generateDeploySql');
+const dropAllDbObjects = require('./dropAllDbObjects');
+
+const dbgateApi = {
+  queryReader,
+  runScript,
+  tableWriter,
+  tableReader,
+  copyStream,
+  jsonLinesWriter,
+  jsonLinesReader,
+  jsonReader,
+  jsonWriter,
+  sqlDataWriter,
+  fakeObjectReader,
+  consoleObjectWriter,
+  jslDataReader,
+  archiveWriter,
+  archiveReader,
+  collectorWriter,
+  finalizer,
+  download,
+  registerPlugins,
+  executeQuery,
+  loadFile,
+  deployDb,
+  initializeApiEnvironment,
+  dumpDatabase,
+  importDatabase,
+  loadDatabase,
+  generateModelSql,
+  modifyJsonLinesReader,
+  dataDuplicator,
+  dbModelToJson,
+  jsonToDbModel,
+  dataTypeMapperTransform,
+  sqlTextReplacementTransform,
+  autoIndexForeignKeysTransform,
+  generateDeploySql,
+  dropAllDbObjects,
+};
+
+requirePlugin.initializeDbgateApi(dbgateApi);
+
+module.exports = dbgateApi;

package/src/shell/initializeApiEnvironment.js

@@ -0,0 +1,9 @@
+const { handleProcessCommunication } = require('../utility/processComm');
+
+async function initializeApiEnvironment() {
+  process.on('message', async message => {
+    handleProcessCommunication(message);
+  });
+}
+
+module.exports = initializeApiEnvironment;

package/src/shell/jslDataReader.js

@@ -0,0 +1,9 @@
+const getJslFileName = require('../utility/getJslFileName');
+const jsonLinesReader = require('./jsonLinesReader');
+
+function jslDataReader({ jslid, ...other }) {
+  const fileName = getJslFileName(jslid);
+  return jsonLinesReader({ fileName, ...other });
+}
+
+module.exports = jslDataReader;

package/src/shell/jsonLinesReader.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const stream = require('stream');
+const byline = require('byline');
+const { getLogger } = require('dbgate-tools');
+const download = require('./download');
+const logger = getLogger('jsonLinesReader');
+
+class ParseStream extends stream.Transform {
+  constructor({ limitRows }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.rowsWritten = 0;
+  }
+  _transform(chunk, encoding, done) {
+    const obj = JSON.parse(chunk);
+    if (!this.wasHeader) {
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
+      this.push(obj);
+      this.rowsWritten += 1;
+    }
+    done();
+  }
+}
+
+async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
+  logger.info(`Reading file ${fileName}`);
+
+  const downloadedFile = await download(fileName);
+
+  const fileStream = fs.createReadStream(
+    downloadedFile,
+    // @ts-ignore
+    encoding
+  );
+  const liner = byline(fileStream);
+  const parser = new ParseStream({ limitRows });
+  liner.pipe(parser);
+  return parser;
+}
+
+module.exports = jsonLinesReader;

package/src/shell/jsonLinesWriter.js

@@ -0,0 +1,36 @@
+const { getLogger } = require('dbgate-tools');
+const fs = require('fs');
+const stream = require('stream');
+const logger = getLogger('jsonLinesWriter');
+
+class StringifyStream extends stream.Transform {
+  constructor({ header }) {
+    super({ objectMode: true });
+    this.header = header;
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+    if (!this.wasHeader) {
+      skip =
+        (chunk.__isStreamHeader && !this.header) ||
+        (chunk.__isStreamHeader && chunk.__isDynamicStructure && !chunk.__keepDynamicStreamHeader);
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      this.push(JSON.stringify(chunk) + '\n');
+    }
+    done();
+  }
+}
+
+async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
+  logger.info(`Writing file ${fileName}`);
+  const stringify = new StringifyStream({ header });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonLinesWriter;