dbgate-api-premium 5.5.7-alpha.52 → 5.5.7-alpha.60

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api-premium",
   "main": "src/index.js",
-  "version": "5.5.7-alpha.52",
+  "version": "5.5.7-alpha.60",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -25,15 +25,14 @@
     "async-lock": "^1.2.6",
     "axios": "^0.21.1",
     "body-parser": "^1.19.0",
-    "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.5.7-alpha.52",
+    "dbgate-datalib": "^5.5.7-alpha.60",
     "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.5.7-alpha.52",
-    "dbgate-tools": "^5.5.7-alpha.52",
+    "dbgate-sqltree": "^5.5.7-alpha.60",
+    "dbgate-tools": "^5.5.7-alpha.60",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -76,13 +75,15 @@
     "start:storage:built": "env-cmd -f env/storage/.env cross-env DEVMODE= BUILTWEBMODE=1 node dist/bundle.js --listen-api",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "ts": "tsc",
-    "build": "webpack"
+    "build": "webpack",
+    "build:doc": "jsdoc2md --template doctpl.hbs  ./src/shell/* > ../../../dbgate.github.io/_docs/apidoc.md"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.5.7-alpha.52",
+    "dbgate-types": "^5.5.7-alpha.60",
     "env-cmd": "^10.1.0",
+    "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
     "typescript": "^4.4.3",

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.5.7-alpha.52',
-  buildTime: '2024-11-15T10:52:57.687Z'
+  version: '5.5.7-alpha.60',
+  buildTime: '2024-11-20T09:21:10.849Z'
 };

package/src/shell/copyStream.js

@@ -2,6 +2,13 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
 const Stream = require('stream');
 const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
 
+/**
+ * Copies reader to writer. Used for import, export tables and transfer data between tables
+ * @param {readerType} input - reader object
+ * @param {writerType} output - writer object
+ * @param {object} options - options
+ * @returns {Promise}
+ */
 function copyStream(input, output, options) {
   const { columns } = options || {};
 

package/src/shell/deployDb.js

@@ -6,6 +6,21 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const loadModelFolder = require('../utility/loadModelFolder');
 const crypto = require('crypto');
 
+/**
+ * Deploys database model stored in modelFolder (table as yamls) to database
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
+ * @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
+ * @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
+ * @param {function[]} options.modelTransforms - array of functions for transforming model
+ * @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
+ * @param {string} options.ignoreNameRegex - regex for ignoring objects by name
+ * @param {string} options.targetSchema - target schema for deployment
+ * @param {number} options.maxMissingTablesRatio - maximum ratio of missing tables in database. Safety check, if missing ratio is highe, deploy is stopped (preventing accidental drop of all tables)
+ */
 async function deployDb({
   connection,
   systemConnection,
@@ -17,6 +32,7 @@ async function deployDb({
   dbdiffOptionsExtra,
   ignoreNameRegex = '',
   targetSchema = null,
+  maxMissingTablesRatio = undefined,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
@@ -41,6 +57,7 @@ async function deployDb({
       dbdiffOptionsExtra,
       ignoreNameRegex,
       targetSchema,
+      maxMissingTablesRatio,
     });
     // console.log('RUNNING DEPLOY SCRIPT:', sql);
     await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });

package/src/shell/dropAllDbObjects.js

@@ -5,6 +5,15 @@ const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
 
 const logger = getLogger('dropAllDbObjects');
 
+/**
+ * Drops all database objects
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
+ * @returns {Promise}
+ */
 async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
   if (!driver) driver = requireEngineDriver(connection);
 

package/src/shell/executeQuery.js

@@ -5,6 +5,16 @@ const { getLogger, getLimitedQuery } = require('dbgate-tools');
 
 const logger = getLogger('execQuery');
 
+/**
+ * Executes SQL query
+ * @param {object} options
+ * @param {connectionType} [options.connection] - connection object
+ * @param {object} [options.systemConnection] - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} [options.driver] - driver object. If not provided, it will be loaded from connection
+ * @param {string} [options.sql] - SQL query
+ * @param {string} [options.sqlFile] - SQL file
+ * @param {boolean} [options.logScriptItems] - whether to log script items instead of whole script
+ */
 async function executeQuery({
   connection = undefined,
   systemConnection = undefined,

package/src/shell/generateDeploySql.js

@@ -15,6 +15,21 @@ const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 
+/**
+ * Generates query for deploying model into database
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
+ * @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
+ * @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
+ * @param {function[]} options.modelTransforms - array of functions for transforming model
+ * @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
+ * @param {string} options.ignoreNameRegex - regex for ignoring objects by name
+ * @param {string} options.targetSchema - target schema for deployment
+ * @param {number} options.maxMissingTablesRatio - maximum ratio of missing tables in database. Safety check, if missing ratio is highe, deploy is stopped (preventing accidental drop of all tables)
+ */
 async function generateDeploySql({
   connection,
   systemConnection = undefined,
@@ -26,25 +41,35 @@ async function generateDeploySql({
   dbdiffOptionsExtra = {},
   ignoreNameRegex = '',
   targetSchema = null,
+  maxMissingTablesRatio = undefined,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+  if (
+    driver?.dialect?.multipleSchema &&
+    !targetSchema &&
+    dbdiffOptionsExtra?.['schemaMode'] !== 'ignore' &&
+    dbdiffOptionsExtra?.['schemaMode'] !== 'ignoreImplicit'
+  ) {
+    throw new Error('targetSchema is required for databases with multiple schemas');
+  }
 
   try {
     if (!analysedStructure) {
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
+    let deployedModelSource = loadedDbModel
+      ? databaseInfoFromYamlModel(loadedDbModel)
+      : await importDbModel(modelFolder);
+
     if (ignoreNameRegex) {
       analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
+      deployedModelSource = skipNamesInStructureByRegex(deployedModelSource, new RegExp(ignoreNameRegex, 'i'));
     }
     analysedStructure = skipDbGateInternalObjects(analysedStructure);
 
-    let deployedModelSource = loadedDbModel
-      ? databaseInfoFromYamlModel(loadedDbModel)
-      : await importDbModel(modelFolder);
-
     for (const transform of modelTransforms || []) {
       deployedModelSource = transform(deployedModelSource);
     }
@@ -71,6 +96,17 @@ async function generateDeploySql({
     const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
     const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
 
+    if (maxMissingTablesRatio != null) {
+      const missingTables = currentModelPaired.tables.filter(
+        x => !deployedModel.tables.find(y => y.pairingId == x.pairingId)
+      );
+      const missingTableCount = missingTables.length;
+      const missingTablesRatio = missingTableCount / (currentModelPaired.tables.length || 1);
+      if (missingTablesRatio > maxMissingTablesRatio) {
+        throw new Error(`Too many missing tables (${missingTablesRatio * 100}%), aborting deploy`);
+      }
+    }
+
     // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
     // console.log('deployedModel', deployedModel.tables[0]);
     // console.log('currentModel', currentModel.tables[0]);

package/src/shell/jsonLinesReader.js

@@ -33,6 +33,14 @@ class ParseStream extends stream.Transform {
   }
 }
 
+/**
+ * Reader function, which reads JSNOL file or URL. JSONL format - text file, every line is JSON encoded row.
+ * @param {Object} options
+ * @param {string} options.fileName - file name or URL
+ * @param {string} options.encoding - encoding of the file
+ * @param {number} options.limitRows - maximum number of rows to read
+ * @returns {Promise<readerType>} - reader object
+ */
 async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   logger.info(`Reading file ${fileName}`);
 

package/src/shell/jsonLinesWriter.js

@@ -24,6 +24,14 @@ class StringifyStream extends stream.Transform {
   }
 }
 
+/**
+ * Returns writer object for {@link copyStream} function. This writer object writes data to JSONL file. JSONL format - text file, every line is JSON encoded row, used eg. by MongoDB.
+ * @param {object} options
+ * @param {string} options.fileName - file name
+ * @param {string} [options.encoding] - encoding of the file
+ * @param {boolean} [options.header] - whether to write header. Header is JSON describing source table structure. Header is specific to DbGate, if you want eg. to import data to MongoDB, you should not write header.
+ * @returns {Promise<writerType>} - writer object
+ */
 async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
   logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ header });

package/src/shell/jsonReader.js

@@ -45,6 +45,17 @@ class ParseStream extends stream.Transform {
   }
 }
 
+/**
+ * Creates reader object for JSON file for {@link copyStream} function.
+ * @param {object} options
+ * @param {string} options.fileName - file name or URL
+ * @param {string} options.jsonStyle - 'object' or 'array'
+ * @param {string} [options.keyField] - key field for object style
+ * @param {string} [options.rootField] - root field for object style
+ * @param {string} [options.encoding] - encoding of the file
+ * @param {number} [options.limitRows] - maximum number of rows to read
+ * @returns {Promise<readerType>} - reader object
+ */
 async function jsonReader({
   fileName,
   jsonStyle,

package/src/shell/jsonWriter.js

@@ -85,6 +85,16 @@ class StringifyStream extends stream.Transform {
   }
 }
 
+/**
+ * Returns writer object for {@link copyStream} function. This writer object writes data to JSON file.
+ * @param {object} options
+ * @param {string} options.fileName - file name
+ * @param {string} [options.jsonStyle] - 'object' or 'array'
+ * @param {string} [options.keyField] - key field for object style
+ * @param {string} [options.rootField] - root field for object style
+ * @param {string} [options.encoding] - encoding of the file
+ * @returns {Promise<writerType>} - writer object
+ */
 async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
   logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream({ jsonStyle, keyField, rootField });

package/src/shell/queryReader.js

@@ -3,6 +3,15 @@ const connectUtility = require('../utility/connectUtility');
 const { getLogger } = require('dbgate-tools');
 const logger = getLogger('queryReader');
 
+/**
+ * Returns reader object for {@link copyStream} function. This reader object reads data from query.
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {string} options.query - SQL query
+ * @param {string} [options.queryType] - query type
+ * @param {string} [options.sql] - SQL query. obsolete; use query instead
+ * @returns {Promise<readerType>} - reader object
+ */
 async function queryReader({
   connection,
   query,

package/src/shell/tableReader.js

@@ -3,6 +3,15 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const logger = getLogger('tableReader');
 
+/**
+ * Creates reader object for {@link copyStream} function. This reader object reads data from table or view.
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {string} options.pureName - table name
+ * @param {string} options.schemaName - schema name
+ * @returns {Promise<readerType>} - reader object
+ */
 async function tableReader({ connection, systemConnection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));

package/src/shell/tableWriter.js

@@ -3,6 +3,20 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const logger = getLogger('tableWriter');
 
+/**
+ * Creates writer object for {@link copyStream} function. This writer object writes data to table. Table could be created if not exists.
+ * @param {object} options
+ * @param {connectionType} options.connection - connection object
+ * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
+ * @param {string} options.pureName - table name
+ * @param {string} options.schemaName - schema name
+ * @param {boolean} options.dropIfExists - drop table if exists
+ * @param {boolean} options.truncate - truncate table before insert
+ * @param {boolean} options.createIfNotExists - create table if not exists
+ * @param {boolean} options.commitAfterInsert - commit transaction after insert
+ * @returns {Promise<writerType>} - writer object
+ */
 async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
   logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`);
 

package/src/shell/types.js

@@ -0,0 +1,27 @@
+/**
+ * Reader (input) object for {@link copyStream} function
+ * @typedef {Object} readerType
+ *
+ */
+
+/**
+ * Writer (output) object for {@link copyStream} function
+ * @typedef {Object} writerType
+ *
+ */
+
+/**
+ * Typ uživatelské role.
+ * @typedef {('mysql@dbgate-plugin-mysql' | 'mariadb@dbgate-plugin-mysql' | 'postgres@dbgate-plugin-postgres'
+ *      |'sqlite@dbgate-plugin-sqlite' | 'oracle@dbgate-plugin-oracle' | 'cockroach@dbgate-plugin-postgres' | 'redshift@dbgate-plugin-postgres')} engineType
+ */
+
+/**
+ * @typedef {Object} connectionType
+ * @property {engineType} engine
+ * @property {string} server
+ * @property {string} user
+ * @property {string} password
+ * @property {string} database
+ * @property {string} port
+ */

package/src/utility/SSHConnection.js

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-const { Client } = require('ssh2');
 const net = require('net');
 const fs = require('fs');
 const os = require('os');
@@ -147,6 +146,7 @@ class SSHConnection {
   }
 
   async connect(host, stream) {
+    const { Client } = require('ssh2');
     this.debug('Connecting to "%s"', host);
     const connection = new Client();
     return new Promise(async (resolve, reject) => {

package/src/utility/directories.js

@@ -110,7 +110,12 @@ function getPluginBackendPath(packageName) {
     return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
   }
 
-  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+  const res = path.join(pluginsdir(), packageName, 'dist', 'backend.js')
+  if (fs.existsSync(res)) {
+    return res;
+  }
+
+  return require.resolve(packageName);
 }
 
 let archiveLinksCache = {};