dbgate-api 5.5.7-alpha.16 → 5.5.7-alpha.25

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.5.7-alpha.16",
+  "version": "5.5.7-alpha.25",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -27,10 +27,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.5.7-alpha.16",
+    "dbgate-datalib": "^5.5.7-alpha.25",
     "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.5.7-alpha.16",
-    "dbgate-tools": "^5.5.7-alpha.16",
+    "dbgate-sqltree": "^5.5.7-alpha.25",
+    "dbgate-tools": "^5.5.7-alpha.25",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -78,7 +78,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.5.7-alpha.16",
+    "dbgate-types": "^5.5.7-alpha.25",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/config.js

@@ -69,7 +69,7 @@ module.exports = {
       !adminConfig?.adminPasswordState
     );
 
-    return {
+    const configResult = {
       runAsPortal: !!connections.portalConnections,
       singleDbConnection: connections.singleDbConnection,
       singleConnection: singleConnection,
@@ -95,13 +95,15 @@ module.exports = {
         !process.env.BASIC_AUTH
       ),
       isAdminPasswordMissing,
-      isInvalidToken: req.isInvalidToken,
+      isInvalidToken: req?.isInvalidToken,
       adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
       connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
+
+    return configResult;
   },
 
   logout_meta: {

package/src/controllers/connections.js

@@ -368,6 +368,11 @@ module.exports = {
 
   get_meta: true,
   async get({ conid }, req) {
+    if (conid == '__model') {
+      return {
+        _id: '__model',
+      };
+    }
     testConnectionPermission(conid, req);
     return this.getCore({ conid, mask: true });
   },

package/src/controllers/databaseConnections.js

@@ -13,6 +13,7 @@ const {
   modelCompareDbDiffOptions,
   getLogger,
   extractErrorLogData,
+  filterStructureBySchema,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -31,6 +32,8 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const crypto = require('crypto');
+const loadModelTransform = require('../utility/loadModelTransform');
+const exportDbModelSql = require('../utility/exportDbModelSql');
 
 const logger = getLogger('databaseConnections');
 
@@ -349,6 +352,11 @@ module.exports = {
 
   syncModel_meta: true,
   async syncModel({ conid, database, isFullRefresh }, req) {
+    if (conid == '__model') {
+      socket.emitChanged('database-structure-changed', { conid, database });
+      return { status: 'ok' };
+    }
+
     testConnectionPermission(conid, req);
     const conn = await this.ensureOpened(conid, database);
     conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
@@ -392,11 +400,12 @@ module.exports = {
   },
 
   structure_meta: true,
-  async structure({ conid, database }, req) {
+  async structure({ conid, database, modelTransFile = null }, req) {
     testConnectionPermission(conid, req);
     if (conid == '__model') {
       const model = await importDbModel(database);
-      return model;
+      const trans = await loadModelTransform(modelTransFile);
+      return trans ? trans(model) : model;
     }
 
     const opened = await this.ensureOpened(conid, database);
@@ -432,14 +441,35 @@ module.exports = {
   },
 
   exportModel_meta: true,
-  async exportModel({ conid, database }, req) {
+  async exportModel({ conid, database, outputFolder, schema }, req) {
+    testConnectionPermission(conid, req);
+
+    const realFolder = outputFolder.startsWith('archive:')
+      ? resolveArchiveFolder(outputFolder.substring('archive:'.length))
+      : outputFolder;
+
+    const model = await this.structure({ conid, database });
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModel(extendDatabaseInfo(filteredModel), realFolder);
+
+    if (outputFolder.startsWith('archive:')) {
+      socket.emitChanged(`archive-files-changed`, { folder: outputFolder.substring('archive:'.length) });
+    }
+    return { status: 'ok' };
+  },
+
+  exportModelSql_meta: true,
+  async exportModelSql({ conid, database, outputFolder, outputFile, schema }, req) {
     testConnectionPermission(conid, req);
-    const archiveFolder = await archive.getNewArchiveFolder({ database });
-    await fs.mkdir(path.join(archivedir(), archiveFolder));
+
+    const connection = await connections.getCore({ conid });
+    const driver = requireEngineDriver(connection);
+
     const model = await this.structure({ conid, database });
-    await exportDbModel(model, path.join(archivedir(), archiveFolder));
-    socket.emitChanged(`archive-folders-changed`);
-    return { archiveFolder };
+    const filteredModel = schema ? filterStructureBySchema(model, schema) : model;
+    await exportDbModelSql(extendDatabaseInfo(filteredModel), driver, outputFolder, outputFile);
+
+    return { status: 'ok' };
   },
 
   generateDeploySql_meta: true,

package/src/controllers/runners.js

@@ -12,6 +12,7 @@ const {
   jsonScriptToJavascript,
   getLogger,
   safeJsonParse,
+  pinoLogRecordToMessageRecord,
 } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
@@ -68,18 +69,20 @@ module.exports = {
 
   dispatchMessage(runid, message) {
     if (message) {
-      const json = safeJsonParse(message.message);
-
-      if (json) logger.log(json);
-      else logger.info(message.message);
-
-      const toEmit = {
-        time: new Date(),
-        ...message,
-        message: json ? json.msg : message.message,
-      };
-
-      if (json && json.level >= 50) {
+      if (_.isPlainObject(message)) logger.log(message);
+      else logger.info(message);
+
+      const toEmit = _.isPlainObject(message)
+        ? {
+            time: new Date(),
+            ...message,
+          }
+        : {
+            message,
+            time: new Date(),
+          };
+
+      if (toEmit.level >= 50) {
         toEmit.severity = 'error';
       }
 
@@ -131,7 +134,16 @@ module.exports = {
       }
     );
     const pipeDispatcher = severity => data => {
-      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+      const json = safeJsonParse(data, null);
+
+      if (json) {
+        return this.dispatchMessage(runid, pinoLogRecordToMessageRecord(json));
+      } else {
+        return this.dispatchMessage(runid, {
+          message: json == null ? data.toString().trim() : null,
+          severity,
+        });
+      }
     };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
@@ -165,7 +177,7 @@ module.exports = {
 
   start_meta: true,
   async start({ script }) {
-    const runid = crypto.randomUUID()
+    const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
       const js = jsonScriptToJavascript(script);

package/src/controllers/serverConnections.js

@@ -134,6 +134,7 @@ module.exports = {
   listDatabases_meta: true,
   async listDatabases({ conid }, req) {
     if (!conid) return [];
+    if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
@@ -172,7 +173,7 @@ module.exports = {
         }
       })
     );
-    socket.setStreamIdFilter(strmid, { conid: conidArray });
+    socket.setStreamIdFilter(strmid, { conid: [...(conidArray ?? []), '__model'] });
     return { status: 'ok' };
   },
 

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.5.7-alpha.16',
-  buildTime: '2024-11-01T11:41:22.229Z'
+  version: '5.5.7-alpha.25',
+  buildTime: '2024-11-12T07:05:33.638Z'
 };

package/src/shell/autoIndexForeignKeysTransform.js

@@ -8,7 +8,7 @@ const autoIndexForeignKeysTransform = () => database => {
           ...(table.indexes || []),
           ...table.foreignKeys.map(fk => ({
             constraintName: `IX_${fk.constraintName}`,
-            columns: fk.columns,
+            columns: fk.columns.map(x => ({ columnName: x.columnName })),
           })),
         ],
       };

package/src/shell/deployDb.js

@@ -1,5 +1,9 @@
 const generateDeploySql = require('./generateDeploySql');
 const executeQuery = require('./executeQuery');
+const { ScriptDrivedDeployer } = require('dbgate-datalib');
+const connectUtility = require('../utility/connectUtility');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const loadModelFolder = require('../utility/loadModelFolder');
 
 async function deployDb({
   connection,
@@ -10,19 +14,41 @@ async function deployDb({
   loadedDbModel,
   modelTransforms,
   dbdiffOptionsExtra,
+  ignoreNameRegex = '',
+  targetSchema = null,
 }) {
-  const { sql } = await generateDeploySql({
-    connection,
-    systemConnection,
-    driver,
-    analysedStructure,
-    modelFolder,
-    loadedDbModel,
-    modelTransforms,
-    dbdiffOptionsExtra,
-  });
-  // console.log('RUNNING DEPLOY SCRIPT:', sql);
-  await executeQuery({ connection, systemConnection, driver, sql, logScriptItems: true });
+  if (!driver) driver = requireEngineDriver(connection);
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
+
+  try {
+    const scriptDeployer = new ScriptDrivedDeployer(
+      dbhan,
+      driver,
+      loadedDbModel ?? (await loadModelFolder(modelFolder))
+    );
+    await scriptDeployer.runPre();
+
+    const { sql } = await generateDeploySql({
+      connection,
+      systemConnection: dbhan,
+      driver,
+      analysedStructure,
+      modelFolder,
+      loadedDbModel,
+      modelTransforms,
+      dbdiffOptionsExtra,
+      ignoreNameRegex,
+      targetSchema,
+    });
+    // console.log('RUNNING DEPLOY SCRIPT:', sql);
+    await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
+
+    await scriptDeployer.runPost();
+  } finally {
+    if (!systemConnection) {
+      await driver.close(dbhan);
+    }
+  }
 }
 
 module.exports = deployDb;

package/src/shell/dropAllDbObjects.js

@@ -0,0 +1,42 @@
+const executeQuery = require('./executeQuery');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
+
+const logger = getLogger('dropAllDbObjects');
+
+async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const dbhan = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(dbhan);
+  }
+
+  analysedStructure = extendDatabaseInfo(analysedStructure);
+
+  const dmp = driver.createDumper();
+
+  for (const table of analysedStructure.tables) {
+    for (const fk of table.foreignKeys) {
+      dmp.dropForeignKey(fk);
+    }
+  }
+  for (const table of analysedStructure.tables) {
+    dmp.dropTable(table);
+  }
+  for (const field of Object.keys(analysedStructure)) {
+    if (dmp.getSqlObjectSqlName(field)) {
+      for (const obj of analysedStructure[field]) {
+        dmp.dropSqlObject(obj);
+      }
+    }
+  }
+
+  await executeQuery({ connection, systemConnection, driver, sql: dmp.s, logScriptItems: true });
+}
+
+module.exports = dropAllDbObjects;

package/src/shell/generateDeploySql.js

@@ -6,6 +6,10 @@ const {
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
   enrichWithPreloadedRows,
+  skipNamesInStructureByRegex,
+  replaceSchemaInStructure,
+  filterStructureBySchema,
+  skipDbGateInternalObjects,
 } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -20,6 +24,8 @@ async function generateDeploySql({
   loadedDbModel = undefined,
   modelTransforms = undefined,
   dbdiffOptionsExtra = {},
+  ignoreNameRegex = '',
+  targetSchema = null,
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
@@ -30,6 +36,11 @@ async function generateDeploySql({
       analysedStructure = await driver.analyseFull(dbhan);
     }
 
+    if (ignoreNameRegex) {
+      analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
+    }
+    analysedStructure = skipDbGateInternalObjects(analysedStructure);
+
     let deployedModelSource = loadedDbModel
       ? databaseInfoFromYamlModel(loadedDbModel)
       : await importDbModel(modelFolder);
@@ -38,6 +49,11 @@ async function generateDeploySql({
       deployedModelSource = transform(deployedModelSource);
     }
 
+    if (targetSchema) {
+      deployedModelSource = replaceSchemaInStructure(deployedModelSource, targetSchema);
+      analysedStructure = filterStructureBySchema(analysedStructure, targetSchema);
+    }
+
     const deployedModel = generateDbPairingId(extendDatabaseInfo(deployedModelSource));
     const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
     const opts = {

package/src/shell/index.js

@@ -34,6 +34,8 @@ const dataTypeMapperTransform = require('./dataTypeMapperTransform');
 const sqlTextReplacementTransform = require('./sqlTextReplacementTransform');
 const autoIndexForeignKeysTransform = require('./autoIndexForeignKeysTransform');
 const generateDeploySql = require('./generateDeploySql');
+const dropAllDbObjects = require('./dropAllDbObjects');
+const scriptDrivedDeploy = require('./scriptDrivedDeploy');
 
 const dbgateApi = {
   queryReader,
@@ -71,6 +73,8 @@ const dbgateApi = {
   sqlTextReplacementTransform,
   autoIndexForeignKeysTransform,
   generateDeploySql,
+  dropAllDbObjects,
+  scriptDrivedDeploy,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

package/src/utility/exportDbModelSql.js

@@ -0,0 +1,80 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { getSchemasUsedByStructure } = require('dbgate-tools');
+
+async function exportDbModelSql(dbModel, driver, outputDir, outputFile) {
+  const { tables, views, procedures, functions, triggers, matviews } = dbModel;
+
+  const usedSchemas = getSchemasUsedByStructure(dbModel);
+  const useSchemaDir = usedSchemas.length > 1;
+
+  const createdDirs = new Set();
+  async function ensureDir(dir) {
+    if (!createdDirs.has(dir)) {
+      await fs.mkdir(dir, { recursive: true });
+      createdDirs.add(dir);
+    }
+  }
+
+  async function writeLists(writeList) {
+    await writeList(views, 'views');
+    await writeList(procedures, 'procedures');
+    await writeList(functions, 'functions');
+    await writeList(triggers, 'triggers');
+    await writeList(matviews, 'matviews');
+  }
+
+  if (outputFile) {
+    const dmp = driver.createDumper();
+    for (const table of tables || []) {
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+    }
+    for (const table of tables || []) {
+      for (const fk of table.foreignKeys || []) {
+        dmp.createForeignKey(fk);
+      }
+    }
+    writeLists((list, folder) => {
+      for (const obj of list || []) {
+        dmp.createSqlObject(obj);
+      }
+    });
+
+    const script = dmp.s;
+    await fs.writeFile(outputFile, script);
+  }
+
+  if (outputDir) {
+    for (const table of tables || []) {
+      const tablesDir = useSchemaDir
+        ? path.join(outputDir, table.schemaName ?? 'default', 'tables')
+        : path.join(outputDir, 'tables');
+      await ensureDir(tablesDir);
+      const dmp = driver.createDumper();
+      dmp.createTable({
+        ...table,
+        foreignKeys: [],
+        dependencies: [],
+      });
+      await fs.writeFile(path.join(tablesDir, `${table.pureName}.sql`), dmp.s);
+    }
+
+    await writeLists(async (list, folder) => {
+      for (const obj of list || []) {
+        const objdir = useSchemaDir
+          ? path.join(outputDir, obj.schemaName ?? 'default', folder)
+          : path.join(outputDir, folder);
+        await ensureDir(objdir);
+        const dmp = driver.createDumper();
+        dmp.createSqlObject(obj);
+        await fs.writeFile(path.join(objdir, `${obj.pureName}.sql`), dmp.s);
+      }
+    });
+  }
+}
+
+module.exports = exportDbModelSql;

package/src/utility/importDbModel.js

@@ -1,28 +1,8 @@
-const fs = require('fs-extra');
-const path = require('path');
-const yaml = require('js-yaml');
 const { databaseInfoFromYamlModel, DatabaseAnalyser } = require('dbgate-tools');
-const { startsWith } = require('lodash');
-const { archivedir, resolveArchiveFolder } = require('./directories');
-const loadFilesRecursive = require('./loadFilesRecursive');
+const loadModelFolder = require('./loadModelFolder');
 
 async function importDbModel(inputDir) {
-  const files = [];
-
-  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
-
-  for (const name of await loadFilesRecursive(dir)) {
-    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
-      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
-
-      files.push({
-        name: path.parse(name).base,
-        text,
-        json: name.endsWith('.yaml') ? yaml.load(text) : null,
-      });
-    }
-  }
-
+  const files = await loadModelFolder(inputDir);
   return databaseInfoFromYamlModel(files);
 }
 

package/src/utility/loadModelFolder.js

@@ -0,0 +1,27 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const { resolveArchiveFolder } = require('./directories');
+const loadFilesRecursive = require('./loadFilesRecursive');
+
+async function loadModelFolder(inputDir) {
+  const files = [];
+
+  const dir = inputDir.startsWith('archive:') ? resolveArchiveFolder(inputDir.substring('archive:'.length)) : inputDir;
+
+  for (const name of await loadFilesRecursive(dir)) {
+    if (name.endsWith('.table.yaml') || name.endsWith('.sql')) {
+      const text = await fs.readFile(path.join(dir, name), { encoding: 'utf-8' });
+
+      files.push({
+        name: path.parse(name).base,
+        text,
+        json: name.endsWith('.yaml') ? yaml.load(text) : null,
+      });
+    }
+  }
+
+  return files;
+}
+
+module.exports = loadModelFolder;

package/src/utility/loadModelTransform.js

@@ -0,0 +1,36 @@
+const { filesdir } = require('./directories');
+const path = require('path');
+const fs = require('fs-extra');
+const _ = require('lodash');
+const dbgateApi = require('../shell');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('loadModelTransform');
+
+function modelTransformFromJson(json) {
+  if (!dbgateApi[json.transform]) return null;
+  const creator = dbgateApi[json.transform];
+  return creator(...json.arguments);
+}
+
+async function loadModelTransform(file) {
+  if (!file) return null;
+  try {
+    const dir = filesdir();
+    const fullPath = path.join(dir, 'modtrans', file);
+    const text = await fs.readFile(fullPath, { encoding: 'utf-8' });
+    const json = JSON.parse(text);
+    if (_.isArray(json)) {
+      const array = _.compact(json.map(x => modelTransformFromJson(x)));
+      return array.length ? structure => array.reduce((acc, val) => val(acc), structure) : null;
+    }
+    if (_.isPlainObject(json)) {
+      return modelTransformFromJson(json);
+    }
+    return null;
+  } catch (err) {
+    logger.error(extractErrorLogData(err), `Error loading model transform ${file}`);
+    return null;
+  }
+}
+
+module.exports = loadModelTransform;

package/src/utility/platformInfo.js

@@ -43,9 +43,17 @@ const platformInfo = {
   platform,
   runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
   allowShellConnection:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_CONNECTION || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_CONNECTION ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowShellScripting:
-    (!processArgs.listenApiChild && !isNpmDist) || !!process.env.SHELL_SCRIPTING || !!isElectron() || !!isDbModel,
+    (!processArgs.listenApiChild && !isNpmDist) ||
+    !!process.env.SHELL_SCRIPTING ||
+    !!isElectron() ||
+    !!isDbModel ||
+    isDevMode,
   allowConnectionFromEnvVariables: !!isDbModel,
   defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
   isAwsUbuntuLayout,