mcdev 3.1.3 → 4.0.0

package/lib/util/devops.js

@@ -1,8 +1,10 @@
+const TYPE = require('../../types/mcdev.d');
 const File = require('./file');
-const path = require('path');
+const path = require('node:path');
 const inquirer = require('inquirer');
 const Util = require('./util');
-const git = require('simple-git/promise')();
+const git = require('simple-git')();
+const Builder = require('../Builder');
 const MetadataType = require('../MetadataTypeInfo');
 const jsonToTable = require('json-to-table');
 /**
@@ -13,25 +15,23 @@ const DevOps = {
     /**
      * Extracts the delta between a commit and the current state for deployment.
      * Interactive commit selection if no commits are passed.
-     * @param {Object} properties central properties object
-     * @param {String} [range] git commit range
+     *
+     * @param {TYPE.Mcdevrc} properties central properties object
+     * @param {string} [range] git commit range
      * @param {boolean} [saveToDeployDir] if true, copy metadata changes into deploy directory
-     * @param {String} [filterPaths] filter file paths that start with any specified path (comma separated)
-     * @returns {Promise<Object>} -
+     * @param {string} [filterPaths] filter file paths that start with any specified path (comma separated)
+     * @returns {Promise.<TYPE.DeltaPkgItem[]>} -
      */
-    async createDeltaPkg(properties, range, saveToDeployDir, filterPaths) {
+    async getDeltaList(properties, range, saveToDeployDir, filterPaths) {
         const rangeUserInput = range;
-        Util.logger.info('Create Delta Package ::');
-        if (filterPaths) {
-            filterPaths = filterPaths.split(',').map((filePath) =>
-                path
-                    .normalize(properties.directories.retrieve + filePath)
-                    .split('\\')
-                    .join('/')
-            );
-        } else {
-            filterPaths = [properties.directories.retrieve];
-        }
+        filterPaths = filterPaths
+            ? filterPaths.split(',').map((filePath) =>
+                  path
+                      .normalize(properties.directories.retrieve + filePath)
+                      .split('\\')
+                      .join('/')
+              )
+            : [properties.directories.retrieve];
         if (range) {
             if (!range.includes('..')) {
                 // we limit the user here somewhat by always comparing to current branch if no range was given
@@ -54,7 +54,7 @@ const DevOps = {
             }));
             display.push(new inquirer.Separator(' ==== '));
 
-            const questions = [
+            const responses = await inquirer.prompt([
                 {
                     type: 'list',
                     message: 'Select base commit for comparison with current commit',
@@ -62,12 +62,7 @@ const DevOps = {
                     pageSize: 10,
                     choices: display,
                 },
-            ];
-            const responses = await new Promise((resolve) => {
-                inquirer.prompt(questions).then((answers) => {
-                    resolve(answers);
-                });
-            });
+            ]);
             range = `${responses.commit}..HEAD`;
         }
 
@@ -78,18 +73,22 @@ const DevOps = {
             'add/update': 0,
             move: 0,
         };
+        /**
+         * @type {TYPE.DeltaPkgItem[]}
+         */
         const delta = (await git.diffSummary([range])).files
             // populate additional info for all changed files
-            .map((file) => {
+            .map((/** @type {TYPE.DeltaPkgItem} */ file) => {
                 // If file was moved it's path needs to be parsed
                 file.moved = file.file.includes('=>');
                 if (file.moved) {
                     const p = file.file;
+                    // TODO: rewrite remaining substring() to slice()
                     const paths = {
-                        base: p.substring(0, p.indexOf('{')),
-                        before: p.substring(p.indexOf('{') + 1, p.indexOf('=>') - 1),
-                        after: p.substring(p.indexOf('=>') + 3, p.indexOf('}')),
-                        end: p.substring(p.indexOf('}') + 1),
+                        base: p.slice(0, Math.max(0, p.indexOf('{'))),
+                        before: p.substring(p.indexOf('{') + 1, p.indexOf('=>') - 1), // eslint-disable-line unicorn/prefer-string-slice
+                        after: p.substring(p.indexOf('=>') + 3, p.indexOf('}')), // eslint-disable-line unicorn/prefer-string-slice
+                        end: p.slice(Math.max(0, p.indexOf('}') + 1)),
                     };
                     file.fromPath = path
                         .normalize(`${paths.base}${paths.before}${paths.end}`)
@@ -113,7 +112,7 @@ const DevOps = {
             // ! Filter happens after initial parse, because if file was moved its new path has to be calculated
             .filter((file) => filterPaths.some((path) => file.file.startsWith(path)))
             // ensure badly named files on unsupported metadata types are not in our subset
-            .filter((file) => {
+            .filter((/** @type {TYPE.DeltaPkgItem} */ file) => {
                 if (!MetadataType[file.type]) {
                     Util.logger.debug(
                         `Unknown metadata-type found for (${file.file}): ` + file.type
@@ -123,7 +122,7 @@ const DevOps = {
                     return true;
                 }
             })
-            .map((file) => {
+            .map((/** @type {TYPE.DeltaPkgItem} */ file) => {
                 // Gets external key based on file name und the assumption that filename = externalKey
                 if (file.type === 'folder') {
                     file.externalKey = null;
@@ -135,7 +134,7 @@ const DevOps = {
 
                 // Check if file doesn't exist in reported path, that means it was a git deletion
                 // TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing
-                if (!File.existsSync(file.file)) {
+                if (!File.pathExistsSync(file.file)) {
                     file.gitAction = 'delete';
                 } else if (file.moved) {
                     file.gitAction = 'move';
@@ -189,8 +188,12 @@ const DevOps = {
             return [];
         }
         // Write into delta.json to serve as documentation
-        File.writeJSONToFile(properties.directories.deltaPackage, 'delta_package', delta);
-        this.document(properties.directories.deltaPackage, delta);
+        const directoryDeltaPkg = File.normalizePath([
+            properties.directories.docs,
+            'deltaPackage/',
+        ]);
+        await File.writeJSONToFile(directoryDeltaPkg, 'delta_package', delta);
+        this.document(directoryDeltaPkg, delta);
         Util.logger.info(
             `- ✔️  Identified changes: Add/Update=${gitActionsCounter['add/update']}, Move=${gitActionsCounter['move']}, Delete=${gitActionsCounter['delete']}`
         );
@@ -199,12 +202,11 @@ const DevOps = {
                 'Please note that deletions have to be done manually on the SFMC website.'
             );
         }
-        Util.logger.info(
-            `Saved report in ${path.join(properties.directories.deltaPackage, 'delta_package.md')}`
-        );
+        Util.logger.info(`Saved report in ${path.join(directoryDeltaPkg, 'delta_package.md')}`);
 
         // Copy filtered list of files into deploy directory
         if (saveToDeployDir) {
+            /** @type {TYPE.DeltaPkgItem} */
             const copied = delta.map((file) =>
                 File.copyFile(
                     file.file,
@@ -229,42 +231,215 @@ const DevOps = {
                 } copied, ${skipped.length} skipped, ${failed.length} failed)`
             );
             if (skipped.length > 0) {
-                skipped.forEach((file) =>
-                    Util.logger.debug(`Skipped - ${file.statusMessage} - ${file.file}`)
-                );
+                for (const file of skipped) {
+                    Util.logger.debug(`Skipped - ${file.statusMessage} - ${file.file}`);
+                }
             }
             if (failed.length > 0) {
-                failed.forEach((file) =>
-                    Util.logger.error(`Failed - ${file.statusMessage} - ${file.file}`)
-                );
+                for (const file of failed) {
+                    Util.logger.error(`Failed - ${file.statusMessage} - ${file.file}`);
+                }
             }
         }
         return delta;
     },
+    /**
+     * wrapper around DevOps.getDeltaList, Builder.buildTemplate and M
+     *
+     * @param {TYPE.Mcdevrc} properties project config file
+     * @param {string} range git commit range
+     * @param {TYPE.SkipInteraction} [skipInteraction] allows to skip interactive wizard
+     */
+    async buildDeltaDefinitions(properties, range, skipInteraction) {
+        // check if sourceTargetMapping is valid
+        if (
+            !properties.options.deployment.sourceTargetMapping ||
+            !Object.keys(properties.options.deployment.sourceTargetMapping).length
+        ) {
+            Util.logger.error('Bad configuration of options.deployment.sourceTargetMapping');
+            return;
+        }
+        const sourceMarketListArr = Object.keys(properties.options.deployment.sourceTargetMapping);
+
+        for (const sourceML of sourceMarketListArr) {
+            // check if sourceTargetMapping has valid values
+            // #1 check source marketlist
+            try {
+                Util.verifyMarketList(sourceML, properties);
+                // remove potentially existing "description"-entry
+                delete properties.marketList[sourceML].description;
+
+                const sourceMarketBuArr = Object.keys(properties.marketList[sourceML]);
+                if (sourceMarketBuArr.length !== 1) {
+                    throw new Error('Only 1 BU is allowed per source marketList');
+                }
+                if ('string' !== typeof properties.marketList[sourceML][sourceMarketBuArr[0]]) {
+                    throw new TypeError('Only 1 market per BU is allowed per source marketList');
+                }
+            } catch (ex) {
+                Util.logger.error('Deployment Source: ' + ex.message);
+                return;
+            }
+            // #2 check corresponding target marketList
+            let targetML;
+            try {
+                targetML = properties.options.deployment.sourceTargetMapping[sourceML];
+                if ('string' !== typeof targetML) {
+                    throw new TypeError(
+                        'Please define one target marketList per source in deployment.sourceTargetMapping (No arrays allowed)'
+                    );
+                }
+                Util.verifyMarketList(targetML, properties);
+                // remove potentially existing "description"-entry
+                delete properties.marketList[targetML].description;
+            } catch (ex) {
+                Util.logger.error('Deployment Target: ' + ex.message);
+            }
+        }
+        // all good let's loop a second time for actual execution
+        for (const sourceMlName of sourceMarketListArr) {
+            const targetMlName = properties.options.deployment.sourceTargetMapping[sourceMlName];
+            const sourceBU = Object.keys(properties.marketList[sourceMlName])[0];
+            const sourceMarket = Object.values(properties.marketList[sourceMlName])[0];
+
+            const delta = await DevOps.getDeltaList(properties, range, false, sourceBU);
+            // If only chaing templating and buildDefinition if required
+            if (!delta || delta.length === 0) {
+                // info/error messages was printed by DevOps.createDeltaPkg() already
+                return;
+            }
+            Util.logger.info('=============');
+
+            // Put files into maps. One map with BU -> type -> file (for retrieveAsTemplate)
+            // Other map only with type -> file (for buildDefinitionBulk)
+            const buTypeDelta = {}; // for bt, with BU info
+            const typeDelta = {}; // for bdb, without BU info - thats taken from the marketList
+            let deltaCounter = 0;
+            for (const file of delta
+                // Only template/build files that were added/updated/moved. no deletions
+                // ! doesn't work for folder, because their name parsing doesnt work at the moment
+                .filter((file) => file.gitAction !== 'delete' && file.name)) {
+                const buPath = `${file._credential}/${file._businessUnit}`;
+                if (!buTypeDelta[buPath]) {
+                    // init object
+                    buTypeDelta[buPath] = {};
+                }
+                if (!buTypeDelta[buPath][file.type]) {
+                    // init array
+                    buTypeDelta[buPath][file.type] = [];
+                }
+                buTypeDelta[buPath][file.type].push(file.externalKey);
+            }
+
+            // Run buildTemplate for each business unit for each type
+            Util.logger.info('Retrieve template from Git delta');
+            // ! needs to be for (.. in ..) loop so that it gets executed in series
+            for (const bu in buTypeDelta) {
+                for (const type in buTypeDelta[bu]) {
+                    // get unique list (original search might include more than one entry for types with docs or extracted code)
+                    const keyArr = [...new Set(buTypeDelta[bu][type])];
+                    Util.logger.info(
+                        `⚡ mcdev bt ${bu} ${type} "${keyArr.join(',')}" ${sourceMarket}`
+                    );
+                    await Builder.buildTemplate(bu, type, keyArr, sourceMarket);
+                    // ensure we have the right key for bd/bdb that matches the name used for rt
+                    if (keyArr.length) {
+                        if (!typeDelta[type]) {
+                            // init array
+                            typeDelta[type] = [];
+                        }
+                        typeDelta[type].push(...keyArr);
+                        deltaCounter += keyArr.length;
+                    }
+                }
+            }
+            if (deltaCounter) {
+                Util.logger.info(`- ✔️  Templates created: ${deltaCounter}`);
+            } else {
+                Util.logger.error(
+                    '- ❌ No Templates or Deploy Definitions created. Check if you expected no changes.'
+                );
+                return;
+            }
+
+            // Run build definitions bulk for each type
+            Util.logger.info('=============');
+            Util.logger.info('Build deploy definitions from delta templates');
+            if (
+                properties.directories.templateBuilds == properties.directories.deploy ||
+                (Array.isArray(properties.directories.templateBuilds) &&
+                    properties.directories.templateBuilds.includes(properties.directories.deploy))
+            ) {
+                let responses;
+                if (!skipInteraction) {
+                    // deploy folder is in targets for definition creation
+                    // recommend to purge their content first
+                    responses = await inquirer.prompt([
+                        {
+                            type: 'confirm',
+                            name: 'isPurgeDeployFolder',
+                            message:
+                                'Do you want to empty the deploy folder (ensures no files from previous deployments remain)?',
+                            default: true,
+                        },
+                    ]);
+                }
+                if (skipInteraction || responses.isPurgeDeployFolders) {
+                    // Clear output folder structure for selected sub-type
+                    await File.remove(File.normalizePath([properties.directories.deploy]));
+                }
+            }
+            const bdPromises = [];
+            for (const type in typeDelta) {
+                Util.logger.info(
+                    `⚡ mcdev bdb ${targetMlName} ${type} "${typeDelta[type].join(',')}"`
+                );
+                // omitting "await" to speed up creation
+                bdPromises.push(
+                    Builder.buildDefinitionBulk(targetMlName, type, typeDelta[type].join(','))
+                );
+            }
+            await Promise.all(bdPromises);
+            Util.logger.info(`- ✔️  Deploy defintions created`);
+            if (
+                properties.directories.templateBuilds == properties.directories.deploy ||
+                (Array.isArray(properties.directories.templateBuilds) &&
+                    properties.directories.templateBuilds.includes(properties.directories.deploy))
+            ) {
+                Util.logger.info(`You can now run deploy on the prepared BUs`);
+            } else {
+                Util.logger.info(
+                    `Your templated defintions are now ready to be copied into the deploy folder. Hint: You can have this auto-copied if you adjust directories.templateBuilds in your config.`
+                );
+            }
+        }
+    },
+
     /**
      * create markdown file for deployment listing
-     * @param {String} directory -
-     * @param {Object} jsonReport -
+     *
+     * @param {string} directory -
+     * @param {object} jsonReport -
      * @returns {void}
      */
     document(directory, jsonReport) {
         const tabled = jsonToTable(jsonReport);
         let output = `# Deployment Report\n\n`;
         let tableSeparator = '';
-        tabled[0].forEach((column) => {
+        for (const column of tabled[0]) {
             if (column !== '') {
                 output += `| ${column} `;
                 tableSeparator += '| --- ';
             }
-        });
+        }
         output += `|\n${tableSeparator}|\n`;
         for (let i = 1; i < tabled.length; i++) {
-            tabled[i].forEach((field) => {
+            for (let field of tabled[i]) {
                 if (field !== '') {
                     field = field === true ? '✓' : field === false ? '✗' : field;
                     output += `| ${field} `;
                 }
-            });
+            }
             output += '|\n';
         }
         try {
@@ -274,6 +449,21 @@ const DevOps = {
             Util.logger.error(`DevOps.document():: error | ` + ex.message);
         }
     },
+    /**
+     * should return only the json for all but asset, query and script that are saved as multiple files
+     * additionally, the documentation for dataExtension and automation should be returned
+     *
+     * @param {TYPE.Mcdevrc} properties central properties object
+     * @param {TYPE.BuObject} buObject references credentials
+     * @param {string} metadataType metadata type to build
+     * @param {string[]} keyArr customerkey of the metadata
+     * @returns {Promise.<string[]>} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
+     */
+    getFilesToCommit(properties, buObject, metadataType, keyArr) {
+        MetadataType[metadataType].properties = properties;
+        MetadataType[metadataType].buObject = buObject;
+        return MetadataType[metadataType].getFilesToCommit(keyArr);
+    },
 };
 
 module.exports = DevOps;