mcdev 7.7.1 → 7.8.0

package/lib/util/devops.js

@@ -266,7 +266,22 @@ const DevOps = {
                 'Please note that deletions have to be done manually on the SFMC website.'
             );
         }
-        Util.logger.info(`Saved report in ${path.join(directoryDeltaPkg, 'delta_package.md')}`);
+        Util.logger.info(
+            `Saved report in ./${directoryDeltaPkg}${Util.logFileName}-delta_package.md`
+        );
+
+        // const deletedTypeKeys = {};
+        // for (const file of delta.filter((file) => file.gitAction === 'delete')) {
+        //     if (deletedTypeKeys[file.type]) {
+        //         deletedTypeKeys[file.type].add(file.externalKey);
+        //     } else {
+        //         deletedTypeKeys[file.type] = new Set([file.externalKey]);
+        //     }
+        // }
+        // for (const deletedType in deletedTypeKeys) {
+        //     const keyArr = [...deletedTypeKeys[deletedType]];
+        //     Util.logger.warn(`Found deleted ${deletedType} keys: ${keyArr.join(', ')}`);
+        // }
 
         // Copy filtered list of files into deploy directory
         // only do that if we do not use templating
@@ -506,6 +521,17 @@ const DevOps = {
                 }
             }
             for (const buFrom in buTypeDelta) {
+                for (const type of Object.keys(buTypeDelta[buFrom])) {
+                    if (
+                        Array.isArray(properties.metaDataTypes.createDeltaPkg) &&
+                        !properties.metaDataTypes.createDeltaPkg.includes(type)
+                    ) {
+                        Util.logger.warn(
+                            `Skipping ${type} for build based on config.metaDataTypes.createDeltaPkg: ${buTypeDelta[buFrom][type].join(', ')}`
+                        );
+                        delete buTypeDelta[buFrom][type];
+                    }
+                }
                 // Run buildTemplate for each business unit for each type
                 await mcdev.build(
                     buFrom,
@@ -516,6 +542,7 @@ const DevOps = {
                     true
                 );
             }
+            this._generateDeleteInstructions(delta, sourceMarket, properties, targetMlName);
         }
         if (!deltaDeployAll.length) {
             Util.logger.error(
@@ -575,6 +602,109 @@ const DevOps = {
         MetadataType[metadataType].buObject = buObject;
         return MetadataType[metadataType].getFilesToCommit(keyArr);
     },
+    /**
+     * helper for {@link DevOps.buildDeltaDefinitions}
+     *
+     * @param {DeltaPkgItem[]} delta git delta
+     * @param {string} sourceMarket market for the source BU
+     * @param {Mcdevrc} properties mcdev config
+     * @param {string} targetMlName marketList used to build for the target BU
+     */
+    _generateDeleteInstructions(delta, sourceMarket, properties, targetMlName) {
+        const deltaDelete = delta
+            .filter((file) => file.gitAction === 'delete')
+            .filter(
+                // in case of assets we need to be careful because extracted code files might get deleted but that doesnt mean the asset was
+                (file) =>
+                    file.type !== 'asset' || (file.type === 'asset' && file.file.endsWith('.json'))
+            );
+        const buTypeDeltaDelete = {}; // for bt, with BU info
+
+        for (const file of deltaDelete) {
+            const buFrom = `${file._credential}/${file._businessUnit}`;
+            if (!buTypeDeltaDelete[buFrom]) {
+                // init object
+                /** @type {TypeKeyCombo} */
+                buTypeDeltaDelete[buFrom] = {};
+            }
+            if (!buTypeDeltaDelete[buFrom][file.type]) {
+                // init array
+                buTypeDeltaDelete[buFrom][file.type] = [];
+            }
+            if (!buTypeDeltaDelete[buFrom][file.type].includes(file.externalKey)) {
+                buTypeDeltaDelete[buFrom][file.type].push(file.externalKey);
+            }
+        }
+        const deleteByBU = {};
+        for (const buFrom in buTypeDeltaDelete) {
+            /** @type {TemplateMap} */
+            const sourceVariables = {};
+            if (Util.checkMarket(sourceMarket, properties)) {
+                Object.assign(sourceVariables, properties.markets[sourceMarket]);
+            }
+            const typeKeyFrom = {};
+            const typeKeyDelete = {};
+            for (const type of Object.keys(buTypeDeltaDelete[buFrom])) {
+                if (
+                    Array.isArray(properties.metaDataTypes.createDeltaPkg) &&
+                    !properties.metaDataTypes.createDeltaPkg.includes(type)
+                ) {
+                    Util.logger.warn(
+                        `Skipping ${type} for delete based on config.metaDataTypes.createDeltaPkg: ${buTypeDeltaDelete[buFrom][type].join(', ')}`
+                    );
+                    delete buTypeDeltaDelete[buFrom][type];
+                    continue;
+                }
+                typeKeyFrom[type] = new Set();
+                if (Object.keys(sourceVariables).length > 0) {
+                    for (let i = 0; i < buTypeDeltaDelete[buFrom][type].length; i++) {
+                        // add templating variables to keys
+                        typeKeyFrom[type].add(
+                            Util.replaceByObject(
+                                buTypeDeltaDelete[buFrom][type][i],
+                                sourceVariables
+                            )
+                        );
+                    }
+                }
+                typeKeyFrom[type] = Array.from(typeKeyFrom[type]);
+                for (const targetBu in properties.marketList[targetMlName]) {
+                    typeKeyDelete[targetBu] ||= {};
+                    typeKeyDelete[targetBu][type] ||= [];
+                    deleteByBU[targetBu] || {};
+                    const market = properties.marketList[targetMlName][targetBu];
+                    const markets = 'string' === typeof market ? [market] : market;
+                    for (const marketArr of markets) {
+                        const templateVariables = {};
+                        for (const market of Array.isArray(marketArr) ? marketArr : [marketArr]) {
+                            if (Util.checkMarket(market, properties)) {
+                                Object.assign(templateVariables, properties.markets[market]);
+                            }
+                        }
+                        typeKeyDelete[targetBu][type].push(
+                            ...typeKeyFrom[type].map((key) =>
+                                MetadataType[type].applyTemplateValues(key, templateVariables)
+                            )
+                        );
+                    }
+                }
+            }
+            for (const targetBu in typeKeyDelete) {
+                const metadataList = [];
+                for (const type in typeKeyDelete[targetBu]) {
+                    for (const key of typeKeyDelete[targetBu][type]) {
+                        metadataList.push(type + ':"' + key + '"');
+                    }
+                }
+                if (metadataList.length) {
+                    Util.logger.warn(
+                        `Run the following to delete ${metadataList.length} potentially existing old files in ${targetBu}`
+                    );
+                    Util.logger.warn(`mcdev delete ${targetBu} -m ${metadataList.join(' ')}`);
+                }
+            }
+        }
+    },
 };
 
 export default DevOps;

package/lib/util/util.js

@@ -309,14 +309,15 @@ export const Util = {
     /**
      * helper for getDefaultProperties()
      *
+     * @param {string} field relevant field in type definition
      * @returns {string[]} type choices
      */
-    getRetrieveTypeChoices() {
+    getTypeChoices(field) {
         const typeChoices = [];
         for (const el in MetadataDefinitions) {
             if (
-                Array.isArray(MetadataDefinitions[el].typeRetrieveByDefault) ||
-                MetadataDefinitions[el].typeRetrieveByDefault === true
+                Array.isArray(MetadataDefinitions[el][field]) ||
+                MetadataDefinitions[el][field] === true
             ) {
                 // complex types like assets are saved as array but to ease upgradability we
                 // save the main type only unless the user deviates from our pre-selection.

package/lib/util/validations.js

@@ -4,6 +4,7 @@ import { Util } from './util.js';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').validationRuleList} validationRuleList
+ * @typedef {import('../../types/mcdev.d.js').validationRuleFix} validationRuleFix
  * @typedef {import('../../types/mcdev.d.js').validationRuleTest} validationRuleTest
  * @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
  */
@@ -40,7 +41,7 @@ async function loadCustomRules() {
  * @param {any} item MetadataItem
  * @param {string} targetDir folder in which the MetadataItem is deployed from (deploy/cred/bu)
  * @param {CodeExtract[]} [codeExtractItemArr] array of code snippets
- * @returns {Promise.<any>} MetadataItem
+ * @returns {Promise.<validationRuleList>} MetadataItem
  */
 export default async function validation(definition, item, targetDir, codeExtractItemArr) {
     try {
@@ -53,8 +54,8 @@ export default async function validation(definition, item, targetDir, codeExtrac
                   definition,
                   item,
                   targetDir,
-                  Util,
-                  codeExtractItemArr
+                  codeExtractItemArr,
+                  Util
               )
             : {};
     } catch (ex) {
@@ -67,6 +68,13 @@ export default async function validation(definition, item, targetDir, codeExtrac
     const defaultRules = {
         noGuidKeys: {
             failedMsg: 'Please update the key to a readable value. Currently still in GUID format.',
+            /**
+             * @type {validationRuleFix}
+             */
+            fix: function () {
+                // to fix we skip the component
+                return this.passed() || null;
+            },
             /**
              * @type {validationRuleTest}
              */
@@ -83,6 +91,13 @@ export default async function validation(definition, item, targetDir, codeExtrac
         },
         noRootFolder: {
             failedMsg: 'Root folder not allowed. Current folder: ' + item.r__folder_Path,
+            /**
+             * @type {validationRuleFix}
+             */
+            fix: function () {
+                // to fix we skip the component
+                return this.passed() || null;
+            },
             /**
              * @type {validationRuleTest}
              */
@@ -108,6 +123,13 @@ export default async function validation(definition, item, targetDir, codeExtrac
         noAmpscriptHtmlTag: {
             failedMsg:
                 'Please use %%[]%% instead of <script runat="server" language="ampscript"></script> for AMPscript',
+            /**
+             * @type {validationRuleFix}
+             */
+            fix: function () {
+                // to fix we skip the component
+                return this.passed() || null;
+            },
             /**
              * @type {validationRuleTest}
              */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcdev",
-    "version": "7.7.1",
+    "version": "7.8.0",
     "description": "Accenture Salesforce Marketing Cloud DevTools",
     "author": "Accenture: joern.berkefeld, douglas.midgley, robert.zimmermann, maciej.barnas",
     "license": "MIT",
@@ -74,11 +74,11 @@
         "console.table": "0.10.0",
         "deep-equal": "2.2.3",
         "fs-extra": "11.3.0",
-        "inquirer": "12.4.1",
+        "inquirer": "12.4.2",
         "json-to-table": "4.2.1",
         "mustache": "4.2.0",
         "p-limit": "6.2.0",
-        "prettier": "3.5.0",
+        "prettier": "3.5.3",
         "prettier-plugin-sql": "0.18.1",
         "semver": "7.7.1",
         "sfmc-sdk": "2.1.2",
@@ -90,26 +90,26 @@
         "yocto-spinner": "0.2.0"
     },
     "devDependencies": {
-        "@eslint/js": "9.20.0",
+        "@eslint/js": "9.21.0",
         "@types/fs-extra": "11.0.4",
         "@types/inquirer": "9.0.7",
         "@types/mocha": "10.0.8",
-        "@types/node": "22.13.1",
+        "@types/node": "22.13.9",
         "@types/yargs": "17.0.33",
         "assert": "2.1.0",
         "axios-mock-adapter": "2.0.0",
         "c8": "10.1.3",
-        "chai": "5.1.2",
+        "chai": "5.2.0",
         "chai-files": "1.4.0",
-        "eslint": "9.20.0",
+        "eslint": "9.21.0",
         "eslint-config-prettier": "10.0.1",
         "eslint-config-ssjs": "2.0.0",
         "eslint-plugin-jsdoc": "50.6.3",
         "eslint-plugin-mocha": "10.5.0",
         "eslint-plugin-prettier": "5.2.3",
         "eslint-plugin-unicorn": "56.0.1",
-        "fast-xml-parser": "4.5.1",
-        "globals": "15.14.0",
+        "fast-xml-parser": "5.0.8",
+        "globals": "16.0.0",
         "husky": "9.1.7",
         "lint-staged": "15.4.3",
         "mocha": "11.1.0",

package/test/general.test.js

@@ -2212,6 +2212,34 @@ describe('GENERAL', () => {
                 );
             });
 
+            it('build but error after buildTemplate step because keys were not found', async () => {
+                const buName = 'testInstance/testBU';
+                const typeKeyCombo = {
+                    asset: ['404'],
+                    dataExtension: ['404'],
+                };
+
+                // handler.setOptions({ skipInteraction: true, purge: false, fix: true });
+
+                await handler.build(
+                    buName,
+                    'ignored',
+                    typeKeyCombo,
+                    ['testSourceMarket'],
+                    ['parent'],
+                    true
+                );
+
+                // THEN
+                assert.equal(process.exitCode, 1, 'build should not have thrown an error');
+
+                assert.equal(
+                    testUtils.getAPIHistoryLength(),
+                    0,
+                    'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
+                );
+            });
+
             it('skip build based on validation rule "filterPrefixByBu" with --fix without error', async () => {
                 const buName = 'testInstance/testBU';
                 const typeKeyCombo = {

package/test/mockRoot/.mcdev-validations.js

@@ -1,4 +1,27 @@
 'use strict';
+/**
+ * @typedef {Object.<string, any>} MetadataTypeItem generic metadata item
+ *
+ * @typedef {object} CodeExtract
+ * @property {string[]} subFolder mostly set to null, otherwise subfolders path split into elements
+ * @property {string} fileName name of file w/o extension
+ * @property {string} fileExt file extension
+ * @property {string} content file content
+ * @property {'base64'} [encoding] optional for binary files
+ *
+ * @callback validationRuleFix
+ * @returns {boolean|null} true = test passed; false = test failed & fixed; null = test failed & item removed to fix
+ *
+ * @callback validationRuleTest
+ * @returns {boolean} true = test passed; false = test failed
+ *
+ * @typedef {object} validationRule
+ * @property {string} failedMsg error message to display in case of a failed test
+ * @property {validationRuleTest} passed test to run
+ * @property {validationRuleFix} [fix] test to run
+ *
+ * @typedef {Object.<string, validationRule>} validationRuleList key=rule name
+ */
 
 /** @type {Object.<string, string[]>} */
 const buPrefixBlacklistMap = {
@@ -8,14 +31,16 @@ const buPrefixBlacklistMap = {
 
 /**
  *
- * @param {any} definition - type defintiion
- * @param {any} item - metadata json
- * @param {string} targetDir - where the metadata is stored ("deploy/cred/bu")
- * @param {any} Util - helper methods
- * @param {{subFolder: string[],fileName: string,fileExt: string,content: string,encoding?: "base64"}[]} [codeExtractItemArr] - actual code blocks
- * @returns {any} validation rule
+ * @param {any} definition type definition
+ * @param {MetadataTypeItem} item MetadataItem
+ * @param {string} targetDir folder in which the MetadataItem is deployed from (deploy/cred/bu)
+ * @param {CodeExtract[]} codeExtractItemArr array of code snippets
+ * @param {any} Util utility functions
+ * @returns {validationRuleList} MetadataItem
  */
-export function validation(definition, item, targetDir, Util, codeExtractItemArr) {
+export function validation(definition, item, targetDir, codeExtractItemArr, Util) {
+    Util.logger.verbose('just here to not get a warning about the unused property Util');
+
     const bu =
         (targetDir.includes('/') ? targetDir.split('/').pop() : targetDir.split('\\').pop()) ||
         'not-found';
@@ -31,9 +56,12 @@ export function validation(definition, item, targetDir, Util, codeExtractItemArr
             get failedMsg() {
                 return `Prefix not allowed on this BU. Blacklisted prefixes: ${prefixBlacklist.join(', ')}`;
             },
-            /**
-             * @returns {boolean|null} true=test passed, false=issue error, null=skip item for current activity due to --fix (build/deploy)
-             */
+            /** @type {validationRuleFix} */
+            fix: function () {
+                // to fix we skip the component
+                return this.passed() || null;
+            },
+            /** @type {validationRuleTest} */
             passed: function () {
                 // this rule aims to prevent deploying things that dont belong on a BU
                 if (prefixBlacklist.length === 0) {
@@ -47,7 +75,7 @@ export function validation(definition, item, targetDir, Util, codeExtractItemArr
                         ('' + item[definition.keyField]).startsWith(prefix)
                     ) {
                         // return false to issue an error or null to skip the item entirely (which is the "fix" of this rule)
-                        return Util.OPTIONS.fix ? null : false;
+                        return false;
                     }
                     // some components have unreadable keys and hence only have the brand prefix in the name
                     if (
@@ -55,7 +83,7 @@ export function validation(definition, item, targetDir, Util, codeExtractItemArr
                         ('' + item[definition.nameField]).startsWith(prefix)
                     ) {
                         // return false to issue an error or null to skip the item entirely (which is the "fix" of this rule)
-                        return Util.OPTIONS.fix ? null : false;
+                        return false;
                     }
                 }
                 // not found

package/test/mockRoot/.mcdevrc.json

@@ -153,7 +153,29 @@
             "transactionalSMS",
             "triggeredSend",
             "verification"
+        ],
+        "createDeltaPkg": [
+            "asset",
+            "automation",
+            "dataExtension",
+            "dataExtract",
+            "emailSend",
+            "event",
+            "fileLocation",
+            "fileTransfer",
+            "importFile",
+            "journey",
+            "mobileCode",
+            "mobileKeyword",
+            "mobileMessage",
+            "query",
+            "script",
+            "sendClassification",
+            "senderProfile",
+            "transactionalPush",
+            "transactionalSMS",
+            "verification"
         ]
     },
-    "version": "7.7.1"
+    "version": "7.8.0"
 }

package/test/utils.js

@@ -192,44 +192,16 @@ export function mockSetup(isDeploy) {
     // no need to execute this again if we ran it a 2nd time for deploy - already done in standard setup
     if (!isDeploy) {
         // reset all options to default
-        handler.setOptions({
-            // test config
-            debug: true,
-            noLogFile: true,
-            // reset
-            api: undefined,
-            autoMidSuffix: undefined,
-            changeKeyField: undefined,
-            changeKeyValue: undefined,
-            commitHistory: undefined,
-            dependencies: undefined,
-            errorLog: undefined,
-            execute: undefined,
-            filter: undefined,
-            fix: undefined,
-            fixShared: undefined,
-            fromRetrieve: undefined,
-            json: undefined,
-            keySuffix: undefined,
-            like: undefined,
-            matchName: undefined,
-            noUpdate: undefined,
-            publish: undefined,
-            purge: undefined,
-            range: undefined,
-            referenceFrom: undefined,
-            referenceTo: undefined,
-            refresh: undefined,
-            retrieve: undefined,
-            schedule: undefined,
-            skipDeploy: undefined,
-            skipInteraction: undefined,
-            skipRetrieve: undefined,
-            skipStatusCheck: undefined,
-            skipValidation: undefined,
-            _runningTest: undefined,
-            _welcomeMessageShown: undefined,
-        });
+        const resetOptions = {};
+        // get known options and make sure none are set
+        for (const option in handler.knownOptions) {
+            resetOptions[option] = undefined;
+        }
+        // test config
+        resetOptions.debug = true;
+        resetOptions.noLogFile = true;
+
+        handler.setOptions(resetOptions);
     }
     apimock = new MockAdapter(axiosInstance, { onNoMatch: 'throwException' });
     // set access_token to mid to allow for autorouting of mock to correct resources

package/types/mcdev.d.js

@@ -463,6 +463,7 @@ complex
  * @property {object} marketList combination of markets and BUs for streamlined deployments
  * @property {object} metaDataTypes templating variables grouped by markets
  * @property {string[]} metaDataTypes.retrieve define what types shall be downloaded by default during retrieve
+ * @property {string[]} metaDataTypes.createDeltaPkg defines what types shall be passed to build() as part of running cdp
  * @property {string[]} metaDataTypes.documentOnRetrieve which types should be parsed & documented after retrieve
  * @property {string} version mcdev version that last updated the config file
  */
@@ -628,16 +629,18 @@ complex
  * @property {string} text seems to be equal to name-attribute?; "Email"
  */
 /**
+ *
+ * @callback validationRuleFix
+ * @returns {boolean|null} true = test passed; false = test failed & fixed; null = test failed & item removed to fix
  *
  * @callback validationRuleTest
  * @returns {boolean} true = test passed; false = test failed
- */
-/**
+ *
  * @typedef {object} validationRule
  * @property {string} failedMsg error message to display in case of a failed test
  * @property {validationRuleTest} passed test to run
- */
-/**
+ * @property {validationRuleFix} [fix] test to run
+ *
  * @typedef {Object.<string, validationRule>} validationRuleList key=rule name
  */