mcdev 5.1.0 → 5.3.0

package/lib/util/cli.js

@@ -53,6 +53,46 @@ const Cli = {
             return null;
         }
     },
+
+    /**
+     *
+     * @param {TYPE.SupportedMetadataTypes} type limit execution to given metadata type
+     * @param {TYPE.SupportedMetadataTypes[]} dependentTypes types that depent on type
+     * @returns {Promise.<boolean>} true if user wants to continue with retrieve
+     */
+    async postFixKeysReretrieve(type, dependentTypes) {
+        if (Util.isTrue(Util.skipInteraction?.fixKeysReretrieve)) {
+            return true;
+        } else if (Util.isFalse(Util.skipInteraction?.fixKeysReretrieve)) {
+            return false;
+        } else {
+            const now = await inquirer.prompt([
+                {
+                    type: 'confirm',
+                    name: 'fixKeysReretrieve',
+                    message: `Do you want to re-retrieve dependent types (${dependentTypes.join(
+                        ', '
+                    )}) now?`,
+                    default: true,
+                },
+            ]);
+            if (Util.OPTIONS._multiBuExecution) {
+                const remember = await inquirer.prompt([
+                    {
+                        type: 'confirm',
+                        name: 'rememberFixKeysReretrieve',
+                        message: `Remember answer for other BUs?`,
+                        default: true,
+                    },
+                ]);
+                if (remember.rememberFixKeysReretrieve) {
+                    Util.skipInteraction ||= {};
+                    Util.skipInteraction.fixKeysReretrieve = now.fixKeysReretrieve;
+                }
+            }
+            return now.fixKeysReretrieve;
+        }
+    },
     /**
      * helper that logs to cli which credentials are already existing in our config file
      *

package/lib/util/file.js

@@ -381,7 +381,7 @@ const File = {
      * @param {string | string[]} directory directory where the file is stored
      * @param {string} filename name of the file without '.json' ending
      * @param {string} filetype filetype suffix
-     * @param {string} [encoding='utf8'] read file with encoding (defaults to utf-8)
+     * @param {string} [encoding] read file with encoding (defaults to utf-8)
      * @returns {Promise.<string> | void} file contents; void on error
      */
     readFilteredFilename: function (directory, filename, filetype, encoding) {
@@ -514,7 +514,7 @@ const File = {
     /**
      * Initalises Prettier formatting lib async.
      *
-     * @param {string} [filetype='html'] filetype ie. JSON or SSJS
+     * @param {string} [filetype] filetype ie. JSON or SSJS
      * @returns {Promise.<boolean>} success of config load
      */
     async initPrettier(filetype = 'html') {

package/lib/util/init.js

@@ -8,6 +8,8 @@ const InitNpm = require('./init.npm');
 const InitConfig = require('./init.config');
 const inquirer = require('inquirer');
 const Util = require('./util');
+const fs = require('node:fs');
+const path = require('node:path');
 
 /**
  * CLI helper class
@@ -22,6 +24,10 @@ const Init = {
      * @returns {Promise.<void>} -
      */
     async initProject(properties, credentialName) {
+        if (!(await Init._checkPathForCloud())) {
+            return;
+        }
+
         const skipInteraction = Util.skipInteraction;
         if (!properties) {
             // try to get cached properties because we return null in case of a crucial error
@@ -192,6 +198,10 @@ const Init = {
      * @returns {Promise.<void>} -
      */
     async joinProject() {
+        if (!(await Init._checkPathForCloud())) {
+            return;
+        }
+
         const responses = await inquirer.prompt([
             {
                 type: 'confirm',
@@ -217,6 +227,9 @@ const Init = {
             ]);
             const repoName = gitRepoQs.gitRepoUrl.split('/').pop().replace('.git', '');
             // clone repo into current folder
+            Util.logger.info(
+                'Cloning initiated. You might be asked for your Git credentials in a pop-up window in a few seconds.'
+            );
             Util.execSync(
                 'git',
                 [
@@ -227,6 +240,16 @@ const Init = {
                     gitRepoQs.gitRepoUrl,
                 ].filter(Boolean)
             );
+
+            if (!fs.existsSync(repoName)) {
+                Util.logger.error(
+                    'Could not clone repository. Please check your Git-Repository URL as well as your credentials and try again.'
+                );
+                Util.logger.info(
+                    'Check if you need an "API-Token" instead of your normal user password to authenticate'
+                );
+                return;
+            }
             // make sure we switch to the new subfolder or else the rest will fail
             process.chdir(repoName);
 
@@ -357,6 +380,10 @@ const Init = {
      * @returns {Promise.<boolean>} success flag
      */
     async upgradeProject(properties, initial, repoName) {
+        if (!(await Init._checkPathForCloud())) {
+            return;
+        }
+
         let status;
         const versionBeforeUpgrade = properties?.version || '0.0.0';
         if (!initial) {
@@ -390,9 +417,66 @@ const Init = {
 
         return true;
     },
+    /**
+     * check if git repo is being saved on a cloud service and warn the user
+     *
+     * @private
+     * @returns {void} throws errors if problems were found
+     */
+    async _checkPathForCloud() {
+        const absolutePath = path.resolve('');
+        // popular cloud services and their respective default name for the absolute path
+        // * CloudDocs is the default folder name for iCloud
+        const cloudServices = ['Dropbox', 'OneDrive', 'Google Drive', 'iCloud', 'CloudDocs'];
+        let cloudServiceFound = false;
+        for (const variable in cloudServices) {
+            if (absolutePath.includes(cloudServices[variable])) {
+                Util.logger.warn(
+                    `It seems your project folder will be synchronized via '${
+                        cloudServices[variable] === 'CloudDocs' ? 'iCloud' : cloudServices[variable]
+                    }'. This can reduce the overall performance of your computer due to conflicts with Git.`
+                );
+                Util.logger.warn(
+                    `We strongly recommend moving your project folder outside of the '${
+                        cloudServices[variable] === 'CloudDocs' ? 'iCloud' : cloudServices[variable]
+                    }' folder.`
+                );
+                cloudServiceFound = true;
+            }
+        }
+        if (!cloudServiceFound && absolutePath.includes(process.env.USERPROFILE)) {
+            // warn user to not place project folder into user profile folder
+            Util.logger.warn(
+                `It seems your project folder is located in your user profile's default folder which is often synchronized to webservices like ${cloudServices.join(
+                    ', '
+                )}. This can reduce the overall performance of your computer due to conflicts between with Git.`
+            );
+            Util.logger.warn(
+                `We strongly recommend moving your project folder outside of this folder.`
+            );
+            cloudServiceFound = true;
+        }
+        if (cloudServiceFound) {
+            const responses = await inquirer.prompt([
+                {
+                    type: 'confirm',
+                    name: 'ignoreCloudWarning',
+                    message: 'Do you want to continue anyways?',
+                    default: false,
+                },
+            ]);
+            if (!responses.ignoreCloudWarning) {
+                Util.logger.error('Exiting due to cloud service warning');
+                return false;
+            }
+        }
+        return true;
+    },
+
     /**
      * finds credentials that are set up in config but not in auth file
      *
+     * @private
      * @param {TYPE.Mcdevrc} properties javascript object in .mcdevrc.json
      * @returns {string[]} list of credential names
      */

package/lib/util/util.js

@@ -3,10 +3,10 @@
 const TYPE = require('../../types/mcdev.d');
 const MetadataDefinitions = require('./../MetadataTypeDefinitions');
 const packageJsonMcdev = require('../../package.json');
+const child_process = require('node:child_process');
 const process = require('node:process');
 const toposort = require('toposort');
 const winston = require('winston');
-const child_process = require('node:child_process');
 
 /**
  * Util that contains logger and simple util methods
@@ -80,6 +80,7 @@ const Util = {
             return false;
         }
     },
+
     /**
      * ensure provided MarketList exists and it's content including markets and BUs checks out
      *
@@ -130,29 +131,32 @@ const Util = {
      * @returns {void}
      */
     signalFatalError() {
-        Util.logger.debug('Util.signalFataError() sets process.exitCode = 1');
-        process.exitCode = 1;
+        Util.logger.debug('Util.signalFataError() sets process.exitCode = 1 unless already set');
+        process.exitCode ||= 1;
     },
     /**
-     * SFMC accepts multiple true values for Boolean attributes for which we are checking here
+     * SFMC accepts multiple true values for Boolean attributes for which we are checking here.
+     * The same problem occurs when evaluating boolean CLI flags
      *
      * @param {*} attrValue value
      * @returns {boolean} attribute value == true ? true : false
      */
     isTrue(attrValue) {
-        return ['true', 'TRUE', 'True', '1', 1, 'Y', true].includes(attrValue);
+        return ['true', 'TRUE', 'True', '1', 1, 'Y', 'y', true].includes(attrValue);
     },
     /**
-     * SFMC accepts multiple false values for Boolean attributes for which we are checking here
+     * SFMC accepts multiple false values for Boolean attributes for which we are checking here.
+     * The same problem occurs when evaluating boolean CLI flags
      *
      * @param {*} attrValue value
      * @returns {boolean} attribute value == false ? true : false
      */
     isFalse(attrValue) {
-        return ['false', 'FALSE', 'False', '0', 0, 'N', false].includes(attrValue);
+        return ['false', 'FALSE', 'False', '0', 0, 'N', 'n', false].includes(attrValue);
     },
+
     /**
-     * helper for {@link Mcdev.retrieve}, {@link Mcdev.retrieveAsTemplate} and {@link Mcdev.deploy}
+     * helper for Mcdev.retrieve, Mcdev.retrieveAsTemplate and Mcdev.deploy
      *
      * @param {TYPE.SupportedMetadataTypes} selectedType type or type-subtype
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
@@ -229,7 +233,7 @@ const Util = {
     /**
      * wrapper around our standard winston logging to console and logfile
      *
-     * @param {boolean} [noLogFile=false] optional flag to indicate if we should log to file; CLI logs are always on
+     * @param {boolean} [noLogFile] optional flag to indicate if we should log to file; CLI logs are always on
      * @returns {object} initiated logger for console and file
      */
     _createNewLoggerTransport: function (noLogFile = false) {
@@ -242,13 +246,21 @@ const Util = {
         //   debug: 5,
         //   silly: 6
         // }
+        if (
+            process.env.VSCODE_AMD_ENTRYPOINT === 'vs/workbench/api/node/extensionHostProcess' ||
+            process.env.VSCODE_CRASH_REPORTER_PROCESS_TYPE === 'extensionHost'
+        ) {
+            Util.OPTIONS.noLogColors = true;
+        }
         const logFileName = new Date().toISOString().split(':').join('.');
         const transports = {
             console: new winston.transports.Console({
                 // Write logs to Console
                 level: Util.OPTIONS.loggerLevel || 'info',
                 format: winston.format.combine(
-                    winston.format.colorize(),
+                    Util.OPTIONS.noLogColors
+                        ? winston.format.uncolorize()
+                        : winston.format.colorize(),
                     winston.format.timestamp({ format: 'HH:mm:ss' }),
                     winston.format.simple(),
                     winston.format.printf(
@@ -263,6 +275,7 @@ const Util = {
                 filename: 'logs/' + logFileName + '.log',
                 level: 'debug', // log everything
                 format: winston.format.combine(
+                    winston.format.uncolorize(),
                     winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
                     winston.format.simple(),
                     winston.format.printf(
@@ -274,7 +287,9 @@ const Util = {
                 // Write logs to additional error-logfile for better visibility of errors
                 filename: 'logs/' + logFileName + '-errors.log',
                 level: 'error', // only log errors
+                lazy: true, // if true, log files will be created on demand, not at the initialization time.
                 format: winston.format.combine(
+                    winston.format.uncolorize(),
                     winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
                     winston.format.simple(),
                     winston.format.printf(
@@ -296,8 +311,8 @@ const Util = {
     /**
      * initiate winston logger
      *
-     * @param {boolean} [restart=false] if true, logger will be restarted; otherwise, an existing logger will be used
-     * @param {boolean} [noLogFile=false] if false, logger will log to file; otherwise, only to console
+     * @param {boolean} [restart] if true, logger will be restarted; otherwise, an existing logger will be used
+     * @param {boolean} [noLogFile] if false, logger will log to file; otherwise, only to console
      * @returns {void}
      */
     startLogger: function (restart = false, noLogFile = false) {
@@ -438,6 +453,7 @@ const Util = {
         }
         return str;
     },
+
     /**
      * get key of an object based on the first matching value
      *
@@ -454,6 +470,31 @@ const Util = {
         throw new Error(`${val} not found in object`);
     },
 
+    /**
+     *helper for Mcdev.fixKeys. Retrieve dependent metadata
+     *
+     * @param {string} fixedType type of the metadata passed as a parameter to fixKeys function
+     * @returns {string[]} array of types that depend on the given type
+     */
+    getDependentMetadata(fixedType) {
+        const dependencies = new Set();
+
+        for (const dependentType of Object.keys(MetadataDefinitions)) {
+            if (MetadataDefinitions[dependentType].dependencies.includes(fixedType)) {
+                // fixedType was found as a dependency of dependentType
+                dependencies.add(dependentType);
+            } else if (
+                MetadataDefinitions[dependentType].dependencies.some((dependency) =>
+                    dependency.startsWith(fixedType + '-')
+                )
+            ) {
+                // if MetadataTypeDefinitions[dependentType].dependencies start with type then also add dependentType to the set; use some to check if any of the dependencies start with type
+                dependencies.add(dependentType);
+            }
+        }
+        return [...dependencies];
+    },
+
     /**
      * Returns Order in which metadata needs to be retrieved/deployed
      *
@@ -736,7 +777,7 @@ const Util = {
         });
     },
     /**
-     * helper for {@link Asset._extractCode} and {@link Script.prepExtractedCode} to determine if a code block is a valid SSJS block
+     * helper for Asset.extractCode and Script.prepExtractedCode to determine if a code block is a valid SSJS block
      *
      * @example the following is invalid:
      * <script runat="server">
@@ -780,6 +821,73 @@ const Util = {
         // no script found
         return null;
     },
+    /**
+     * allows us to filter just like with SQL's LIKE operator
+     *
+     * @param {string} testString field value to test
+     * @param {string} search search string in SQL LIKE format
+     * @returns {boolean} true if testString matches search
+     */
+    stringLike(testString, search) {
+        if (typeof search !== 'string' || this === null) {
+            return false;
+        }
+        // Remove special chars
+        search = search.replaceAll(
+            new RegExp('([\\.\\\\\\+\\*\\?\\[\\^\\]\\$\\(\\)\\{\\}\\=\\!\\<\\>\\|\\:\\-])', 'g'),
+            '\\$1'
+        );
+        // Replace % and _ with equivalent regex
+        search = search.replaceAll('%', '.*').replaceAll('_', '.');
+        // Check matches
+        return new RegExp('^' + search + '$', 'gi').test(testString);
+    },
+    /**
+     * returns true if no LIKE filter is defined or if all filters match
+     *
+     * @param {TYPE.MetadataTypeItem} metadata a single metadata item
+     * @param {object} [filters] only used in recursive calls
+     * @returns {boolean} true if no LIKE filter is defined or if all filters match
+     */
+    fieldsLike(metadata, filters) {
+        if (metadata.json && metadata.codeArr) {
+            // Compensate for CodeExtractItem format
+            metadata = metadata.json;
+        }
+        filters ||= Util.OPTIONS.like;
+        if (!filters) {
+            return true;
+        }
+        const fields = Object.keys(filters);
+        return fields.every((field) => {
+            // to allow passing in an array via cli, e.g. --like=field1,field2, we need to convert comma separated lists into arrays
+            const filter =
+                typeof filters[field] === 'string' && filters[field].includes(',')
+                    ? filters[field].split(',')
+                    : filters[field];
+            if (Array.isArray(metadata[field])) {
+                return metadata[field].some((f) => Util.fieldsLike(f, filter));
+            } else {
+                if (typeof filter === 'string') {
+                    return Util.stringLike(metadata[field], filter);
+                } else if (Array.isArray(filter)) {
+                    return filter.some((f) => Util.stringLike(metadata[field], f));
+                } else if (typeof filter === 'object') {
+                    return Util.fieldsLike(metadata[field], filter);
+                }
+            }
+            return false;
+        });
+    },
+    /**
+     * helper used by SOAP methods to ensure the type always uses an upper-cased first letter
+     *
+     * @param {string} str string to capitalize
+     * @returns {string} str with first letter capitalized
+     */
+    capitalizeFirstLetter(str) {
+        return str.charAt(0).toUpperCase() + str.slice(1);
+    },
 };
 
 Util.startLogger(false, true);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcdev",
-    "version": "5.1.0",
+    "version": "5.3.0",
     "description": "Accenture Salesforce Marketing Cloud DevTools",
     "author": "Accenture: joern.berkefeld, douglas.midgley, robert.zimmermann, maciej.barnas",
     "license": "MIT",
@@ -63,38 +63,38 @@
         "command-exists": "1.2.9",
         "conf": "10.2.0",
         "console.table": "0.10.0",
-        "deep-equal": "2.2.1",
+        "deep-equal": "2.2.2",
         "fs-extra": "11.1.0",
-        "inquirer": "8.2.5",
+        "inquirer": "8.2.6",
         "json-to-table": "4.2.1",
         "mustache": "4.2.0",
         "p-limit": "3.1.0",
         "prettier": "2.8.8",
         "prettier-plugin-sql": "0.14.0",
-        "semver": "7.5.0",
+        "semver": "7.5.4",
         "sfmc-sdk": "1.0.1",
         "simple-git": "3.18.0",
         "toposort": "2.0.2",
         "update-notifier": "5.1.0",
-        "winston": "3.9.0",
+        "winston": "3.10.0",
         "yargs": "17.7.2"
     },
     "devDependencies": {
         "assert": "2.0.0",
-        "axios-mock-adapter": "1.21.3",
-        "chai": "4.3.7",
+        "axios-mock-adapter": "1.21.5",
+        "chai": "4.3.8",
         "chai-files": "1.4.0",
-        "eslint": "8.42.0",
-        "eslint-config-prettier": "8.7.0",
+        "eslint": "8.48.0",
+        "eslint-config-prettier": "9.0.0",
         "eslint-config-ssjs": "1.1.11",
-        "eslint-plugin-jsdoc": "46.2.5",
+        "eslint-plugin-jsdoc": "46.5.0",
         "eslint-plugin-mocha": "10.1.0",
         "eslint-plugin-prettier": "4.2.1",
-        "eslint-plugin-unicorn": "47.0.0",
-        "fast-xml-parser": "4.2.4",
+        "eslint-plugin-unicorn": "48.0.0",
+        "fast-xml-parser": "4.2.7",
         "husky": "8.0.3",
         "jsdoc-to-markdown": "8.0.0",
-        "lint-staged": "13.2.2",
+        "lint-staged": "14.0.1",
         "mocha": "10.2.0",
         "mock-fs": "5.2.0",
         "npm-check": "6.0.1",

package/test/mockRoot/.mcdevrc.json

@@ -78,5 +78,5 @@
             "triggeredSend"
         ]
     },
-    "version": "5.1.0"
+    "version": "5.3.0"
 }

package/test/mockRoot/deploy/testInstance/_ParentBU_/dataExtension/testExisting_dataExtensionShared.dataExtension-meta.json

@@ -0,0 +1,59 @@
+{
+    "CustomerKey": "testExisting_dataExtensionShared",
+    "Name": "testExisting_dataExtensionShared",
+    "Description": "Container for my test emails",
+    "IsSendable": true,
+    "IsTestable": true,
+    "SendableDataExtensionField": { "Name": "ContactKey" },
+    "SendableSubscriberField": { "Name": "Subscriber Key" },
+    "DataRetentionPeriodLength": 6,
+    "DataRetentionPeriodUnitOfMeasure": 5,
+    "RowBasedRetention": true,
+    "ResetRetentionPeriodOnImport": false,
+    "DeleteAtEndOfRetentionPeriod": false,
+    "RetainUntil": "",
+    "Fields": [
+        {
+            "Name": "FirstName",
+            "DefaultValue": "",
+            "MaxLength": 50,
+            "IsRequired": false,
+            "IsPrimaryKey": false,
+            "FieldType": "Text"
+        },
+        {
+            "Name": "LastName",
+            "DefaultValue": "",
+            "MaxLength": 55,
+            "IsRequired": false,
+            "IsPrimaryKey": false,
+            "FieldType": "Text"
+        },
+        {
+            "Name": "EmailAddress",
+            "DefaultValue": "",
+            "MaxLength": 254,
+            "IsRequired": true,
+            "IsPrimaryKey": false,
+            "FieldType": "EmailAddress"
+        },
+        {
+            "Name": "newField",
+            "DefaultValue": "",
+            "MaxLength": 254,
+            "IsRequired": false,
+            "IsPrimaryKey": false,
+            "FieldType": "Text"
+        },
+        {
+            "Name": "ContactKey",
+            "DefaultValue": "",
+            "MaxLength": 50,
+            "IsRequired": true,
+            "IsPrimaryKey": true,
+            "FieldType": "Text"
+        }
+    ],
+    "r__folder_ContentType": "shared_dataextension",
+    "r__folder_Path": "Shared Items/Shared Data Extensions"
+}

package/test/mockRoot/deploy/testInstance/_ParentBU_/dataExtension/testNew_dataExtensionShared.dataExtension-meta.json

@@ -0,0 +1,23 @@
+{
+    "CustomerKey": "testNew_dataExtensionShared",
+    "Name": "testNew_dataExtensionShared",
+    "Description": "",
+    "IsSendable": false,
+    "IsTestable": false,
+    "RowBasedRetention": false,
+    "ResetRetentionPeriodOnImport": false,
+    "DeleteAtEndOfRetentionPeriod": false,
+    "RetainUntil": "",
+    "Fields": [
+        {
+            "Name": "testField",
+            "DefaultValue": "",
+            "MaxLength": 254,
+            "IsRequired": true,
+            "IsPrimaryKey": false,
+            "FieldType": "Text"
+        }
+    ],
+    "r__folder_ContentType": "shared_dataextension",
+    "r__folder_Path": "Shared Items/Shared Data Extensions"
+}

package/test/mockRoot/deploy/testInstance/testBU/automation/testExisting_automation.automation-meta.json

@@ -7,8 +7,7 @@
         "endDate": "2022-07-30T00:00:00",
         "icalRecur": "FREQ=DAILY;COUNT=1;INTERVAL=1",
         "startDate": "2022-07-30T00:00:00",
-        "timezoneName": "W. Europe Standard Time",
-        "typeId": 3
+        "timezoneName": "W. Europe Standard Time"
     },
     "status": "PausedSchedule",
     "steps": [

package/test/mockRoot/deploy/testInstance/testBU/automation/testNew_automation.automation-meta.json

@@ -4,13 +4,12 @@
     "name": "testNew_automation",
     "r__folder_Path": "my automations",
     "schedule": {
-        "endDate": "2022-07-30T00:00:00",
-        "icalRecur": "FREQ=DAILY;COUNT=1;INTERVAL=1",
-        "startDate": "2022-07-30T00:00:00",
-        "timezoneName": "W. Europe Standard Time",
-        "typeId": 3
+        "startDate": "2020-05-14T02:30:32.11",
+        "endDate": "2079-06-06T21:00:00",
+        "icalRecur": "FREQ=MINUTELY;UNTIL=20790607T050000;INTERVAL=5",
+        "timezoneName": "W. Europe Standard Time"
     },
-    "status": "PausedSchedule",
+    "status": "Scheduled",
     "steps": [
         {
             "activities": [
@@ -37,6 +36,10 @@
                 {
                     "name": "testExisting_script",
                     "r__type": "script"
+                },
+                {
+                    "name": "testNew_39f6a488-20eb-4ba0-b0b9",
+                    "r__type": "verification"
                 }
             ],
             "name": ""

package/test/mockRoot/deploy/testInstance/testBU/dataExtension/testExisting_dataExtension.dataExtension-meta.json

@@ -31,6 +31,7 @@
         },
         {
             "Name": "EmailAddress",
+            "Name_new": "Email",
             "DefaultValue": "",
             "MaxLength": 254,
             "IsRequired": true,