mcdev 9.0.3 → 10.0.0

package/lib/util/util.js

@@ -2,14 +2,22 @@
 
 import MetadataDefinitions from './../MetadataTypeDefinitions.js';
 import process from 'node:process';
-import toposort from 'toposort';
-import winston from 'winston';
+import { topologicalSort } from './topological-sort.js';
 import child_process from 'node:child_process';
 import path from 'node:path';
-// import just to resolve cyclical - TO DO consider if could move to file or context
-import { readJsonSync } from 'fs-extra/esm';
-import { fileURLToPath } from 'node:url';
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
+import packageJsonMcdev from '../../package.json' with { type: 'json' };
+import winston from 'winston';
+const WinstonConsoleTransport = winston.transports.Console;
+const WinstonFileTransport = winston.transports.File;
+const WinstonFormat = {
+    combine: winston.format.combine,
+    uncolorize: winston.format.uncolorize,
+    colorize: winston.format.colorize,
+    timestamp: winston.format.timestamp,
+    simple: winston.format.simple,
+    printf: winston.format.printf,
+};
+const defaultBoilerplateRoot = path.resolve(import.meta.dirname, '../../boilerplate');
 
 /**
  * @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject
@@ -41,28 +49,60 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 export const Util = {
     authFileName: '.mcdev-auth.json',
     boilerplateDirectory: '../../boilerplate',
+    boilerplateRoot: defaultBoilerplateRoot,
     configFileName: '.mcdevrc.json',
     defaultGitBranch: 'main',
     parentBuName: '_ParentBU_',
     standardizedSplitChar: '/',
     /** used to replace '/' in folder names to avoid confusion with the path separator */
-    folderNameSlashEscapeChar: '\u2215',
+    folderNameSlashEscapeChar: '\u{2215}',
     /** @type {SkipInteraction} */
     skipInteraction: null,
-    packageJsonMcdev: readJsonSync(path.join(__dirname, '../../package.json')),
+    packageJsonMcdev,
     OPTIONS: {},
     changedKeysMap: {},
     matchedByName: {},
 
+    /**
+     * Resolve a path within the bundled project boilerplate.
+     *
+     * @param {...string} segments path segments below the boilerplate root
+     * @returns {string} absolute boilerplate path
+     */
+    getBoilerplatePath(...segments) {
+        return path.resolve(Util.boilerplateRoot, ...segments);
+    },
+
+    /**
+     * Override the boilerplate root for embedded consumers and tests.
+     *
+     * @param {string} root absolute or cwd-relative boilerplate root
+     * @returns {string} resolved boilerplate root
+     */
+    setBoilerplateRoot(root) {
+        Util.boilerplateRoot = path.resolve(root);
+        return Util.boilerplateRoot;
+    },
+
+    /**
+     * Restore the package's default boilerplate root.
+     *
+     * @returns {string} restored boilerplate root
+     */
+    resetBoilerplateRoot() {
+        Util.boilerplateRoot = defaultBoilerplateRoot;
+        return Util.boilerplateRoot;
+    },
+
     /**
      * helper that allows filtering an object by its keys
      *
-     * @param {Object.<string,*>} originalObj object that you want to filter
+     * @param {Object.<string, unknown>} originalObj object that you want to filter
      * @param {string[]} [whitelistArr] positive filter. if not provided, returns originalObj without filter
-     * @returns {Object.<string,*>} filtered object that only contains keys you provided
+     * @returns {Object.<string, unknown>} filtered object that only contains keys you provided
      */
     filterObjByKeys(originalObj, whitelistArr) {
-        if (!whitelistArr || !Array.isArray(whitelistArr)) {
+        if (!originalObj || !whitelistArr || !Array.isArray(whitelistArr)) {
             return originalObj;
         }
         return Object.keys(originalObj)
@@ -148,24 +188,26 @@ export const Util = {
             // check if BUs in marketList are valid
             let buCounter = 0;
             for (const businessUnit in properties.marketList[mlName]) {
-                if (businessUnit !== 'description' && businessUnit !== 'filter') {
-                    buCounter++;
-                    Util.isValidBU(properties, businessUnit, true);
-                    if (!Util.isValidBU(properties, businessUnit, true)) {
-                        throw new Error(`'${businessUnit}' in Market ${mlName} is not defined.`);
-                    }
+                if (businessUnit === 'description' || businessUnit === 'filter') {
+                    continue;
+                }
 
-                    // check if markets are valid
-                    let marketArr = properties.marketList[mlName][businessUnit];
-                    if ('string' === typeof marketArr) {
-                        marketArr = [marketArr];
-                    }
-                    for (const market of marketArr) {
-                        if (properties.markets[market]) {
-                            // * markets can be empty or include variables. Nothing we can test here
-                        } else {
-                            throw new Error(`Market '${market}' is not defined.`);
-                        }
+                buCounter++;
+                Util.isValidBU(properties, businessUnit, true);
+                if (!Util.isValidBU(properties, businessUnit, true)) {
+                    throw new Error(`'${businessUnit}' in Market ${mlName} is not defined.`);
+                }
+
+                // check if markets are valid
+                let marketArr = properties.marketList[mlName][businessUnit];
+                if ('string' === typeof marketArr) {
+                    marketArr = [marketArr];
+                }
+                for (const market of marketArr) {
+                    if (properties.markets[market]) {
+                        // * markets can be empty or include variables. Nothing we can test here
+                    } else {
+                        throw new Error(`Market '${market}' is not defined.`);
                     }
                 }
             }
@@ -222,7 +264,7 @@ export const Util = {
      * SFMC accepts multiple true values for Boolean attributes for which we are checking here.
      * The same problem occurs when evaluating boolean CLI flags
      *
-     * @param {*} attrValue value
+     * @param {string|number|boolean|null|undefined} attrValue value
      * @returns {boolean} attribute value == true ? true : false
      */
     isTrue(attrValue) {
@@ -233,7 +275,7 @@ export const Util = {
      * SFMC accepts multiple false values for Boolean attributes for which we are checking here.
      * The same problem occurs when evaluating boolean CLI flags
      *
-     * @param {*} attrValue value
+     * @param {string|number|boolean|null|undefined} attrValue value
      * @returns {boolean} attribute value == false ? true : false
      */
     isFalse(attrValue) {
@@ -295,7 +337,7 @@ export const Util = {
     },
 
     /**
-     * helper for Mcdev.retrieve, Mcdev.retrieveAsTemplate and Mcdev.deploy
+     * helper for Mcdev.retrieve and Mcdev.deploy
      *
      * @param {string} selectedType type or type-subtype
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
@@ -322,7 +364,7 @@ export const Util = {
     },
 
     /**
-     * helper for Mcdev.retrieve, Mcdev.retrieveAsTemplate and Mcdev.deploy
+     * helper for Mcdev.retrieve and Mcdev.deploy
      *
      * @param {Mcdevrc} properties javascript object in .mcdevrc.json
      * @param {string} businessUnit name of BU
@@ -448,49 +490,49 @@ export const Util = {
         ) {
             Util.OPTIONS.noLogColors = true;
         }
-        Util.logFileName = new Date().toISOString().split(':').join('.');
+        Util.logFileName = new Date().toISOString().replaceAll(':', '.');
         const transports = {
-            console: new winston.transports.Console({
+            console: new WinstonConsoleTransport({
                 // Write logs to Console
                 level: Util.OPTIONS.loggerLevel || 'info',
-                format: winston.format.combine(
+                format: WinstonFormat.combine(
                     Util.OPTIONS.noLogColors
-                        ? winston.format.uncolorize()
-                        : winston.format.colorize(),
-                    winston.format.timestamp({ format: 'HH:mm:ss' }),
-                    winston.format.simple(),
-                    winston.format.printf(
+                        ? WinstonFormat.uncolorize()
+                        : WinstonFormat.colorize(),
+                    WinstonFormat.timestamp({ format: 'HH:mm:ss' }),
+                    WinstonFormat.simple(),
+                    WinstonFormat.printf(
                         (info) => `${info.timestamp} ${info.level}: ${info.message}`
                     )
                 ),
             }),
         };
         if (!noLogFile) {
-            transports.file = new winston.transports.File({
+            transports.file = new WinstonFileTransport({
                 // Write logs to logfile
                 filename: 'logs/' + Util.logFileName + '.log',
                 level: 'debug', // log everything
-                format: winston.format.combine(
-                    winston.format.uncolorize(),
-                    winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
-                    winston.format.simple(),
-                    winston.format.printf(
+                format: WinstonFormat.combine(
+                    WinstonFormat.uncolorize(),
+                    WinstonFormat.timestamp({ format: 'HH:mm:ss.SSS' }),
+                    WinstonFormat.simple(),
+                    WinstonFormat.printf(
                         (info) => `${info.timestamp} ${info.level}: ${info.message}`
                     )
                 ),
             });
             if (Util.OPTIONS.errorLog) {
                 // used by CI/CD solutions like Copado to quickly show the error message to admins/users
-                transports.fileError = new winston.transports.File({
+                transports.fileError = new WinstonFileTransport({
                     // Write logs to additional error-logfile for better visibility of errors
                     filename: 'logs/' + Util.logFileName + '-errors.log',
                     level: 'error', // only log errors
                     lazy: true, // if true, log files will be created on demand, not at the initialization time.
-                    format: winston.format.combine(
-                        winston.format.uncolorize(),
-                        winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
-                        winston.format.simple(),
-                        winston.format.printf(
+                    format: WinstonFormat.combine(
+                        WinstonFormat.uncolorize(),
+                        WinstonFormat.timestamp({ format: 'HH:mm:ss.SSS' }),
+                        WinstonFormat.simple(),
+                        WinstonFormat.printf(
                             (info) => `${info.timestamp} ${info.level}: ${info.message}`
                         )
                     ),
@@ -506,7 +548,7 @@ export const Util = {
      *
      * @type {Logger}
      */
-    logger: null,
+    logger: createPreStartLogger(),
 
     /**
      * initiate winston logger
@@ -516,13 +558,11 @@ export const Util = {
      * @returns {void}
      */
     startLogger: function (restart = false, noLogFile = false) {
-        if (
-            !(
-                Util.loggerTransports === null ||
-                restart ||
-                (!Util.loggerTransports?.file && !noLogFile && !Util.OPTIONS?.noLogFile)
-            )
-        ) {
+        if (!(
+            Util.loggerTransports === null ||
+            restart ||
+            (!Util.loggerTransports?.file && !noLogFile && !Util.OPTIONS?.noLogFile)
+        )) {
             // logger already started
             return;
         }
@@ -562,7 +602,7 @@ export const Util = {
                     }
                 }
                 let stack;
-                /* eslint-disable unicorn/prefer-ternary */
+
                 if (
                     [
                         'ETIMEDOUT',
@@ -577,7 +617,7 @@ export const Util = {
                 } else {
                     stack = ex.stack;
                 }
-                /* eslint-enable unicorn/prefer-ternary */
+
                 myWinston.debug(stack);
             },
 
@@ -607,7 +647,7 @@ export const Util = {
      * @param {string} level of log (error, info, warn)
      * @param {string} type of metadata being referenced
      * @param {string} method name which log was called from
-     * @param {*} payload generic object which details the error
+     * @param {Error|string|object} payload generic object which details the error
      * @param {string} [source] key/id of metadata which relates to error
      * @returns {void}
      */
@@ -642,20 +682,33 @@ export const Util = {
             convertType = true;
             str = JSON.stringify(str);
         }
-        // sort by value length
+        // Sort by search length; retain legacy skipping of falsy scalar values.
         const sortable = [];
         for (const key in obj) {
-            // only push in value if not null
-            if (obj[key]) {
-                sortable.push([key, obj[key]]);
+            const value = obj[key];
+            if (!value) {
+                continue;
             }
+            const isRegex = typeof value === 'object';
+            const search = isRegex ? value.search : value.toString();
+            const template = isRegex
+                ? (value.template ?? '{{{' + key + '}}}')
+                : '{{{' + key + '}}}';
+            const sortLength = isRegex
+                ? search.length
+                : typeof value === 'string'
+                  ? value.length
+                  : undefined;
+            sortable.push({ search, template, isRegex, sortLength });
         }
 
-        sortable.sort((a, b) => b[1].length - a[1].length);
-        for (const pair of sortable) {
-            const escVal = pair[1].toString().replaceAll(/[-/\\^$*+?.()|[\]{}]/g, String.raw`\$&`);
-            const regString = new RegExp(escVal, 'g');
-            str = str.replace(regString, '{{{' + pair[0] + '}}}');
+        sortable.sort((a, b) => b.sortLength - a.sortLength);
+        for (const { search, template, isRegex } of sortable) {
+            const source = isRegex
+                ? search
+                : search.replaceAll(/[-/\\^$*+?.()|[\]{}]/g, String.raw`\$&`);
+            // A replacement string preserves native JavaScript capture expressions.
+            str = str.replaceAll(new RegExp(source, 'g'), template);
         }
         if (convertType) {
             str = JSON.parse(str);
@@ -715,15 +768,15 @@ export const Util = {
         // loop through all metadata types which are being retrieved/deployed
         const subTypeDeps = {};
         for (const typeSubType of typeArr) {
-            const type = typeSubType.split('-')[0];
+            const type = typeSubType.split('-', 1)[0];
             // if they have dependencies then add a dependency pair for each type
             if (MetadataDefinitions[type].dependencies.length > 0) {
                 dependencies.push(
                     ...MetadataDefinitions[type].dependencies.map((dep) => {
                         if (dep.includes('-')) {
                             // log subtypes to be able to replace them if main type is also present
-                            subTypeDeps[dep.split('-')[0]] ||= new Set();
-                            subTypeDeps[dep.split('-')[0]].add(dep);
+                            subTypeDeps[dep.split('-', 1)[0]] ||= new Set();
+                            subTypeDeps[dep.split('-', 1)[0]].add(dep);
                         }
                         return [dep, typeSubType];
                     })
@@ -751,7 +804,7 @@ export const Util = {
         }
 
         // sort list & remove the undefined dependencies
-        const flatList = toposort(dependencies).filter((a) => !!a);
+        const flatList = topologicalSort(dependencies).filter((a) => !!a);
         /** @type {Object.<string, null | Set.<string>>} */
         const setList = {};
         // group subtypes per type
@@ -801,7 +854,7 @@ export const Util = {
      *
      * @param {string} path 'fieldA.fieldB.fieldC'
      * @param {object} obj some parent object
-     * @returns {any} value of obj.path
+     * @returns {ReturnType<JSON['parse']>} value of obj.path
      */
     resolveObjPath(path, obj) {
         return path.split('.').reduce((prev, curr) => (prev ? prev[curr] : null), obj);
@@ -922,7 +975,7 @@ export const Util = {
     /**
      * helper for MetadataType class to issue a similar error message for unsupported methods
      *
-     * @param {any} definition type definition object
+     * @param {{type:string, keyField:string, nameField:string}} definition type definition object
      * @param {string} method name of the method thats not supported
      * @param {MetadataTypeItem} [item] metadata item
      */
@@ -934,33 +987,33 @@ export const Util = {
 
     // defined colors for logging things in different colors
     color: {
-        reset: '\x1B[0m',
-        dim: '\x1B[2m',
-        bright: '\x1B[1m',
-        underscore: '\x1B[4m',
-        blink: '\x1B[5m',
-        reverse: '\x1B[7m',
-        hidden: '\x1B[8m',
-
-        fgBlack: '\x1B[30m',
-        fgRed: '\x1B[31m',
-        fgGreen: '\x1B[32m',
-        fgYellow: '\x1B[33m',
-        fgBlue: '\x1B[34m',
-        fgMagenta: '\x1B[35m',
-        fgCyan: '\x1B[36m',
-        fgWhite: '\x1B[37m',
-        fgGray: '\x1B[90m',
-
-        bgBlack: '\x1B[40m',
-        bgRed: '\x1B[41m',
-        bgGreen: '\x1B[42m',
-        bgYellow: '\x1B[43m',
-        bgBlue: '\x1B[44m',
-        bgMagenta: '\x1B[45m',
-        bgCyan: '\x1B[46m',
-        bgWhite: '\x1B[47m',
-        bgGray: '\x1B[100m',
+        reset: '\u{1B}[0m',
+        dim: '\u{1B}[2m',
+        bright: '\u{1B}[1m',
+        underscore: '\u{1B}[4m',
+        blink: '\u{1B}[5m',
+        reverse: '\u{1B}[7m',
+        hidden: '\u{1B}[8m',
+
+        fgBlack: '\u{1B}[30m',
+        fgRed: '\u{1B}[31m',
+        fgGreen: '\u{1B}[32m',
+        fgYellow: '\u{1B}[33m',
+        fgBlue: '\u{1B}[34m',
+        fgMagenta: '\u{1B}[35m',
+        fgCyan: '\u{1B}[36m',
+        fgWhite: '\u{1B}[37m',
+        fgGray: '\u{1B}[90m',
+
+        bgBlack: '\u{1B}[40m',
+        bgRed: '\u{1B}[41m',
+        bgGreen: '\u{1B}[42m',
+        bgYellow: '\u{1B}[43m',
+        bgBlue: '\u{1B}[44m',
+        bgMagenta: '\u{1B}[45m',
+        bgCyan: '\u{1B}[46m',
+        bgWhite: '\u{1B}[47m',
+        bgGray: '\u{1B}[100m',
     },
 
     /**
@@ -970,13 +1023,16 @@ export const Util = {
      * @returns {string} gray msg
      */
     getGrayMsg(msg) {
+        if (Util.OPTIONS?.noLogColors) {
+            return msg;
+        }
         return `${Util.color.dim}${msg}${Util.color.reset}`;
     },
 
     /**
      * helper that returns the prefix of item specific log messages
      *
-     * @param {any} definition metadata definition
+     * @param {{type:string, keyField:string, nameField:string}} definition metadata definition
      * @param {MetadataTypeItem} metadataItem metadata item
      * @returns {string} msg prefix
      */
@@ -987,7 +1043,7 @@ export const Util = {
     /**
      * helper that returns the prefix of item specific log messages
      *
-     * @param {any} definition metadata definition
+     * @param {{type:string, keyField:string, nameField:string}} definition metadata definition
      * @param {MetadataTypeItem} metadataItem metadata item
      * @returns {string} key or key/name combo
      */
@@ -998,7 +1054,7 @@ export const Util = {
     /**
      * helper that returns the prefix of item specific log messages
      *
-     * @param {any} definition metadata definition
+     * @param {{type:string, keyField:string, nameField:string}} definition metadata definition
      * @param {MetadataTypeItem} metadataItem metadata item
      * @returns {string} key or key/name combo
      */
@@ -1105,7 +1161,7 @@ export const Util = {
         // 3: the actual code
         if (regexMatches?.length > 1 && regexMatches[3]) {
             // script found
-            /* eslint-disable unicorn/prefer-ternary */
+
             if (regexMatches[3].includes('<script')) {
                 // nested script found
                 return null;
@@ -1113,7 +1169,6 @@ export const Util = {
                 // no nested script found: return the assumed SSJS-code
                 return regexMatches[3];
             }
-            /* eslint-enable unicorn/prefer-ternary */
         }
         // no script found
         return null;
@@ -1302,9 +1357,10 @@ export const Util = {
      * async version of Array.find()
      * returns the first element in the provided array that satisfies the provided testin function
      *
-     * @param {Array} arr your test array
-     * @param {Function} asyncCallback callback
-     * @returns {Promise.<any | undefined>} first element that passed the test
+     * @template T
+     * @param {T[]} arr your test array
+     * @param {(element: T) => Promise.<boolean>} asyncCallback callback
+     * @returns {Promise.<T | undefined>} first element that passed the test
      */
     async findAsync(arr, asyncCallback) {
         for (const element of arr) {
@@ -1328,21 +1384,21 @@ export const Util = {
     /**
      * recursively find all values of the given key in the object
      *
-     * @param {any} object data to search in
+     * @param {object} object data to search in
      * @param {string} key attribute to find
      * @returns {Array} all values of the given key
      */
     findLeafVals(object, key) {
         const values = [];
-        Object.keys(object).map((k) => {
+        for (const k of Object.keys(object)) {
             if (k === key) {
                 values.push(object[k]);
-                return true;
+                continue;
             }
             if (object[k] && typeof object[k] === 'object') {
                 values.push(...this.findLeafVals(object[k], key));
             }
-        });
+        }
         return [...new Set(values.toSorted())];
     },
     /**
@@ -1361,4 +1417,33 @@ export const Util = {
     },
 };
 
-Util.startLogger(false, true);
+/**
+ * Create a synchronous logger facade for imports that run before explicit logger startup.
+ *
+ * @returns {Logger} console-backed logger compatible with the mcdev logger surface
+ */
+function createPreStartLogger() {
+    /** @type {Record<string, (...args: unknown[]) => unknown>} */
+    const logger = {};
+    for (const level of ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly']) {
+        logger[level] = (message) => {
+            if (level === 'error') {
+                Util.signalFatalError();
+            }
+            if (['error', 'warn'].includes(level)) {
+                console.error(message); // eslint-disable-line no-console
+            }
+            return logger;
+        };
+    }
+    logger.errorStack = (ex, message) => {
+        const error = /** @type {SDKError} */ (ex);
+        if (message) {
+            logger.error(
+                `${message}: ${error.message}${error.code ? ' (' + error.code + ')' : ''}`
+            );
+        }
+        logger.debug(error.stack);
+    };
+    return /** @type {Logger} */ (/** @type {unknown} */ (logger));
+}

package/lib/util/validations.js

@@ -1,5 +1,6 @@
 'use strict';
 import path from 'node:path';
+import { pathToFileURL } from 'node:url';
 import { Util } from './util.js';
 
 /**
@@ -12,17 +13,24 @@ import { Util } from './util.js';
 /** @type {validationRuleList} */
 let customRules = {};
 let customRuleImport;
+let customRuleUrl;
 /**
  * helper for validation
  *
+ * @param {string} [root] project root containing the custom validation module
  * @returns {Promise.<void>} -
  */
-async function loadCustomRules() {
+export async function loadCustomRules(root = process.cwd()) {
+    const rulesUrl = pathToFileURL(path.resolve(root, '.mcdev-validations.js')).href;
+    if (customRuleUrl === rulesUrl) {
+        return;
+    }
+    customRuleUrl = rulesUrl;
+    customRuleImport = undefined;
     try {
-        customRuleImport = await import(
-            'file://' + path.join(process.cwd(), '.mcdev-validations.js')
-        );
+        customRuleImport = await import(rulesUrl);
     } catch (ex) {
+        customRuleUrl = undefined;
         if (ex instanceof SyntaxError) {
             Util.logger.error('SyntaxError in .mcdev-validations.js: ' + ex.message);
         } else if (ex.code === 'ERR_MODULE_NOT_FOUND') {
@@ -37,18 +45,16 @@ async function loadCustomRules() {
 }
 /**
  *
- * @param {any} definition type definition
- * @param {any} item MetadataItem
+ * @param {{type:string, keyField:string, nameField:string}} definition type definition
+ * @param {ReturnType<JSON['parse']>} item MetadataItem
  * @param {string} targetDir folder in which the MetadataItem is deployed from (deploy/cred/bu)
  * @param {CodeExtract[]} [codeExtractItemArr] array of code snippets
+ * @param {string} [root] project root containing .mcdev-validations.js
  * @returns {Promise.<validationRuleList>} MetadataItem
  */
-export default async function validation(definition, item, targetDir, codeExtractItemArr) {
+export default async function validation(definition, item, targetDir, codeExtractItemArr, root) {
     try {
-        if (!customRuleImport) {
-            // we might need to re-run this for mocha tests
-            await loadCustomRules();
-        }
+        await loadCustomRules(root);
         customRules = customRuleImport
             ? await customRuleImport.validation(
                   definition,