mcdev 7.3.0 → 7.4.0

package/lib/metadataTypes/MetadataType.js

@@ -14,6 +14,7 @@ import deepEqual from 'deep-equal';
 import pLimit from 'p-limit';
 import Mustache from 'mustache';
 import MetadataTypeInfo from '../MetadataTypeInfo.js';
+import validationsRules from '../util/validations.js';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -127,23 +128,23 @@ class MetadataType {
     /**
      * Deploys metadata
      *
-     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
      * @param {string} retrieveDir directory where metadata after deploy should be saved
      * @returns {Promise.<MetadataTypeMap>} Promise of keyField => metadata map
      */
-    static async deploy(metadata, deployDir, retrieveDir) {
-        const upsertResults = await this.upsert(metadata, deployDir);
-        const savedMetadata = await this.saveResults(upsertResults, retrieveDir, null);
+    static async deploy(metadataMap, deployDir, retrieveDir) {
+        const upsertedMetadataMap = await this.upsert(metadataMap, deployDir);
+        const savedMetadataMap = await this.saveResults(upsertedMetadataMap, retrieveDir, null);
         if (
             this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type) &&
             !this.definition.documentInOneFile
         ) {
             // * do not await here as this might take a while and has no impact on the deploy
             // * this should only be run if documentation is on a per metadata record level. Types that document an overview into a single file will need a full retrieve to work instead
-            this.document(savedMetadata, true);
+            this.document(savedMetadataMap, true);
         }
-        return upsertResults;
+        return upsertedMetadataMap;
     }
 
     /**
@@ -710,12 +711,15 @@ class MetadataType {
      *
      * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
+     * @param {boolean} [runUpsertSequentially] when a type has self-dependencies creates need to run one at a time and created/changed keys need to be cached to ensure following creates/updates have thoses keys available
      * @returns {Promise.<MetadataTypeMap>} keyField => metadata map
      */
-    static async upsert(metadataMap, deployDir) {
+    static async upsert(metadataMap, deployDir, runUpsertSequentially = false) {
         const orignalMetadataMap = structuredClone(metadataMap);
         const metadataToUpdate = [];
         const metadataToCreate = [];
+        let createResults = [];
+        let updateResults = [];
         let filteredByPreDeploy = 0;
         for (const metadataKey in metadataMap) {
             let hasError = false;
@@ -727,6 +731,12 @@ class MetadataType {
                         metadataMap[metadataKey],
                         deployDir
                     );
+
+                    deployableMetadata = await this.validation(
+                        'deploy',
+                        deployableMetadata,
+                        deployDir
+                    );
                 } catch (ex) {
                     // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
                     hasError = true;
@@ -744,13 +754,58 @@ class MetadataType {
                 if (deployableMetadata) {
                     metadataMap[metadataKey] = deployableMetadata;
                     // create normalizedKey off of whats in the json rather than from "metadataKey" because preDeployTasks might have altered something (type asset)
-                    await this.createOrUpdate(
+                    const action = await this.createOrUpdate(
                         metadataMap,
                         metadataKey,
                         hasError,
                         metadataToUpdate,
                         metadataToCreate
                     );
+                    if (runUpsertSequentially) {
+                        if (action === 'create') {
+                            // handle creates sequentially here becasue we might have interdepencies
+                            const result = await this.create(metadataMap[metadataKey], deployDir);
+                            if (result) {
+                                createResults.push(result);
+
+                                // make this newly created item available in cache for other itmes that might reference it
+                                const newObject = {};
+                                newObject[metadataKey] = metadataMap[metadataKey];
+                                cache.mergeMetadata(this.definition.type, newObject);
+                            }
+                        } else if (action === 'update' && !Util.OPTIONS.noUpdate) {
+                            const metadataEntry = metadataToUpdate.find(
+                                (el) =>
+                                    el !== null &&
+                                    el.after[this.definition.keyField] ===
+                                        metadataMap[metadataKey][this.definition.keyField]
+                            );
+                            if (!metadataEntry) {
+                                Util.logger.error(
+                                    ` - ${this.definition.type} ${metadataKey} / ${metadataMap[metadataKey][this.definition.keyField]} not found in update list`
+                                );
+                                continue;
+                            }
+                            // handle updates sequentially here becasue we might have interdepencies
+                            // this is especially important when we use features like --matchName which are updates but change the key
+                            const result = await this.update(
+                                metadataEntry.after,
+                                metadataEntry.before
+                            );
+                            if (result) {
+                                updateResults.push(result);
+
+                                // make this newly created item available in cache for other itmes that might reference it
+                                const newObject = {};
+                                newObject[metadataKey] = structuredClone(metadataMap[metadataKey]);
+                                if (result.objectID) {
+                                    // required for assets
+                                    newObject[metadataKey].objectID = result.objectID;
+                                }
+                                cache.mergeMetadata(this.definition.type, newObject);
+                            }
+                        }
+                    }
                 } else {
                     filteredByPreDeploy++;
                 }
@@ -758,37 +813,40 @@ class MetadataType {
                 Util.logger.errorStack(ex, `Upserting ${this.definition.type} failed`);
             }
         }
-        const createLimit = pLimit(10);
-        const createResults = (
-            await Promise.all(
-                metadataToCreate
-                    .filter((r) => r !== undefined && r !== null)
-                    .map((metadataEntry) =>
-                        createLimit(() => this.create(metadataEntry, deployDir))
-                    )
-            )
-        ).filter((r) => r !== undefined && r !== null);
+        if (!runUpsertSequentially) {
+            // create
+            const createLimit = pLimit(10);
+            createResults = (
+                await Promise.all(
+                    metadataToCreate
+                        .filter((r) => r !== undefined && r !== null)
+                        .map((metadataEntry) =>
+                            createLimit(() => this.create(metadataEntry, deployDir))
+                        )
+                )
+            ).filter((r) => r !== undefined && r !== null);
 
-        if (Util.OPTIONS.noUpdate && metadataToUpdate.length > 0) {
-            Util.logger.info(
-                ` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
-            );
+            // update
+            if (Util.OPTIONS.noUpdate && metadataToUpdate.length) {
+                Util.logger.info(
+                    ` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
+                );
+            } else if (metadataToUpdate.length) {
+                const updateLimit = pLimit(10);
+                updateResults = (
+                    await Promise.all(
+                        metadataToUpdate
+                            .filter((r) => r !== undefined && r !== null)
+                            .map((metadataEntry) =>
+                                updateLimit(() =>
+                                    this.update(metadataEntry.after, metadataEntry.before)
+                                )
+                            )
+                    )
+                ).filter((r) => r !== undefined && r !== null);
+            }
         }
 
-        const updateLimit = pLimit(10);
-        const updateResults = Util.OPTIONS.noUpdate
-            ? []
-            : (
-                  await Promise.all(
-                      metadataToUpdate
-                          .filter((r) => r !== undefined && r !== null)
-                          .map((metadataEntry) =>
-                              updateLimit(() =>
-                                  this.update(metadataEntry.after, metadataEntry.before)
-                              )
-                          )
-                  )
-              ).filter((r) => r !== undefined && r !== null);
         // Logging
         Util.logger.info(
             `${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
@@ -886,6 +944,11 @@ class MetadataType {
         } else if (cacheMatchedByKey || cacheMatchedByName) {
             // normal way of processing update files
             const cachedVersion = cacheMatchedByKey || cacheMatchedByName;
+            if (!cacheMatchedByKey && cacheMatchedByName) {
+                Util.matchedByName[this.definition.type] ||= {};
+                Util.matchedByName[this.definition.type][metadataKey] =
+                    cacheMatchedByName[this.definition.keyField];
+            }
             if (!this.hasChanged(cachedVersion, metadataMap[metadataKey])) {
                 hasError = true;
             }
@@ -1903,6 +1966,11 @@ class MetadataType {
                     filterCounter++;
                     continue;
                 }
+                results[originalKey] = await this.validation(
+                    'retrieve',
+                    results[originalKey],
+                    retrieveDir
+                );
 
                 savedResults[originalKey] = await this.saveToDisk(
                     results,
@@ -2206,6 +2274,11 @@ class MetadataType {
         );
 
         try {
+            await this.validation(
+                'buildDefinition',
+                metadata,
+                Array.isArray(targetDir) ? targetDir[0] : targetDir
+            );
             // write to file
             const targetDirArr = Array.isArray(targetDir) ? targetDir : [targetDir];
             for (const targetDir of targetDirArr) {
@@ -2234,7 +2307,9 @@ class MetadataType {
 
             return { metadata: metadata, type: this.definition.type };
         } catch (ex) {
-            throw new Error(`${this.definition.type}:: ${ex.message}`);
+            Util.logger.error(
+                ` ☇ skipped ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}`
+            );
         }
     }
 
@@ -2702,6 +2777,76 @@ class MetadataType {
 
         return newKey;
     }
+
+    /**
+     * @typedef {'off'|'warn'|'error'} ValidationLevel
+     */
+    /**
+     * @typedef {object} ValidationRules
+     * @property {ValidationLevel} [noGuidKeys] flags metadata that did not get a proper key
+     * @property {ValidationLevel} [noRootFolder] flags metadata that did not get a proper key
+     * @property {{type:string[], options: ValidationRules}[]} [overrides] flags metadata that did not get a proper key
+     */
+
+    /**
+     * Gets executed before deploying metadata
+     *
+     * @param {'retrieve'|'buildDefinition'|'deploy'} method used to select the right config
+     * @param {MetadataTypeItem | CodeExtractItem} item a single metadata item
+     * @param {string} targetDir folder where files for deployment are stored
+     * @returns {Promise.<MetadataTypeItem | CodeExtractItem>} Promise of a single metadata item
+     */
+    static async validation(method, item, targetDir) {
+        if (!this.properties.options?.validation?.[method]) {
+            return item;
+        }
+        /** @type {MetadataTypeItem} */
+        const metadataItem = item.json && item.codeArr ? item.json : item;
+        /** @type {ValidationRules} */
+        const validationConfig = structuredClone(this.properties.options?.validation?.[method]);
+
+        // check if the config contains overrides for the current type
+        const overrides = Array.isArray(validationConfig.overrides)
+            ? validationConfig.overrides
+            : [validationConfig.overrides];
+        if (validationConfig.overrides) {
+            delete validationConfig.overrides;
+            for (const override of overrides) {
+                if (
+                    override.type?.includes(this.definition.type) &&
+                    override.options &&
+                    Object.keys(override.options)
+                ) {
+                    Object.assign(validationConfig, override.options);
+                }
+            }
+        }
+
+        // get default and custom validation rules
+        const definition = this.definition;
+        const validationRules = await validationsRules(this.definition, metadataItem, targetDir);
+
+        // run validation rules
+        for (const rule of Object.keys(validationRules)) {
+            if (
+                validationConfig[rule] &&
+                validationConfig[rule] !== 'off' &&
+                !this.definition.skipValidation?.[rule] &&
+                !(await validationRules[rule].passed())
+            ) {
+                if (!Util.OPTIONS.skipValidation && validationConfig[rule] === 'error') {
+                    throw new Error(validationRules[rule].failedMsg);
+                } else if (Util.OPTIONS.skipValidation || validationConfig[rule] === 'warn') {
+                    Util.logger.warn(
+                        ` - ${this.definition.type} ${metadataItem[this.definition.nameField]} (${
+                            metadataItem[this.definition.keyField]
+                        }): ${validationRules[rule].failedMsg}`
+                    );
+                }
+            }
+        }
+        return item;
+    }
 }
 
 MetadataType.definition = {

package/lib/metadataTypes/Role.js

@@ -73,41 +73,6 @@ class Role extends MetadataType {
         const results = await this.client.soap.retrieve('Role', fields, requestParams);
 
         const parsed = this.parseResponseBody(results);
-        if (!retrieveDir) {
-            // retrieve "Marketing Cloud%" roles not returned by SOAP API
-            const { roles, timeZones } = await this.client.rest.get(
-                '/platform/v1/setup/quickflow/data'
-            );
-            // this endpoint does not provide keys
-            const roleNameKeyMap = {
-                'Marketing Cloud Administrator': 'SYS_DEF_IMHADMIN',
-                'Marketing Cloud Channel Manager': 'SYS_DEF_CHANNELMANAGER',
-                'Marketing Cloud Content Editor/Publisher': 'SYS_DEF_CONTENTEDIT',
-                'Marketing Cloud Security Administrator': 'SYS_DEF_SECURITYADMIN',
-                'Marketing Cloud Viewer': 'SYS_DEF_VIEWER',
-            };
-            for (const role of roles) {
-                if (roleNameKeyMap[role.roleName]) {
-                    parsed[roleNameKeyMap[role.roleName]] = {
-                        CustomerKey: roleNameKeyMap[role.roleName],
-                        Name: role.roleName,
-                        ObjectID: role.roleID,
-                        Desscription: role.description,
-                        CreatedDate: '2012-02-21T02:09:19.983',
-                        IsSystemDefined: true,
-                        c__notAssignable: true,
-                    };
-                }
-            }
-            // the languages object is incomplete. the actual list is much longer --> ignoring it here
-            // convert timeZones to object
-            const timeZonesObj = {};
-            for (const timeZone of timeZones) {
-                timeZonesObj[timeZone.id] = timeZone;
-            }
-            // cache timeZones to share it with other type-classes
-            cache.setMetadata('_timezone', timeZonesObj);
-        }
         if (retrieveDir) {
             const savedMetadata = await super.saveResults(parsed, retrieveDir, null);
             Util.logger.info(
@@ -117,9 +82,56 @@ class Role extends MetadataType {
 
             await this.runDocumentOnRetrieve(key, savedMetadata);
         }
+
+        await this.cacheDefaultRolesAndTimezones(parsed);
+
         return { metadata: parsed, type: this.definition.type };
     }
 
+    /**
+     * adds default roles to the list of retrieved roles for proper caching (but not storing)
+     * also caches available timezones for retrieve-user
+     *
+     * @param {MetadataTypeMap} parsed list or previously retrieved items as reference
+     */
+    static async cacheDefaultRolesAndTimezones(parsed) {
+        // retrieve "Marketing Cloud%" roles not returned by SOAP API for cache (required by retrieve-user)
+        // also cache available timezones for retrieve-user
+        Util.logger.info(Util.getGrayMsg(' - Caching default roles and timezones'));
+        const { roles, timeZones } = await this.client.rest.get(
+            '/platform/v1/setup/quickflow/data'
+        );
+        // this endpoint does not provide keys
+        const roleNameKeyMap = {
+            'Marketing Cloud Administrator': 'SYS_DEF_IMHADMIN',
+            'Marketing Cloud Channel Manager': 'SYS_DEF_CHANNELMANAGER',
+            'Marketing Cloud Content Editor/Publisher': 'SYS_DEF_CONTENTEDIT',
+            'Marketing Cloud Security Administrator': 'SYS_DEF_SECURITYADMIN',
+            'Marketing Cloud Viewer': 'SYS_DEF_VIEWER',
+        };
+        for (const role of roles) {
+            if (roleNameKeyMap[role.roleName]) {
+                parsed[roleNameKeyMap[role.roleName]] = {
+                    CustomerKey: roleNameKeyMap[role.roleName],
+                    Name: role.roleName,
+                    ObjectID: role.roleID,
+                    Desscription: role.description,
+                    CreatedDate: '2012-02-21T02:09:19.983',
+                    IsSystemDefined: true,
+                    c__notAssignable: true,
+                };
+            }
+        }
+        // the languages object is incomplete. the actual list is much longer --> ignoring it here
+        // convert timeZones to object
+        const timeZonesObj = {};
+        for (const timeZone of timeZones) {
+            timeZonesObj[timeZone.id] = timeZone;
+        }
+        // cache timeZones to share it with other type-classes
+        cache.setMetadata('_timezone', timeZonesObj);
+    }
+
     /**
      * Gets executed before deploying metadata
      *

package/lib/util/init.config.js

@@ -374,7 +374,6 @@ const Init = {
      * @returns {Promise.<boolean>} install successful or error occured
      */
     async _createIdeConfigFile(fileNameArr, relevantForced, boilerplateFileContent) {
-        let update = false;
         const fileName = fileNameArr.join('');
         const boilerplateFileName = path.resolve(
             __dirname,
@@ -409,17 +408,20 @@ const Init = {
                 Util.logger.info(
                     `- ✋  ${fileName} found with differences to the new standard version. We recommend updating it.`
                 );
-                if (!Util.skipInteraction) {
+                if (Util.skipInteraction) {
+                    todo = 'update';
+                } else {
                     const overrideFile = await confirm({
                         message: 'Would you like to update (override) it?',
                         default: true,
                     });
-                    if (!overrideFile) {
+                    if (overrideFile) {
+                        todo = 'update';
+                    } else {
                         // skip override without error
                         return true;
                     }
                 }
-                update = true;
             }
 
             // ensure our update is not leading to data loss in case config files were not versioned correctly by the user
@@ -438,14 +440,16 @@ const Init = {
             if (saveStatus) {
                 Util.logger.info(
                     `- ✔️  ${fileName} ${
-                        update
+                        todo === 'update'
                             ? `updated (we created a backup of the old file under ${fileName + '.BAK'})`
                             : 'created'
                     }`
                 );
                 return true;
             } else {
-                Util.logger.warn(`- ❌  ${fileName} ${update ? 'update' : 'creation'} failed`);
+                Util.logger.warn(
+                    `- ❌  ${fileName} ${todo === 'update' ? 'update' : 'creation'} failed`
+                );
                 return false;
             }
         } else if (todo === 'delete') {

package/lib/util/replaceContentBlockReference.js

@@ -208,50 +208,54 @@ export default class ReplaceContentBlockReference {
      * ensures we cache the right things from disk and if required from server
      *
      * @param {Mcdevrc} properties properties for auth
-saved
      * @param {BuObject} buObject properties for auth
      * @param {boolean} [retrieveSharedOnly] for --dependencies only, do not have to re-retrieve local assets
      * @returns {Promise.<void>} -
      */
-    static async createCacheMap(properties, buObject, retrieveSharedOnly = false) {
+    static async createCache(properties, buObject, retrieveSharedOnly = false) {
         const { localAssets, sharedAssets } = await ReplaceContentBlockReference._retrieveCache(
             buObject,
             properties,
             retrieveSharedOnly
         );
 
-        ReplaceContentBlockReference._createCacheMap(localAssets);
-        ReplaceContentBlockReference._createCacheMap(sharedAssets);
+        ReplaceContentBlockReference.createCacheForMap(localAssets);
+        ReplaceContentBlockReference.createCacheForMap(sharedAssets);
     }
 
     /**
-     * helper for {@link this.createCacheMap} that converts AssetMap into Asset
+     * helper for {@link ReplaceContentBlockReference.createCache} that converts AssetMap into AssetItemSimple entries in this.assetCacheMap
      *
      * @param {AssetMap} metadataMap list of local or shared assets
      */
-    static _createCacheMap(metadataMap) {
+    static createCacheForMap(metadataMap) {
         for (const element of Object.values(metadataMap)) {
             // create actual cache map
             /** @type {AssetItemSimple} */
             const simpleAsset = {
                 id: element.id,
                 key: element.customerKey,
+                // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
                 name: element.r__folder_Path
                     ? element.r__folder_Path.replaceAll('/', '\\') + '\\' + element.name
                     : null,
             };
-            // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
-            this.assetCacheMap.id[simpleAsset.id] = simpleAsset;
-            this.assetCacheMap.key[simpleAsset.key] = simpleAsset;
+            // if this method was filled by Asset.upsert it might have been run before with more accurate (retrieved) data including the id that we do not want to override
+            this.assetCacheMap.key[simpleAsset.key] ||= simpleAsset;
+            if (simpleAsset.id) {
+                // if this method was filled by Asset.upsert it won't have ids
+                this.assetCacheMap.id[simpleAsset.id] = simpleAsset;
+            }
             if (simpleAsset.name) {
                 // while asset without path could still be found via search, it would no longer referencable via ContentBlockByName
-                this.assetCacheMap.name[simpleAsset.name] = simpleAsset;
+                // if this method was filled by Asset.upsert it might have been run before with more accurate (retrieved) data including the id that we do not want to override
+                this.assetCacheMap.name[simpleAsset.name] ||= simpleAsset;
             }
         }
     }
 
     /**
-     * helper for {@link this.createCacheMap}
+     * helper for {@link ReplaceContentBlockReference.createCache}
      *
      * @param {BuObject} buObject references credentials
      * @param {Mcdevrc} properties central properties object

package/lib/util/util.js

@@ -53,6 +53,7 @@ export const Util = {
     packageJsonMcdev: readJsonSync(path.join(__dirname, '../../package.json')),
     OPTIONS: {},
     changedKeysMap: {},
+    matchedByName: {},
 
     /**
      * helper that allows filtering an object by its keys
@@ -575,39 +576,38 @@ export const Util = {
     /**
      * Returns Order in which metadata needs to be retrieved/deployed
      *
-     * @param {string[]} metadataTypes which should be retrieved/deployed
+     * @param {string[]} typeArr which should be retrieved/deployed
      * @returns {Object.<string, string[]>} retrieve/deploy order as array
      */
-    getMetadataHierachy(metadataTypes) {
+    getMetadataHierachy(typeArr) {
         const dependencies = [];
         // loop through all metadata types which are being retrieved/deployed
         const subTypeDeps = {};
-        for (const metadataType of metadataTypes) {
-            const type = metadataType.split('-')[0];
+        for (const typeSubType of typeArr) {
+            const type = typeSubType.split('-')[0];
             // if they have dependencies then add a dependency pair for each type
             if (MetadataDefinitions[type].dependencies.length > 0) {
                 dependencies.push(
                     ...MetadataDefinitions[type].dependencies.map((dep) => {
                         if (dep.includes('-')) {
                             // log subtypes to be able to replace them if main type is also present
-                            subTypeDeps[dep.split('-')[0]] =
-                                subTypeDeps[dep.split('-')[0]] || new Set();
+                            subTypeDeps[dep.split('-')[0]] ||= new Set();
                             subTypeDeps[dep.split('-')[0]].add(dep);
                         }
-                        return [dep, metadataType];
+                        return [dep, typeSubType];
                     })
                 );
             }
             // if they have no dependencies then just add them with undefined.
             else {
-                dependencies.push([undefined, metadataType]);
+                dependencies.push([undefined, typeSubType]);
             }
         }
         const allDeps = dependencies.map((dep) => dep[0]);
         // remove subtypes if main type is in the list
         for (const type of Object.keys(subTypeDeps)
             // only look at subtype deps that are also supposed to be retrieved or cached fully
-            .filter((type) => metadataTypes.includes(type) || allDeps.includes(type))) {
+            .filter((type) => typeArr.includes(type) || allDeps.includes(type))) {
             // convert set into array to walk its elements
             for (const subType of subTypeDeps[type]) {
                 for (const item of dependencies) {
@@ -1118,6 +1118,26 @@ export const Util = {
             ? []
             : [array.splice(0, chunk_size)].concat(this.chunk(array, chunk_size));
     },
+    /**
+     * recursively find all values of the given key in the object
+     *
+     * @param {any} object data to search in
+     * @param {string} key attribute to find
+     * @returns {Array} all values of the given key
+     */
+    findLeafVals(object, key) {
+        const values = [];
+        Object.keys(object).map((k) => {
+            if (k === key) {
+                values.push(object[k]);
+                return true;
+            }
+            if (object[k] && typeof object[k] === 'object') {
+                values.push(...this.findLeafVals(object[k], key));
+            }
+        });
+        return [...new Set(values.sort())];
+    },
 };
 
 Util.startLogger(false, true);