mcdev 7.3.1 → 7.4.1

package/lib/metadataTypes/MetadataType.js

@@ -14,6 +14,7 @@ import deepEqual from 'deep-equal';
 import pLimit from 'p-limit';
 import Mustache from 'mustache';
 import MetadataTypeInfo from '../MetadataTypeInfo.js';
+import validationsRules from '../util/validations.js';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -127,23 +128,23 @@ class MetadataType {
     /**
      * Deploys metadata
      *
-     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
      * @param {string} retrieveDir directory where metadata after deploy should be saved
      * @returns {Promise.<MetadataTypeMap>} Promise of keyField => metadata map
      */
-    static async deploy(metadata, deployDir, retrieveDir) {
-        const upsertResults = await this.upsert(metadata, deployDir);
-        const savedMetadata = await this.saveResults(upsertResults, retrieveDir, null);
+    static async deploy(metadataMap, deployDir, retrieveDir) {
+        const upsertedMetadataMap = await this.upsert(metadataMap, deployDir);
+        const savedMetadataMap = await this.saveResults(upsertedMetadataMap, retrieveDir, null);
         if (
             this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type) &&
             !this.definition.documentInOneFile
         ) {
             // * do not await here as this might take a while and has no impact on the deploy
             // * this should only be run if documentation is on a per metadata record level. Types that document an overview into a single file will need a full retrieve to work instead
-            this.document(savedMetadata, true);
+            this.document(savedMetadataMap, true);
         }
-        return upsertResults;
+        return upsertedMetadataMap;
     }
 
     /**
@@ -710,12 +711,15 @@ class MetadataType {
      *
      * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
+     * @param {boolean} [runUpsertSequentially] when a type has self-dependencies creates need to run one at a time and created/changed keys need to be cached to ensure following creates/updates have thoses keys available
      * @returns {Promise.<MetadataTypeMap>} keyField => metadata map
      */
-    static async upsert(metadataMap, deployDir) {
+    static async upsert(metadataMap, deployDir, runUpsertSequentially = false) {
         const orignalMetadataMap = structuredClone(metadataMap);
         const metadataToUpdate = [];
         const metadataToCreate = [];
+        let createResults = [];
+        let updateResults = [];
         let filteredByPreDeploy = 0;
         for (const metadataKey in metadataMap) {
             let hasError = false;
@@ -727,6 +731,12 @@ class MetadataType {
                         metadataMap[metadataKey],
                         deployDir
                     );
+
+                    deployableMetadata = await this.validation(
+                        'deploy',
+                        deployableMetadata,
+                        deployDir
+                    );
                 } catch (ex) {
                     // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
                     hasError = true;
@@ -744,13 +754,58 @@ class MetadataType {
                 if (deployableMetadata) {
                     metadataMap[metadataKey] = deployableMetadata;
                     // create normalizedKey off of whats in the json rather than from "metadataKey" because preDeployTasks might have altered something (type asset)
-                    await this.createOrUpdate(
+                    const action = await this.createOrUpdate(
                         metadataMap,
                         metadataKey,
                         hasError,
                         metadataToUpdate,
                         metadataToCreate
                     );
+                    if (runUpsertSequentially) {
+                        if (action === 'create') {
+                            // handle creates sequentially here becasue we might have interdepencies
+                            const result = await this.create(metadataMap[metadataKey], deployDir);
+                            if (result) {
+                                createResults.push(result);
+
+                                // make this newly created item available in cache for other itmes that might reference it
+                                const newObject = {};
+                                newObject[metadataKey] = metadataMap[metadataKey];
+                                cache.mergeMetadata(this.definition.type, newObject);
+                            }
+                        } else if (action === 'update' && !Util.OPTIONS.noUpdate) {
+                            const metadataEntry = metadataToUpdate.find(
+                                (el) =>
+                                    el !== null &&
+                                    el.after[this.definition.keyField] ===
+                                        metadataMap[metadataKey][this.definition.keyField]
+                            );
+                            if (!metadataEntry) {
+                                Util.logger.error(
+                                    ` - ${this.definition.type} ${metadataKey} / ${metadataMap[metadataKey][this.definition.keyField]} not found in update list`
+                                );
+                                continue;
+                            }
+                            // handle updates sequentially here becasue we might have interdepencies
+                            // this is especially important when we use features like --matchName which are updates but change the key
+                            const result = await this.update(
+                                metadataEntry.after,
+                                metadataEntry.before
+                            );
+                            if (result) {
+                                updateResults.push(result);
+
+                                // make this newly created item available in cache for other itmes that might reference it
+                                const newObject = {};
+                                newObject[metadataKey] = structuredClone(metadataMap[metadataKey]);
+                                if (result.objectID) {
+                                    // required for assets
+                                    newObject[metadataKey].objectID = result.objectID;
+                                }
+                                cache.mergeMetadata(this.definition.type, newObject);
+                            }
+                        }
+                    }
                 } else {
                     filteredByPreDeploy++;
                 }
@@ -758,37 +813,40 @@ class MetadataType {
                 Util.logger.errorStack(ex, `Upserting ${this.definition.type} failed`);
             }
         }
-        const createLimit = pLimit(10);
-        const createResults = (
-            await Promise.all(
-                metadataToCreate
-                    .filter((r) => r !== undefined && r !== null)
-                    .map((metadataEntry) =>
-                        createLimit(() => this.create(metadataEntry, deployDir))
-                    )
-            )
-        ).filter((r) => r !== undefined && r !== null);
+        if (!runUpsertSequentially) {
+            // create
+            const createLimit = pLimit(10);
+            createResults = (
+                await Promise.all(
+                    metadataToCreate
+                        .filter((r) => r !== undefined && r !== null)
+                        .map((metadataEntry) =>
+                            createLimit(() => this.create(metadataEntry, deployDir))
+                        )
+                )
+            ).filter((r) => r !== undefined && r !== null);
 
-        if (Util.OPTIONS.noUpdate && metadataToUpdate.length > 0) {
-            Util.logger.info(
-                ` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
-            );
+            // update
+            if (Util.OPTIONS.noUpdate && metadataToUpdate.length) {
+                Util.logger.info(
+                    ` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
+                );
+            } else if (metadataToUpdate.length) {
+                const updateLimit = pLimit(10);
+                updateResults = (
+                    await Promise.all(
+                        metadataToUpdate
+                            .filter((r) => r !== undefined && r !== null)
+                            .map((metadataEntry) =>
+                                updateLimit(() =>
+                                    this.update(metadataEntry.after, metadataEntry.before)
+                                )
+                            )
+                    )
+                ).filter((r) => r !== undefined && r !== null);
+            }
         }
 
-        const updateLimit = pLimit(10);
-        const updateResults = Util.OPTIONS.noUpdate
-            ? []
-            : (
-                  await Promise.all(
-                      metadataToUpdate
-                          .filter((r) => r !== undefined && r !== null)
-                          .map((metadataEntry) =>
-                              updateLimit(() =>
-                                  this.update(metadataEntry.after, metadataEntry.before)
-                              )
-                          )
-                  )
-              ).filter((r) => r !== undefined && r !== null);
         // Logging
         Util.logger.info(
             `${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
@@ -886,6 +944,11 @@ class MetadataType {
         } else if (cacheMatchedByKey || cacheMatchedByName) {
             // normal way of processing update files
             const cachedVersion = cacheMatchedByKey || cacheMatchedByName;
+            if (!cacheMatchedByKey && cacheMatchedByName) {
+                Util.matchedByName[this.definition.type] ||= {};
+                Util.matchedByName[this.definition.type][metadataKey] =
+                    cacheMatchedByName[this.definition.keyField];
+            }
             if (!this.hasChanged(cachedVersion, metadataMap[metadataKey])) {
                 hasError = true;
             }
@@ -1903,6 +1966,11 @@ class MetadataType {
                     filterCounter++;
                     continue;
                 }
+                results[originalKey] = await this.validation(
+                    'retrieve',
+                    results[originalKey],
+                    retrieveDir
+                );
 
                 savedResults[originalKey] = await this.saveToDisk(
                     results,
@@ -2206,6 +2274,11 @@ class MetadataType {
         );
 
         try {
+            await this.validation(
+                'buildDefinition',
+                metadata,
+                Array.isArray(targetDir) ? targetDir[0] : targetDir
+            );
             // write to file
             const targetDirArr = Array.isArray(targetDir) ? targetDir : [targetDir];
             for (const targetDir of targetDirArr) {
@@ -2234,7 +2307,9 @@ class MetadataType {
 
             return { metadata: metadata, type: this.definition.type };
         } catch (ex) {
-            throw new Error(`${this.definition.type}:: ${ex.message}`);
+            Util.logger.error(
+                ` ☇ skipped ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}`
+            );
         }
     }
 
@@ -2702,6 +2777,76 @@ class MetadataType {
 
         return newKey;
     }
+
+    /**
+     * @typedef {'off'|'warn'|'error'} ValidationLevel
+     */
+    /**
+     * @typedef {object} ValidationRules
+     * @property {ValidationLevel} [noGuidKeys] flags metadata that did not get a proper key
+     * @property {ValidationLevel} [noRootFolder] flags metadata that did not get a proper key
+     * @property {{type:string[], options: ValidationRules}[]} [overrides] flags metadata that did not get a proper key
+     */
+
+    /**
+     * Gets executed before deploying metadata
+     *
+     * @param {'retrieve'|'buildDefinition'|'deploy'} method used to select the right config
+     * @param {MetadataTypeItem | CodeExtractItem} item a single metadata item
+     * @param {string} targetDir folder where files for deployment are stored
+     * @returns {Promise.<MetadataTypeItem | CodeExtractItem>} Promise of a single metadata item
+     */
+    static async validation(method, item, targetDir) {
+        if (!this.properties.options?.validation?.[method]) {
+            return item;
+        }
+        /** @type {MetadataTypeItem} */
+        const metadataItem = item.json && item.codeArr ? item.json : item;
+        /** @type {ValidationRules} */
+        const validationConfig = structuredClone(this.properties.options?.validation?.[method]);
+
+        // check if the config contains overrides for the current type
+        const overrides = Array.isArray(validationConfig.overrides)
+            ? validationConfig.overrides
+            : [validationConfig.overrides];
+        if (validationConfig.overrides) {
+            delete validationConfig.overrides;
+            for (const override of overrides) {
+                if (
+                    override.type?.includes(this.definition.type) &&
+                    override.options &&
+                    Object.keys(override.options)
+                ) {
+                    Object.assign(validationConfig, override.options);
+                }
+            }
+        }
+
+        // get default and custom validation rules
+        const definition = this.definition;
+        const validationRules = await validationsRules(this.definition, metadataItem, targetDir);
+
+        // run validation rules
+        for (const rule of Object.keys(validationRules)) {
+            if (
+                validationConfig[rule] &&
+                validationConfig[rule] !== 'off' &&
+                !this.definition.skipValidation?.[rule] &&
+                !(await validationRules[rule].passed())
+            ) {
+                if (!Util.OPTIONS.skipValidation && validationConfig[rule] === 'error') {
+                    throw new Error(validationRules[rule].failedMsg);
+                } else if (Util.OPTIONS.skipValidation || validationConfig[rule] === 'warn') {
+                    Util.logger.warn(
+                        ` - ${this.definition.type} ${metadataItem[this.definition.nameField]} (${
+                            metadataItem[this.definition.keyField]
+                        }): ${validationRules[rule].failedMsg}`
+                    );
+                }
+            }
+        }
+        return item;
+    }
 }
 
 MetadataType.definition = {

package/lib/metadataTypes/definitions/TriggeredSend.definition.js

@@ -67,10 +67,11 @@ export default {
             templating: true,
         },
         BccEmail: {
+            // while this can be retrieved, it seems to be always returned empty
             isCreateable: true,
             isUpdateable: true,
-            retrieving: true,
-            templating: true,
+            retrieving: false,
+            templating: false,
         },
         CategoryID: {
             isCreateable: true,
@@ -79,6 +80,7 @@ export default {
             templating: true,
         },
         CCEmail: {
+            // this field is updatable but not retrievable for some reason
             isCreateable: true,
             isUpdateable: true,
             retrieving: false,

package/lib/util/config.js

@@ -202,7 +202,7 @@ const config = {
                                         Array.isArray(defaultProps[key][subkey])
                                             ? 'Array'
                                             : typeof defaultProps[key][subkey]
-                                    }): ${defaultProps[key][subkey]}`
+                                    }): ${typeof defaultProps[key][subkey] === 'object' ? JSON.stringify(defaultProps[key][subkey]) : defaultProps[key][subkey]}`
                                 );
                                 solutionSet.add(
                                     `Run 'mcdev upgrade' to fix missing or changed configuration options`
@@ -270,15 +270,15 @@ const config = {
                     errorMsgOutput.push(' - ' + msg);
                 }
                 Util.logger.error(errorMsgOutput.join('\n'));
-                if (Util.skipInteraction) {
-                    return false;
-                }
                 Util.logger.info(
                     [
                         'Here is what you can do to fix these issues:',
                         ...Array.from(solutionSet),
                     ].join('\n- ')
                 );
+                if (Util.skipInteraction) {
+                    return false;
+                }
                 const runUpgradeNow = await confirm({
                     message: `Do you want to run 'mcdev upgrade' now?`,
                     default: true,

package/lib/util/init.config.js

@@ -374,7 +374,6 @@ const Init = {
      * @returns {Promise.<boolean>} install successful or error occured
      */
     async _createIdeConfigFile(fileNameArr, relevantForced, boilerplateFileContent) {
-        let update = false;
         const fileName = fileNameArr.join('');
         const boilerplateFileName = path.resolve(
             __dirname,
@@ -409,17 +408,20 @@ const Init = {
                 Util.logger.info(
                     `- ✋  ${fileName} found with differences to the new standard version. We recommend updating it.`
                 );
-                if (!Util.skipInteraction) {
+                if (Util.skipInteraction) {
+                    todo = 'update';
+                } else {
                     const overrideFile = await confirm({
                         message: 'Would you like to update (override) it?',
                         default: true,
                     });
-                    if (!overrideFile) {
+                    if (overrideFile) {
+                        todo = 'update';
+                    } else {
                         // skip override without error
                         return true;
                     }
                 }
-                update = true;
             }
 
             // ensure our update is not leading to data loss in case config files were not versioned correctly by the user
@@ -438,14 +440,16 @@ const Init = {
             if (saveStatus) {
                 Util.logger.info(
                     `- ✔️  ${fileName} ${
-                        update
+                        todo === 'update'
                             ? `updated (we created a backup of the old file under ${fileName + '.BAK'})`
                             : 'created'
                     }`
                 );
                 return true;
             } else {
-                Util.logger.warn(`- ❌  ${fileName} ${update ? 'update' : 'creation'} failed`);
+                Util.logger.warn(
+                    `- ❌  ${fileName} ${todo === 'update' ? 'update' : 'creation'} failed`
+                );
                 return false;
             }
         } else if (todo === 'delete') {

package/lib/util/replaceContentBlockReference.js

@@ -208,50 +208,54 @@ export default class ReplaceContentBlockReference {
      * ensures we cache the right things from disk and if required from server
      *
      * @param {Mcdevrc} properties properties for auth
-saved
      * @param {BuObject} buObject properties for auth
      * @param {boolean} [retrieveSharedOnly] for --dependencies only, do not have to re-retrieve local assets
      * @returns {Promise.<void>} -
      */
-    static async createCacheMap(properties, buObject, retrieveSharedOnly = false) {
+    static async createCache(properties, buObject, retrieveSharedOnly = false) {
         const { localAssets, sharedAssets } = await ReplaceContentBlockReference._retrieveCache(
             buObject,
             properties,
             retrieveSharedOnly
         );
 
-        ReplaceContentBlockReference._createCacheMap(localAssets);
-        ReplaceContentBlockReference._createCacheMap(sharedAssets);
+        ReplaceContentBlockReference.createCacheForMap(localAssets);
+        ReplaceContentBlockReference.createCacheForMap(sharedAssets);
     }
 
     /**
-     * helper for {@link this.createCacheMap} that converts AssetMap into Asset
+     * helper for {@link ReplaceContentBlockReference.createCache} that converts AssetMap into AssetItemSimple entries in this.assetCacheMap
      *
      * @param {AssetMap} metadataMap list of local or shared assets
      */
-    static _createCacheMap(metadataMap) {
+    static createCacheForMap(metadataMap) {
         for (const element of Object.values(metadataMap)) {
             // create actual cache map
             /** @type {AssetItemSimple} */
             const simpleAsset = {
                 id: element.id,
                 key: element.customerKey,
+                // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
                 name: element.r__folder_Path
                     ? element.r__folder_Path.replaceAll('/', '\\') + '\\' + element.name
                     : null,
             };
-            // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
-            this.assetCacheMap.id[simpleAsset.id] = simpleAsset;
-            this.assetCacheMap.key[simpleAsset.key] = simpleAsset;
+            // if this method was filled by Asset.upsert it might have been run before with more accurate (retrieved) data including the id that we do not want to override
+            this.assetCacheMap.key[simpleAsset.key] ||= simpleAsset;
+            if (simpleAsset.id) {
+                // if this method was filled by Asset.upsert it won't have ids
+                this.assetCacheMap.id[simpleAsset.id] = simpleAsset;
+            }
             if (simpleAsset.name) {
                 // while asset without path could still be found via search, it would no longer referencable via ContentBlockByName
-                this.assetCacheMap.name[simpleAsset.name] = simpleAsset;
+                // if this method was filled by Asset.upsert it might have been run before with more accurate (retrieved) data including the id that we do not want to override
+                this.assetCacheMap.name[simpleAsset.name] ||= simpleAsset;
             }
         }
     }
 
     /**
-     * helper for {@link this.createCacheMap}
+     * helper for {@link ReplaceContentBlockReference.createCache}
      *
      * @param {BuObject} buObject references credentials
      * @param {Mcdevrc} properties central properties object

package/lib/util/util.js

@@ -53,6 +53,7 @@ export const Util = {
     packageJsonMcdev: readJsonSync(path.join(__dirname, '../../package.json')),
     OPTIONS: {},
     changedKeysMap: {},
+    matchedByName: {},
 
     /**
      * helper that allows filtering an object by its keys
@@ -575,39 +576,38 @@ export const Util = {
     /**
      * Returns Order in which metadata needs to be retrieved/deployed
      *
-     * @param {string[]} metadataTypes which should be retrieved/deployed
+     * @param {string[]} typeArr which should be retrieved/deployed
      * @returns {Object.<string, string[]>} retrieve/deploy order as array
      */
-    getMetadataHierachy(metadataTypes) {
+    getMetadataHierachy(typeArr) {
         const dependencies = [];
         // loop through all metadata types which are being retrieved/deployed
         const subTypeDeps = {};
-        for (const metadataType of metadataTypes) {
-            const type = metadataType.split('-')[0];
+        for (const typeSubType of typeArr) {
+            const type = typeSubType.split('-')[0];
             // if they have dependencies then add a dependency pair for each type
             if (MetadataDefinitions[type].dependencies.length > 0) {
                 dependencies.push(
                     ...MetadataDefinitions[type].dependencies.map((dep) => {
                         if (dep.includes('-')) {
                             // log subtypes to be able to replace them if main type is also present
-                            subTypeDeps[dep.split('-')[0]] =
-                                subTypeDeps[dep.split('-')[0]] || new Set();
+                            subTypeDeps[dep.split('-')[0]] ||= new Set();
                             subTypeDeps[dep.split('-')[0]].add(dep);
                         }
-                        return [dep, metadataType];
+                        return [dep, typeSubType];
                     })
                 );
             }
             // if they have no dependencies then just add them with undefined.
             else {
-                dependencies.push([undefined, metadataType]);
+                dependencies.push([undefined, typeSubType]);
             }
         }
         const allDeps = dependencies.map((dep) => dep[0]);
         // remove subtypes if main type is in the list
         for (const type of Object.keys(subTypeDeps)
             // only look at subtype deps that are also supposed to be retrieved or cached fully
-            .filter((type) => metadataTypes.includes(type) || allDeps.includes(type))) {
+            .filter((type) => typeArr.includes(type) || allDeps.includes(type))) {
             // convert set into array to walk its elements
             for (const subType of subTypeDeps[type]) {
                 for (const item of dependencies) {
@@ -1118,6 +1118,40 @@ export const Util = {
             ? []
             : [array.splice(0, chunk_size)].concat(this.chunk(array, chunk_size));
     },
+    /**
+     * recursively find all values of the given key in the object
+     *
+     * @param {any} object data to search in
+     * @param {string} key attribute to find
+     * @returns {Array} all values of the given key
+     */
+    findLeafVals(object, key) {
+        const values = [];
+        Object.keys(object).map((k) => {
+            if (k === key) {
+                values.push(object[k]);
+                return true;
+            }
+            if (object[k] && typeof object[k] === 'object') {
+                values.push(...this.findLeafVals(object[k], key));
+            }
+        });
+        return [...new Set(values.sort())];
+    },
+    /**
+     * helper that returns a new object with sorted attributes of the given object
+     *
+     * @param {object} obj object with unsorted attributes
+     * @returns {object} obj but with sorted attributes
+     */
+    sortObjectAttributes(obj) {
+        return Object.keys(obj)
+            .sort()
+            .reduce((acc, key) => {
+                acc[key] = obj[key];
+                return acc;
+            }, {});
+    },
 };
 
 Util.startLogger(false, true);

package/lib/util/validations.js

@@ -0,0 +1,66 @@
+'use strict';
+import path from 'node:path';
+import { Util } from './util.js';
+
+let customRules = {};
+let customRuleImport;
+try {
+    customRuleImport = await import('file://' + path.join(process.cwd(), '.mcdev-validations.js'));
+} catch {
+    Util.logger.debug('.mcdev-validations.js not found');
+}
+/**
+ *
+ * @param {any} definition type definition
+ * @param {any} item MetadataItem
+ * @returns {Promise.<any>} MetadataItem
+ * @param {string} targetDir folder in which the MetadataItem is deployed from (deploy/cred/bu)
+ */
+export default async function validation(definition, item, targetDir) {
+    try {
+        if (customRuleImport) {
+            customRules = customRuleImport
+                ? await customRuleImport.validation(definition, item, targetDir, Util)
+                : {};
+        }
+    } catch (ex) {
+        Util.logger.error(
+            'Could not load custom validation rules from .mcdev-validations.js: ' + ex.message
+        );
+    }
+    const defaultRules = {
+        noGuidKeys: {
+            failedMsg: 'Please update the key to a readable value. Currently still in GUID format.',
+            /**
+             * @returns {boolean} true=test passed
+             */
+            passed: function () {
+                const key = item[definition.keyField];
+                if (key) {
+                    const regex = /^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$/i;
+                    return !regex.test(String(key).toLowerCase());
+                } else {
+                    Util.logger.debug('validation-noGuidKeys: key not found');
+                    return true;
+                }
+            },
+        },
+        noRootFolder: {
+            failedMsg: 'Root folder not allowed. Current folder: ' + item.r__folder_Path,
+            /**
+             * @returns {boolean} true=test passed
+             */
+            passed: function () {
+                /** @type {string} */
+                const folderPath = item.r__folder_Path;
+                if (!folderPath) {
+                    // some types do not support folders
+                    return true;
+                }
+
+                return folderPath.includes('/');
+            },
+        },
+    };
+    return Object.assign({}, defaultRules, customRules);
+}