mcdev 7.3.0 → 7.4.0

package/lib/cli.js

@@ -163,7 +163,14 @@ yargs(hideBin(process.argv))
                     group: 'Options for deploy:',
                     describe:
                         'Some metadata types allow updating resources despite a key mismatch by matching the name. That avoids clean-ups on all BUs but instead allows you to continously get higher environmetns into a better shape.',
+                })
+                .option('skipValidation', {
+                    alias: 'sv',
+                    group: 'Options for deploy:',
+                    describe:
+                        'allows reducing validation rules from error to warn to handle edge cases',
                 }),
+
         (argv) => {
             Mcdev.setOptions(argv);
 
@@ -395,6 +402,12 @@ yargs(hideBin(process.argv))
                     group: 'Options for build:',
                     describe:
                         're-retrieves potentially relevant metadata before running buildTemplate (all if --dependencies is used)',
+                })
+                .option('skipValidation', {
+                    alias: 'sv',
+                    group: 'Options for build:',
+                    describe:
+                        'allows reducing validation rules from error to warn to handle edge cases',
                 }),
         (argv) => {
             Mcdev.setOptions(argv);
@@ -521,6 +534,12 @@ yargs(hideBin(process.argv))
                     group: 'Options for buildDefinition:',
                     describe: 'market used for building deployable definition',
                 })
+                .option('skipValidation', {
+                    alias: 'sv',
+                    group: 'Options for buildDefinition:',
+                    describe:
+                        'allows reducing validation rules from error to warn to handle edge cases',
+                })
                 .check((argv) => {
                     if (!argv.MARKET && !argv.market) {
                         throw new Error(
@@ -572,6 +591,12 @@ yargs(hideBin(process.argv))
                     alias: 'm',
                     group: 'Options for buildDefinitionBulk:',
                     describe: 'type:templateName combos to build template for',
+                })
+                .option('skipValidation', {
+                    alias: 'sv',
+                    group: 'Options for deploy:',
+                    describe:
+                        'allows reducing validation rules from error to warn to handle edge cases',
                 }),
         (argv) => {
             Mcdev.setOptions(argv);
@@ -679,8 +704,8 @@ yargs(hideBin(process.argv))
         }
     )
     .command(
-        ['execute <BU> [TYPE] [KEY]', 'exec', 'start'],
-        'executes the entity (query/journey/automation etc.)',
+        ['execute <BU> [TYPE] [KEY]', 'exec', 'start', 'resume'],
+        'executes the entity',
         (yargs) =>
             yargs
                 .positional('BU', {
@@ -733,7 +758,7 @@ yargs(hideBin(process.argv))
         }
     )
     .command(
-        ['publish <BU> [TYPE] [KEY]'],
+        ['publish <BU> [TYPE] [KEY]', 'activate'],
         'publishes the entity',
         (yargs) =>
             yargs

package/lib/index.js

@@ -113,6 +113,7 @@ class Mcdev {
             'skipInteraction',
             'skipRetrieve',
             'skipStatusCheck',
+            'skipValidation',
             '_runningTest',
             '_welcomeMessageShown',
         ];
@@ -939,7 +940,7 @@ class Mcdev {
             'Searching for additional dependencies that were linked via ContentBlockByKey, ContentBlockByName and ContentBlockById'
         );
 
-        await ReplaceContentBlockReference.createCacheMap(properties, buObject, true);
+        await ReplaceContentBlockReference.createCache(properties, buObject, true);
 
         // because we re-use the replaceReference logic here we need to manually set this value
         /** @type {ContentBlockConversionTypes[]} */
@@ -2073,7 +2074,7 @@ class Mcdev {
             buObject.businessUnit,
         ]);
 
-        await ReplaceContentBlockReference.createCacheMap(properties, buObject);
+        await ReplaceContentBlockReference.createCache(properties, buObject);
 
         try {
             let metadataMap;

package/lib/metadataTypes/Asset.js

@@ -9,6 +9,7 @@ import cache from '../util/cache.js';
 import TriggeredSend from './TriggeredSend.js';
 import Folder from './Folder.js';
 import ReplaceCbReference from '../util/replaceContentBlockReference.js';
+import toposort from 'toposort';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -27,6 +28,7 @@ import ReplaceCbReference from '../util/replaceContentBlockReference.js';
  * @typedef {import('../../types/mcdev.d.js').AssetMap} AssetMap
  * @typedef {import('../../types/mcdev.d.js').AssetItem} AssetItem
  * @typedef {import('../../types/mcdev.d.js').AssetRequestParams} AssetRequestParams
+ * @typedef {import('../../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
  */
 
 /**
@@ -185,6 +187,81 @@ class Asset extends MetadataType {
         /* eslint-enable unicorn/prefer-ternary */
     }
 
+    /**
+     * Returns Order in which metadata needs to be retrieved/deployed
+     *
+     * @param {AssetMap} metadataMap metadata mapped by their keyField
+     * @param {string} deployDir directory where deploy metadata are saved
+     * @returns {Promise.<AssetMap>} keyField => metadata map but sorted to ensure dependencies are deployed in correct order
+     */
+    static async _getUpsertOrder(metadataMap, deployDir) {
+        /**
+         * one entry for each dependency with the first item being the key thats required by the second item
+         *
+         * @type {string[][]}
+         */
+        const dependencies = [];
+        /** @type {ContentBlockConversionTypes[]} */
+        Util.OPTIONS.referenceFrom = ['key', 'name', 'id'];
+        /** @type {ContentBlockConversionTypes} */
+        Util.OPTIONS.referenceTo = 'key';
+        // loop through all metadata types which are being retrieved/deployed
+        for (const key in metadataMap) {
+            const findAssetKeys = new Set();
+            try {
+                // find asset references in code
+                await this.replaceCbReference(metadataMap[key], deployDir, findAssetKeys);
+                // find asset references in metadata
+                const findAssetKeyMeta = Util.findLeafVals(metadataMap[key], 'r__asset_key');
+                for (const metaKey of findAssetKeyMeta) {
+                    findAssetKeys.add(metaKey);
+                }
+
+                const dependentKeys = [...findAssetKeys];
+                if (dependentKeys.length > 0) {
+                    dependencies.push(...dependentKeys.map((depKey) => [depKey, key]));
+                } else {
+                    Util.logger.debug('Asset._getUpsertOrder: this case should not happen');
+                    dependencies.push([undefined, key]);
+                }
+            } catch (ex) {
+                if (ex.code !== 200) {
+                    Util.logger.errorStack(ex, 'Cannot find related code blocks for ' + key);
+                }
+                // no dependent keys found
+                dependencies.push([undefined, key]);
+            }
+        }
+
+        // sort list & remove the undefined dependencies
+        const flatList = toposort(dependencies).filter((a) => !!a);
+
+        /** @type {AssetMap} */
+        const metadataTypeMapSorted = {};
+        // group subtypes per type
+        for (const key of flatList) {
+            metadataTypeMapSorted[key] = metadataMap[key];
+        }
+        return metadataTypeMapSorted;
+    }
+
+    /**
+     * MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
+     *
+     * @param {AssetMap} metadataMap metadata mapped by their keyField
+     * @param {string} deployDir directory where deploy metadata are saved
+     * @returns {Promise.<AssetMap>} keyField => metadata map
+     */
+    static async upsert(metadataMap, deployDir) {
+        if (Object.keys(metadataMap).length > 1) {
+            // fill the cache map with our deployment package to ensure we can find
+            ReplaceCbReference.createCacheForMap(metadataMap);
+            // assets can link to other assets (via template, content block reference and SSJS/AMPscript) and deployment would fail if we did not sort this here
+            metadataMap = await this._getUpsertOrder(metadataMap, deployDir);
+        }
+        return super.upsert(metadataMap, deployDir, true);
+    }
+
     /**
      * Creates a single asset
      *
@@ -1037,8 +1114,9 @@ class Asset extends MetadataType {
      * generic script that retrieves the folder path from cache and updates the given metadata with it after retrieve
      *
      * @param {MetadataTypeItem} metadata a single script activity definition
+     * @param {boolean} [hideWarning] when checking content blocks we do want to set the folder path but if we cant, lets not cludder the log with warnings about it
      */
-    static setFolderPath(metadata) {
+    static setFolderPath(metadata, hideWarning = false) {
         try {
             metadata.r__folder_Path = cache.searchForField(
                 'folder',
@@ -1048,11 +1126,13 @@ class Asset extends MetadataType {
             );
             delete metadata.category;
         } catch (ex) {
-            Util.logger.warn(
-                ` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
-                    metadata[this.definition.keyField]
-                }): Could not find folder (${ex.message})`
-            );
+            if (!hideWarning) {
+                Util.logger.warn(
+                    ` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
+                        metadata[this.definition.keyField]
+                    }): Could not find folder (${ex.message})`
+                );
+            }
         }
     }
 
@@ -1827,7 +1907,7 @@ class Asset extends MetadataType {
                             }
                         }
                         if (asset.category?.id) {
-                            this.setFolderPath(asset);
+                            this.setFolderPath(asset, true);
                         }
 
                         if (asset.slots) {

package/lib/metadataTypes/DataExtension.js

@@ -42,9 +42,10 @@ class DataExtension extends MetadataType {
      * if create or update operation is needed.
      *
      * @param {DataExtensionMap} metadataMap dataExtensions mapped by their customerKey
-     * @returns {Promise} Promise
+     * @param {string} deployDir directory where deploy metadata are saved
+     * @returns {Promise.<MetadataTypeMap>} keyField => metadata map
      */
-    static async upsert(metadataMap) {
+    static async upsert(metadataMap, deployDir) {
         // get existing DE-fields for DE-keys in deployment package to properly handle add/update/delete of fields
         // we need to use IN here because it would fail otherwise if we try to deploy too many DEs at the same time
         /** @type {SoapRequestParams} */
@@ -66,6 +67,11 @@ class DataExtension extends MetadataType {
         for (const metadataKey in metadataMap) {
             try {
                 metadataMap[metadataKey] = await this.preDeployTasks(metadataMap[metadataKey]);
+                metadataMap[metadataKey] = await this.validation(
+                    'deploy',
+                    metadataMap[metadataKey],
+                    deployDir
+                );
             } catch (ex) {
                 // output error & remove from deploy list
                 Util.logger.error(
@@ -187,7 +193,7 @@ class DataExtension extends MetadataType {
             DataExtension.oldFields[metadataMap[metadataKey][this.definition.keyField]] =
                 await DataExtensionField.prepareDeployColumnsOnUpdate(
                     metadataMap[metadataKey].Fields,
-                    metadataKey
+                    Util.matchedByName?.[this.definition.type]?.[metadataKey] || metadataKey
                 );
 
             if (
@@ -893,7 +899,7 @@ class DataExtension extends MetadataType {
             if (metadata[field?.DataExtension?.CustomerKey]) {
                 metadata[field.DataExtension.CustomerKey].Fields.push(field);
             } else {
-                Util.logger.warn(` - Issue retrieving data extension fields. key='${key}'`);
+                // field was retrieved for which we do not have the right dataExtension. This might be due to us having to resort to not using a DE filter to avoid the "String or binary data would be truncated." error
             }
         }
 
@@ -1632,6 +1638,70 @@ class DataExtension extends MetadataType {
             return super.getFilesToCommit(keyArr);
         }
     }
+    /**
+     * helper for {@link MetadataType.createOrUpdate}
+     *
+     * @param {MetadataTypeItem} metadataItem to be deployed item
+     * @returns {MetadataTypeItem} cached item or undefined
+     */
+    static getCacheMatchedByName(metadataItem) {
+        let cacheMatchedByName;
+
+        if (Util.OPTIONS.matchName) {
+            // make sure to run the search ONLY if OPTIONS.matchName is true and definition.allowMatchingByName signals support
+            const typeCache = cache.getCache()?.[this.definition.type];
+            const potentials = [];
+            for (const key in typeCache) {
+                const cachedItem = typeCache[key];
+                if (
+                    cachedItem[this.definition.nameField] ===
+                    metadataItem[this.definition.nameField]
+                ) {
+                    potentials.push(cachedItem);
+                }
+            }
+            if (potentials.length > 1) {
+                throw new Error(
+                    `found multiple name matches in cache for ${this.definition.type} ${metadataItem[this.definition.keyField]} / ${metadataItem[this.definition.nameField]}. Check their keys for more details: ${potentials.map((p) => p[this.definition.keyField]).join(', ')}`
+                );
+            } else if (potentials.length === 1) {
+                // only one item found, confirm that it's in the same folder
+                const deployFolderPath = cache.searchForField(
+                    'folder',
+                    metadataItem[this.definition.folderIdField],
+                    'ID',
+                    'Path'
+                );
+                if (
+                    potentials[0][this.definition.folderIdField] ===
+                    metadataItem[this.definition.folderIdField]
+                ) {
+                    cacheMatchedByName = potentials[0];
+
+                    Util.logger.info(
+                        Util.getGrayMsg(
+                            ` - found ${this.definition.type} ${metadataItem[this.definition.keyField]} in cache by name "${metadataItem[this.definition.nameField]}" and folder "${deployFolderPath}": ${cacheMatchedByName[this.definition.keyField]}`
+                        )
+                    );
+                } else {
+                    const cacheFolderPath = cache.searchForField(
+                        'folder',
+                        potentials[0][this.definition.folderIdField],
+                        'ID',
+                        'Path'
+                    );
+                    throw new Error(
+                        `found ${this.definition.type} ${metadataItem[this.definition.keyField]} in cache by name "${metadataItem[this.definition.nameField]}" but in different folders: "${deployFolderPath}" vs "${cacheFolderPath}": ${potentials[0][this.definition.keyField]}`
+                    );
+                }
+            } else {
+                Util.logger.debug(
+                    ` - no name-match found for ${this.definition.type} ${metadataItem[this.definition.keyField]}. Creating new ${this.definition.type} instead.`
+                );
+            }
+        }
+        return cacheMatchedByName;
+    }
 }
 
 // Assign definition to static attributes

package/lib/metadataTypes/DataExtensionField.js

@@ -48,7 +48,13 @@ class DataExtensionField extends MetadataType {
      * @returns {Promise.<{metadata: DataExtensionFieldMap, type: string}>} Promise of items
      */
     static async retrieveForCacheDE(requestParams, additionalFields) {
-        return super.retrieveSOAP(null, requestParams, null, additionalFields);
+        let response;
+        response = await super.retrieveSOAP(null, requestParams, null, additionalFields);
+        if (!response) {
+            // try again but without filters as a workaround for the "String or binary data would be truncated." issue
+            response = await super.retrieveSOAP(null, {}, null, additionalFields);
+        }
+        return response;
     }
 
     /**