mcdev 4.3.0 → 4.3.2

package/.fork/custom-commands.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "Create PR to develop branch",
+    "target": "ref",
+    "refTargets": [
+      "localbranch",
+      "remotebranch"
+    ],
+    "action": {
+      "type": "url",
+      "url": "https://github.com/Accenture/sfmc-devtools/compare/develop...$shortname?expand=1"
+    }
+  }
+]

package/.github/ISSUE_TEMPLATE/bug.yml

@@ -39,6 +39,8 @@ body:
           label: Version
           description: What version of our software are you running? (mcdev --version)
           options:
+              - 4.3.2
+              - 4.3.1
               - 4.3.0
               - 4.2.1
               - 4.2.0

package/boilerplate/gitignore-template

@@ -7,3 +7,4 @@ node_modules/
 /template/
 **/QueryStudioResults at*
 .mcdev-auth.json
+*.error.log

package/docs/dist/documentation.md

@@ -736,10 +736,7 @@ MessageSendActivity MetadataType
 * [AccountUser](#AccountUser) ⇐ [<code>MetadataType</code>](#MetadataType)
     * [.retrieve(retrieveDir, _, [__], [key])](#AccountUser.retrieve) ⇒ <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code>
     * [.retrieveChangelog()](#AccountUser.retrieveChangelog) ⇒ <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code>
-    * [.timeSinceDate(date)](#AccountUser.timeSinceDate) ⇒ <code>number</code>
-    * [.getBuName(id)](#AccountUser.getBuName) ⇒ <code>string</code>
     * [.document([metadata])](#AccountUser.document) ⇒ <code>Promise.&lt;void&gt;</code>
-    * [._generateDocMd(users, type, columnsToPrint)](#AccountUser._generateDocMd) ⇒ <code>string</code>
     * [.postRetrieveTasks(metadata)](#AccountUser.postRetrieveTasks) ⇒ <code>TYPE.MetadataTypeItem</code>
     * [.parseMetadata(metadata)](#AccountUser.parseMetadata) ⇒ <code>TYPE.MetadataTypeItem</code>
 
@@ -765,28 +762,6 @@ Retrieves SOAP based metadata of metadata type into local filesystem. executes c
 
 **Kind**: static method of [<code>AccountUser</code>](#AccountUser)  
 **Returns**: <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code> - Promise of metadata  
-<a name="AccountUser.timeSinceDate"></a>
-
-### AccountUser.timeSinceDate(date) ⇒ <code>number</code>
-**Kind**: static method of [<code>AccountUser</code>](#AccountUser)  
-**Returns**: <code>number</code> - time difference  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| date | <code>string</code> | first date |
-
-<a name="AccountUser.getBuName"></a>
-
-### AccountUser.getBuName(id) ⇒ <code>string</code>
-helper to print bu names
-
-**Kind**: static method of [<code>AccountUser</code>](#AccountUser)  
-**Returns**: <code>string</code> - "bu name (bu id)""  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| id | <code>number</code> | bu id |
-
 <a name="AccountUser.document"></a>
 
 ### AccountUser.document([metadata]) ⇒ <code>Promise.&lt;void&gt;</code>
@@ -799,18 +774,6 @@ Creates markdown documentation of all roles
 | --- | --- | --- |
 | [metadata] | <code>TYPE.MetadataTypeMap</code> | user list |
 
-<a name="AccountUser._generateDocMd"></a>
-
-### AccountUser.\_generateDocMd(users, type, columnsToPrint) ⇒ <code>string</code>
-**Kind**: static method of [<code>AccountUser</code>](#AccountUser)  
-**Returns**: <code>string</code> - markdown  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| users | <code>Array.&lt;object&gt;</code> | list of users and installed package |
-| type | <code>&#x27;Installed Package&#x27;</code> \| <code>&#x27;User&#x27;</code> | choose what sub type to print |
-| columnsToPrint | <code>Array.&lt;Array&gt;</code> | helper array |
-
 <a name="AccountUser.postRetrieveTasks"></a>
 
 ### AccountUser.postRetrieveTasks(metadata) ⇒ <code>TYPE.MetadataTypeItem</code>
@@ -849,7 +812,7 @@ FileTransfer MetadataType
     * [.retrieveAsTemplate(templateDir, name, templateVariables, [selectedSubType])](#Asset.retrieveAsTemplate) ⇒ <code>Promise.&lt;{metadata: TYPE.AssetItem, type: string}&gt;</code>
     * [.create(metadata)](#Asset.create) ⇒ <code>Promise</code>
     * [.update(metadata)](#Asset.update) ⇒ <code>Promise</code>
-    * [.requestSubType(subType, subTypeArray, [retrieveDir], [templateName], [templateVariables], key)](#Asset.requestSubType) ⇒ <code>Promise</code>
+    * [.requestSubType(subType, [retrieveDir], [templateName], [templateVariables], key)](#Asset.requestSubType) ⇒ <code>Promise</code>
     * [.requestAndSaveExtended(items, subType, retrieveDir, [templateVariables])](#Asset.requestAndSaveExtended) ⇒ <code>Promise</code>
     * [._retrieveExtendedFile(metadata, subType, retrieveDir)](#Asset._retrieveExtendedFile) ⇒ <code>Promise.&lt;void&gt;</code>
     * [._readExtendedFileFromFS(metadata, subType, deployDir, [pathOnly])](#Asset._readExtendedFileFromFS) ⇒ <code>Promise.&lt;string&gt;</code>
@@ -939,7 +902,7 @@ Updates a single asset
 
 <a name="Asset.requestSubType"></a>
 
-### Asset.requestSubType(subType, subTypeArray, [retrieveDir], [templateName], [templateVariables], key) ⇒ <code>Promise</code>
+### Asset.requestSubType(subType, [retrieveDir], [templateName], [templateVariables], key) ⇒ <code>Promise</code>
 Retrieves Metadata of a specific asset type
 
 **Kind**: static method of [<code>Asset</code>](#Asset)  
@@ -948,7 +911,6 @@ Retrieves Metadata of a specific asset type
 | Param | Type | Description |
 | --- | --- | --- |
 | subType | <code>TYPE.AssetSubType</code> | group of similar assets to put in a folder (ie. images) |
-| subTypeArray | <code>Array.&lt;TYPE.AssetSubType&gt;</code> | list of all asset types within this subtype |
 | [retrieveDir] | <code>string</code> | target directory for saving assets |
 | [templateName] | <code>string</code> | name of the metadata file |
 | [templateVariables] | <code>TYPE.TemplateMap</code> | variables to be replaced in the metadata |

package/lib/Retriever.js

@@ -62,29 +62,33 @@ class Retriever {
         // ensure we know which real dependencies we have to ensure we cache those completely
         const dependencies = this._getTypeDependencies(metadataTypes);
         const deployOrder = Util.getMetadataHierachy(metadataTypes);
-        for (const metadataType in deployOrder) {
-            const type = metadataType;
-            const subTypeArr = deployOrder[metadataType];
+        for (const type in deployOrder) {
+            const subTypeArr = deployOrder[type];
             // if types were added by getMetadataHierachy() for caching, make sure the key-list is set to [null] for them which will retrieve all
-            typeKeyMap[metadataType] = typeKeyMap[metadataType] || [null];
+            typeKeyMap[type] = typeKeyMap[type] || [null];
             // add client to metadata process class instead of passing every time
             MetadataTypeInfo[type].client = auth.getSDK(this.buObject);
             MetadataTypeInfo[type].properties = this.properties;
             MetadataTypeInfo[type].buObject = this.buObject;
             try {
                 let result;
-                if (!metadataTypes.includes(type) && !metadataTypes.includes(metadataType)) {
+                if (
+                    !metadataTypes.includes(type) &&
+                    (!Array.isArray(subTypeArr) ||
+                        (Array.isArray(subTypeArr) &&
+                            !metadataTypes.includes(`${type}-${subTypeArr?.[0]}`)))
+                ) {
                     // type not in list of types to retrieve, but is a dependency of one of them
                     if (changelogOnly && type !== 'folder') {
                         // no extra caching needed for list view except for folders
                         continue;
                     }
-                    Util.logger.info(`Caching dependent Metadata: ${metadataType}`);
+                    Util.logger.info(`Caching dependent Metadata: ${type}`);
                     Util.logSubtypes(subTypeArr);
                     result = await MetadataTypeInfo[type].retrieveForCache(null, subTypeArr);
                 } else if (templateVariables) {
                     // type is in list of types to retrieve and we have template variables
-                    Util.logger.info(`Retrieving as Template: ${metadataType}`);
+                    Util.logger.info(`Retrieving as Template: ${type}`);
                     if (subTypeArr?.length > 1) {
                         Util.logger.warn(
                             `retrieveAsTemplate only works with one subtype, ignoring all but first subtype from your list: ${subTypeArr.join(
@@ -93,7 +97,7 @@ class Retriever {
                         );
                     }
                     result = await Promise.all(
-                        typeKeyMap[metadataType].map((name) =>
+                        typeKeyMap[type].map((name) =>
                             MetadataTypeInfo[type].retrieveAsTemplate(
                                 this.templateDir,
                                 name,
@@ -106,12 +110,11 @@ class Retriever {
                     // type is in list of types to retrieve and we don't have template variables
                     let cacheResult = null;
                     if (
-                        (typeKeyMap[metadataType].length > 1 ||
-                            typeKeyMap[metadataType][0] !== null) &&
-                        (dependencies.includes(type) || dependencies.includes(metadataType))
+                        (typeKeyMap[type].length > 1 || typeKeyMap[type][0] !== null) &&
+                        (dependencies.includes(type) || dependencies.includes(type))
                     ) {
                         // if we have a key-list and the type is a dependency, we need to cache the whole type
-                        Util.logger.info(`Caching dependent Metadata: ${metadataType}`);
+                        Util.logger.info(`Caching dependent Metadata: ${type}`);
                         Util.logSubtypes(subTypeArr);
                         cacheResult = await MetadataTypeInfo[type].retrieveForCache(
                             null,
@@ -119,15 +122,15 @@ class Retriever {
                         );
                     }
                     Util.logger.info(
-                        `Retrieving: ${metadataType}` +
-                            (typeKeyMap[metadataType][0] === null
+                        `Retrieving: ${type}` +
+                            (typeKeyMap[type][0] === null
                                 ? ''
-                                : Util.getKeysString(typeKeyMap[metadataType]))
+                                : Util.getKeysString(typeKeyMap[type]))
                     );
                     result = await (changelogOnly
                         ? MetadataTypeInfo[type].retrieveChangelog(null, subTypeArr)
                         : Promise.all(
-                              typeKeyMap[metadataType].map((key) =>
+                              typeKeyMap[type].map((key) =>
                                   MetadataTypeInfo[type].retrieve(
                                       this.savePath,
                                       null,
@@ -154,14 +157,14 @@ class Retriever {
                                 cache.mergeMetadata(type, result_i.metadata);
                             }
                         }
-                        if (metadataTypes.includes(type) || metadataTypes.includes(metadataType)) {
+                        if (metadataTypes.includes(type) || metadataTypes.includes(type)) {
                             retrieveChangelog[type] = result
                                 .filter((el) => !!el)
                                 .map((element) => element.metadata);
                         }
                     } else {
                         cache.setMetadata(type, result.metadata);
-                        if (metadataTypes.includes(type) || metadataTypes.includes(metadataType)) {
+                        if (metadataTypes.includes(type) || metadataTypes.includes(type)) {
                             retrieveChangelog[type] = result.metadata;
                         }
                     }
@@ -175,7 +178,7 @@ class Retriever {
                     Util.logger.error(ex.message);
                     break;
                 } else {
-                    Util.logger.errorStack(ex, ` - Retrieving ${metadataType} failed`);
+                    Util.logger.errorStack(ex, ` - Retrieving ${type} failed`);
                 }
             }
         }

package/lib/metadataTypes/AccountUser.js

@@ -60,10 +60,17 @@ class AccountUser extends MetadataType {
         for (const item of resultsBatch) {
             this.userIdBuMap[item.AccountUser.AccountUserID] =
                 this.userIdBuMap[item.AccountUser.AccountUserID] || [];
-            this.userIdBuMap[item.AccountUser.AccountUserID].push({
-                ID: item.Account.ID,
-                Name: item.Account.Name,
-            });
+            // push to array if not already in array
+            if (
+                !this.userIdBuMap[item.AccountUser.AccountUserID].some(
+                    (bu) => bu.ID === item.Account.ID
+                )
+            ) {
+                this.userIdBuMap[item.AccountUser.AccountUserID].push({
+                    ID: item.Account.ID,
+                    Name: item.Account.Name,
+                });
+            }
         }
         // get actual user details
         /** @type {TYPE.SoapRequestParams} */
@@ -114,10 +121,11 @@ class AccountUser extends MetadataType {
     }
     /**
      *
+     * @private
      * @param {string} date first date
      * @returns {number} time difference
      */
-    static timeSinceDate(date) {
+    static _timeSinceDate(date) {
         const interval = 'days';
         const second = 1000,
             minute = second * 60,
@@ -172,13 +180,28 @@ class AccountUser extends MetadataType {
     /**
      * helper to print bu names
      *
+     * @private
      * @param {number} id bu id
      * @returns {string} "bu name (bu id)""
      */
-    static getBuName(id) {
+    static _getBuName(id) {
         const name = this.buObject.eid == id ? '_ParentBU_' : this.buIdName[id];
         return `<nobr>${name} (${id})</nobr>`;
     }
+
+    /**
+     * helper that gets BU names from config
+     *
+     * @private
+     */
+    static _getBuNames() {
+        this.buIdName = {};
+        for (const cred in this.properties.credentials) {
+            for (const buName in this.properties.credentials[cred].businessUnits) {
+                this.buIdName[this.properties.credentials[cred].businessUnits[buName]] = buName;
+            }
+        }
+    }
     /**
      * Creates markdown documentation of all roles
      *
@@ -209,7 +232,7 @@ class AccountUser extends MetadataType {
             }
         }
         // init map of BU Ids > BU Name
-        this.buIdName = {};
+        this._getBuNames();
 
         // initialize permission object
         this.allPermissions = {};
@@ -244,14 +267,19 @@ class AccountUser extends MetadataType {
             }
             let associatedBus = '';
             if (user.AssociatedBusinessUnits__c) {
-                associatedBus = user.AssociatedBusinessUnits__c.map((item) => {
-                    this.buIdName[item.ID] = item.Name;
-                    return this.getBuName(item.ID);
-                })
+                user.AssociatedBusinessUnits__c.push({
+                    ID: user.DefaultBusinessUnit,
+                });
+                // ensure associatedBus have no duplicates
+                associatedBus = [
+                    ...new Set(
+                        user.AssociatedBusinessUnits__c.map((item) => this._getBuName(item.ID))
+                    ),
+                ]
                     .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
                     .join(',<br> ');
             }
-            const defaultBUName = this.getBuName(user.DefaultBusinessUnit);
+            const defaultBUName = this._getBuName(user.DefaultBusinessUnit);
             users.push({
                 TYPE: user.type__c,
                 UserID: user.UserID,
@@ -267,7 +295,7 @@ class AccountUser extends MetadataType {
                 AssociatedBusinessUnits__c: associatedBus,
                 Roles: roles,
                 UserPermissions: userPermissions,
-                LastSuccessfulLogin: this.timeSinceDate(user.LastSuccessfulLogin),
+                LastSuccessfulLogin: this._timeSinceDate(user.LastSuccessfulLogin),
                 CreatedDate: user.CreatedDate.split('T').join(' '),
                 ModifiedDate: user.ModifiedDate.split('T').join(' '),
             });
@@ -325,6 +353,7 @@ class AccountUser extends MetadataType {
     }
     /**
      *
+     * @private
      * @param {object[]} users list of users and installed package
      * @param {'Installed Package'|'User'} type choose what sub type to print
      * @param {Array[]} columnsToPrint helper array

package/lib/metadataTypes/Asset.js

@@ -31,16 +31,7 @@ class Asset extends MetadataType {
         for (const subType of subTypeArr) {
             // each subtype contains multiple different specific types (images contains jpg and png for example)
             // we use await here to limit the risk of too many concurrent api requests at time
-            items.push(
-                ...(await this.requestSubType(
-                    subType,
-                    this.definition.extendedSubTypes[subType],
-                    retrieveDir,
-                    null,
-                    null,
-                    key
-                ))
-            );
+            items.push(...(await this.requestSubType(subType, retrieveDir, null, null, key)));
         }
         const metadata = this.parseResponseBody({ items: items });
         if (retrieveDir) {
@@ -81,13 +72,7 @@ class Asset extends MetadataType {
             // each subtype contains multiple different specific types (images contains jpg and png for example)
             // we use await here to limit the risk of too many concurrent api requests at time
             items.push(
-                ...(await this.requestSubType(
-                    subType,
-                    this.definition.extendedSubTypes[subType],
-                    templateDir,
-                    name,
-                    templateVariables
-                ))
+                ...(await this.requestSubType(subType, templateDir, name, templateVariables))
             );
         }
         const metadata = this.parseResponseBody({ items: items });
@@ -146,27 +131,21 @@ class Asset extends MetadataType {
      * Retrieves Metadata of a specific asset type
      *
      * @param {TYPE.AssetSubType} subType group of similar assets to put in a folder (ie. images)
-     * @param {TYPE.AssetSubType[]} subTypeArray list of all asset types within this subtype
      * @param {string} [retrieveDir] target directory for saving assets
      * @param {string} [templateName] name of the metadata file
      * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
      * @param {string} key customer key to filter by
      * @returns {Promise} Promise
      */
-    static async requestSubType(
-        subType,
-        subTypeArray,
-        retrieveDir,
-        templateName,
-        templateVariables,
-        key
-    ) {
+    static async requestSubType(subType, retrieveDir, templateName, templateVariables, key) {
         if (retrieveDir) {
             Util.logger.info(`- Retrieving Subtype: ${subType}`);
         } else {
             Util.logger.info(` - Caching Subtype: ${subType}`);
         }
-        const subtypeIds = subTypeArray?.map(
+        /** @type {TYPE.AssetSubType[]} */
+        const extendedSubTypeArr = this.definition.extendedSubTypes[subType];
+        const subtypeIds = extendedSubTypeArr?.map(
             (subTypeItemName) => Asset.definition.typeMapping[subTypeItemName]
         );
         const uri = '/asset/v1/content/assets/query';
@@ -338,22 +317,30 @@ class Asset extends MetadataType {
                                 } catch (ex) {
                                     failed.push({ item: item, error: ex });
                                 }
+                                // even if the extended file failed, still save the metadata
                                 metadata[item.customerKey] = item;
                             }
                             // this is a complex type which stores data in the asset itself
                             else if (!completed.includes(item.id)) {
-                                const extendedItem = await this.client.rest.get(
-                                    'asset/v1/content/assets/' + item.id
-                                );
-                                metadata[item.customerKey] = extendedItem;
+                                try {
+                                    const extendedItem = await this.client.rest.get(
+                                        'asset/v1/content/assets/' + item.id
+                                    );
+                                    // only save the metadata if we have extended content
+                                    metadata[item.customerKey] = extendedItem;
+                                } catch (ex) {
+                                    failed.push({ item: item, error: ex });
+                                }
                             }
                             completed.push(item.id);
-                            await this.saveResults(
-                                metadata,
-                                retrieveDir,
-                                'asset-' + subType,
-                                templateVariables
-                            );
+                            if (metadata[item.customerKey]) {
+                                await this.saveResults(
+                                    metadata,
+                                    retrieveDir,
+                                    'asset-' + subType,
+                                    templateVariables
+                                );
+                            }
                             // update the current value in your application..
                             extendedBar.increment();
                         })
@@ -367,7 +354,7 @@ class Asset extends MetadataType {
             } catch (ex) {
                 extendedBar.stop();
                 Asset._resetLogLevel(loggerLevelBak, failed);
-                // timeouts should be retried, others can be retried
+                // timeouts should be retried, others can be exited
                 if (ex.code !== 'ETIMEDOUT') {
                     throw ex;
                 }
@@ -415,11 +402,17 @@ class Asset extends MetadataType {
             );
             for (const fail of failed) {
                 Util.logger.warn(
-                    `   -  "${fail.item.name}" (${fail.item.customerKey}) in ${fail.item.r__folder_Path}: ${fail.error.message}`
+                    `   - "${fail.item.name}" (${fail.item.customerKey}): ${fail.error.message} (${
+                        fail.error.code
+                    })${
+                        fail.error.endpoint
+                            ? Util.getGrayMsg(
+                                  ' - ' +
+                                      fail.error.endpoint.split('rest.marketingcloudapis.com')[1]
+                              )
+                            : ''
+                    }`
                 );
-                Util.logger.debug(`-- Error: ${fail.error.message}`);
-                Util.logger.debug(`-- AssetType: ${fail.item.assetType.name}`);
-                Util.logger.debug(`-- fileProperties: ${JSON.stringify(fail.item.fileProperties)}`);
             }
             Util.logger.info(
                 ' - You will still find a JSON file for each of these in the download directory.'
@@ -977,6 +970,7 @@ class Asset extends MetadataType {
 
                 break;
             }
+            case 'template': // template-template
             case 'buttonblock': // block - Button Block
             case 'freeformblock': // block
             case 'htmlblock': // block
@@ -1198,6 +1192,7 @@ class Asset extends MetadataType {
                     subFolder: [customerKey],
                 };
             }
+            case 'template': // template-template
             case 'buttonblock': // block - Button Block
             case 'freeformblock': // block
             case 'htmlblock': // block

package/lib/metadataTypes/Automation.js

@@ -63,7 +63,7 @@ class Automation extends MetadataType {
                           }
                           // if we do get here, we should log the error and continue instead of failing to download all automations
                           Util.logger.error(
-                              ` - skipping retrieving Automation ${a.ObjectID}: ${ex.message} ${ex.code}`
+                              ` ☇ skipping Automation ${a.ObjectID}: ${ex.message} ${ex.code}`
                           );
                           return null;
                       }

package/lib/metadataTypes/DataExtensionField.js

@@ -242,9 +242,11 @@ class DataExtensionField extends MetadataType {
         }
 
         Util.logger.info(
-            ` - Found ${deployColumns.length} added/updated Fields for ${deKey}${
-                deployColumns.length ? ': ' : ''
-            }` + deployColumns.map((item) => item.Name).join(', ')
+            Util.getGrayMsg(
+                ` - Found ${deployColumns.length} added/updated Fields for ${deKey}${
+                    deployColumns.length ? ': ' : ''
+                }` + deployColumns.map((item) => item.Name).join(', ')
+            )
         );
         return existingFieldByName;
     }