mcdev 5.2.0 → 5.3.0

package/lib/metadataTypes/Journey.js

@@ -191,14 +191,21 @@ class Journey extends MetadataType {
             );
             const results = this.parseResponseBody(response, key);
             singleKey = results[key].id;
+            if (version && version > results[key].version) {
+                Util.logger.error(
+                    `The chosen version (${version}) is higher than the latest known version (${results[key].version}). Please choose a lower version.`
+                );
+                return false;
+            }
             Util.logger.debug(`Deleting interaction ${key} via its ID ${singleKey}`);
         }
         if (!/^\d+$/.test(version)) {
-            throw new TypeError(
+            Util.logger.error(
                 'Version is required for deleting interactions to avoid accidental deletion of the wrong item. Please append it at the end of the key or id, separated by forward-slash. Example for deleting version 4: ' +
                     key +
                     '/4'
             );
+            return false;
         }
         Util.logger.warn(
             `Deleting Journeys via this command breaks following retrieve-by-key/id requests until you've deployed/created a new draft version! You can get still get the latest available version of your journey by retrieving all interactions on this BU.`

package/lib/metadataTypes/MetadataType.js

@@ -139,9 +139,10 @@ class MetadataType {
      *
      * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {object} apiResponse varies depending on the API call
+     * @param {TYPE.MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
      * @returns {void}
      */
-    static postCreateTasks(metadataEntry, apiResponse) {}
+    static postCreateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {}
 
     /**
      * helper for {@link MetadataType.updateREST}
@@ -265,11 +266,10 @@ class MetadataType {
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {string[]} [subTypeArr] optionally limit to a single subtype
-     * @param {string} [key] customer key of single item to retrieve
      * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
      */
-    static async retrieveForCache(additionalFields, subTypeArr, key) {
-        return this.retrieve(null, additionalFields, subTypeArr, key);
+    static async retrieveForCache(additionalFields, subTypeArr) {
+        return this.retrieve(null, additionalFields, subTypeArr);
     }
     /**
      * Gets metadata cache with limited fields and does not store value to disk
@@ -808,11 +808,12 @@ class MetadataType {
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
     static async createREST(metadataEntry, uri) {
+        const metadataClone = JSON.parse(JSON.stringify(metadataEntry));
         this.removeNotCreateableFields(metadataEntry);
         try {
             // set to empty object in case API returned nothing to be able to update it in helper classes
             const response = (await this.client.rest.post(uri, metadataEntry)) || {};
-            await this.postCreateTasks(metadataEntry, response);
+            await this.postCreateTasks(metadataEntry, response, metadataClone);
             Util.logger.info(
                 ` - created ${this.definition.type}: ${
                     metadataEntry[this.definition.keyField] ||
@@ -1060,6 +1061,7 @@ class MetadataType {
         const results = this.parseResponseBody(response, singleRetrieve);
         // get extended metadata if applicable
         if (this.definition.hasExtended) {
+            Util.logger.debug(' - retrieving extended metadata');
             const extended = await this.client.rest.getCollection(
                 Object.keys(results).map((key) => uri + results[key][this.definition.idField])
             );
@@ -1088,6 +1090,62 @@ class MetadataType {
             type: this.definition.type,
         };
     }
+    /**
+     *
+     * @param {object[]} urlArray {uri: string, id: string} combo of URL and ID/key of metadata
+     * @param {number} [concurrentRequests] optionally set a different amount of concurrent requests
+     * @param {boolean} [logAmountOfUrls] if true, prints an info message about to-be loaded amount of metadata
+     * @returns {Promise.<{metadata: (TYPE.MetadataTypeMap | TYPE.MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
+     */
+    static async retrieveRESTcollection(urlArray, concurrentRequests = 10, logAmountOfUrls = true) {
+        if (logAmountOfUrls) {
+            Util.logger.info(
+                Util.getGrayMsg(
+                    ` - ${urlArray?.length} ${this.definition.type}${
+                        urlArray?.length === 1 ? '' : 's'
+                    } found. Retrieving details...`
+                )
+            );
+        }
+        const rateLimit = pLimit(concurrentRequests);
+
+        const metadataArr = urlArray.length
+            ? await Promise.all(
+                  urlArray.map(async (item) =>
+                      rateLimit(async () => {
+                          try {
+                              return await this.client.rest.get(item.uri);
+                          } catch (ex) {
+                              return this.handleRESTErrors(ex, item.id);
+                          }
+                      })
+                  )
+              )
+            : [];
+        const results = {};
+        for (const item of metadataArr) {
+            const key = item[this.definition.keyField];
+            results[key] = item;
+        }
+        return {
+            metadata: results,
+            type: this.definition.type,
+        };
+    }
+
+    /**
+     * helper for {@link this.retrieveRESTcollection}
+     *
+     * @param {Error} ex exception
+     * @param {string} id id or key of item
+     * @returns {null} -
+     */
+    static handleRESTErrors(ex, id) {
+        // if the ID is too short, the system will throw the 400 error
+        Util.logger.debug(` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`);
+
+        return null;
+    }
     /**
      * Used to execute a query/automation etc.
      *

package/lib/metadataTypes/MobileKeyword.js

@@ -501,7 +501,7 @@ class MobileKeyword extends MetadataType {
      */
     static async deleteByKey(key) {
         // get id from cache
-        const { metadata } = await this.retrieveForCache(key);
+        const { metadata } = await this.retrieveForCache(undefined, undefined, key);
         if (!metadata[key]) {
             Util.logger.error(`Could not find ${this.definition.type} with key ${key}.`);
             return false;

package/lib/metadataTypes/TransactionalEmail.js

@@ -40,21 +40,24 @@ class TransactionalEmail extends TransactionalMessage {
      */
     static async preDeployTasks(metadata) {
         // asset
-        if (metadata.content?.customerKey) {
+        if (metadata.r__asset_customerKey) {
             // we merely want to be able to show an error if it does not exist
-            cache.searchForField(
-                'asset',
-                metadata.content.customerKey,
-                'customerKey',
-                'customerKey'
-            );
+            metadata.content = {
+                customerKey: cache.searchForField(
+                    'asset',
+                    metadata.r__asset_customerKey,
+                    'customerKey',
+                    'customerKey'
+                ),
+            };
+            delete metadata.r__asset_customerKey;
         }
         // subscriptions: dataExtension
-        if (metadata.subscriptions?.dataExtension) {
+        if (metadata.subscriptions?.r__dataExtension_CustomerKey) {
             // we merely want to be able to show an error if it does not exist
-            cache.searchForField(
+            metadata.subscriptions.dataExtension = cache.searchForField(
                 'dataExtension',
-                metadata.subscriptions.dataExtension,
+                metadata.subscriptions.r__dataExtension_CustomerKey,
                 'CustomerKey',
                 'CustomerKey'
             );
@@ -69,15 +72,49 @@ class TransactionalEmail extends TransactionalMessage {
         }
 
         // journey
-        if (metadata.journey?.interactionKey) {
-            // ! update & create enpoints dont accept journey.interactionKey. They only allow to create a new journey
-            metadata.options ||= {};
-            metadata.options.createJourney = true; // only send this during create or else we might end up with an unexpected outcome
-            delete metadata.journey.interactionKey;
-        }
+        // ! update & create enpoints dont accept journey.interactionKey. They only allow to create a new journey
+        metadata.options ||= {};
+        metadata.options.createJourney = true; // only send this during create or else we might end up with an unexpected outcome
 
         return metadata;
     }
+    /**
+     * helper for {@link TransactionalEmail.createREST}
+     *
+     * @param {TYPE.MetadataTypeItem} _ not used
+     * @param {object} apiResponse varies depending on the API call
+     * @returns {void}
+     */
+    static async postCreateTasks(_, apiResponse) {
+        if (apiResponse.journey?.interactionKey) {
+            Util.logger.warn(
+                ` - created journey: ${apiResponse.journey.interactionKey} (auto-created when ${this.definition.type} ${apiResponse.definitionKey} was created)`
+            );
+            // when we create new transactionalEmails, we should also download the new journey that was created with it
+            this._createdJourneyKeys ||= [];
+            this._createdJourneyKeys.push(apiResponse.journey?.interactionKey);
+
+            // do what postRetrieveTasks won't be able to do without spending lots of time on caching
+            apiResponse.r__journey_key = apiResponse.journey.interactionKey;
+            delete apiResponse.journey;
+        }
+    }
+    /**
+     * Gets executed after deployment of metadata type
+     *
+     * @returns {void}
+     */
+    static postDeployTasks() {
+        if (this._createdJourneyKeys?.length) {
+            Util.logger.warn(
+                `Please download related journeys via: mcdev r ${this.buObject.credential}/${
+                    this.buObject.businessUnit
+                } journey "${this._createdJourneyKeys.join(',')}"`
+            );
+        }
+        delete this._createdJourneyKeys;
+    }
+
     /**
      * manages post retrieve steps
      *
@@ -87,6 +124,7 @@ class TransactionalEmail extends TransactionalMessage {
     static postRetrieveTasks(metadata) {
         // asset
         if (metadata.content?.customerKey) {
+            metadata.r__asset_customerKey = metadata.content.customerKey;
             try {
                 // we merely want to be able to show an error if it does not exist
                 cache.searchForField(
@@ -102,9 +140,13 @@ class TransactionalEmail extends TransactionalMessage {
                     }): ${ex.message}.`
                 );
             }
+            delete metadata.content;
         }
+
         // subscriptions: dataExtension
         if (metadata.subscriptions?.dataExtension) {
+            metadata.subscriptions.r__dataExtension_CustomerKey =
+                metadata.subscriptions.dataExtension;
             try {
                 // we merely want to be able to show a warning if it does not exist
                 cache.searchForField(
@@ -120,6 +162,7 @@ class TransactionalEmail extends TransactionalMessage {
                     }): ${ex.message}.`
                 );
             }
+            delete metadata.subscriptions.dataExtension;
         }
         // subscriptions: list
         if (metadata.subscriptions?.list) {
@@ -140,9 +183,15 @@ class TransactionalEmail extends TransactionalMessage {
         }
         // journey
         if (metadata.journey?.interactionKey) {
+            metadata.r__journey_key = metadata.journey.interactionKey;
             try {
                 // we merely want to be able to show a warning if it does not exist
-                cache.searchForField('journey', metadata.journey.interactionKey, 'key', 'key');
+                metadata.r__journey_key = cache.searchForField(
+                    'journey',
+                    metadata.journey.interactionKey,
+                    'key',
+                    'key'
+                );
             } catch (ex) {
                 Util.logger.warn(
                     ` - ${this.definition.type} ${metadata[this.definition.nameField]} (${
@@ -150,10 +199,38 @@ class TransactionalEmail extends TransactionalMessage {
                     }): ${ex.message}.`
                 );
             }
+            delete metadata.journey;
         }
 
         return metadata;
     }
+    /**
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {string} key Identifier of item
+     * @returns {Promise.<boolean>} deletion success status
+     */
+    static async deleteByKey(key) {
+        const metadataMapObj = await this.retrieveForCache(key);
+        const journeyKey = metadataMapObj?.metadata?.[key]?.journey?.interactionKey;
+
+        const isDeleted = await super.deleteByKeyREST(
+            '/messaging/v1/' + this.subType + '/definitions/' + key,
+            key,
+            false
+        );
+        if (isDeleted && journeyKey) {
+            const Journey = require('./Journey');
+            Util.logger.info(
+                ` - deleted ${Journey.definition.type}: ${journeyKey} (SFMC auto-deletes the related journey of ${this.definition.type} ${key})`
+            );
+            Journey.buObject = this.buObject;
+            Journey.properties = this.properties;
+            Journey.client = this.client;
+            Journey.postDeleteTasks(journeyKey);
+        }
+        return isDeleted;
+    }
 }
 
 // Assign definition to static attributes

package/lib/metadataTypes/TransactionalMessage.js

@@ -67,13 +67,14 @@ class TransactionalMessage extends MetadataType {
     /**
      * Retrieves event definition metadata for caching
      *
+     * @param {string} [key] customer key of single item to cache
      * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
      */
-    static retrieveForCache() {
+    static retrieveForCache(key) {
         // the call to /messaging/v1/email/definitions/ does not return definitionId
         // definitionId is required for resolving dependencies on interactions.
         // we should therefore use the already defined retrieve method
-        return this.retrieve();
+        return this.retrieve(undefined, undefined, undefined, key);
     }
     /**
      * Updates a single item

package/lib/metadataTypes/Verification.js

@@ -0,0 +1,230 @@
+'use strict';
+
+const TYPE = require('../../types/mcdev.d');
+const MetadataType = require('./MetadataType');
+const Automation = require('./Automation');
+const Util = require('../util/util');
+const cache = require('../util/cache');
+
+/**
+ * Verification MetadataType
+ *
+ * @augments MetadataType
+ */
+class Verification extends MetadataType {
+    /**
+     * Retrieves Metadata of Data Verification Activity.
+     *
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {void} [_] unused parameter
+     * @param {void} [__] unused parameter
+     * @param {string} key customer key of single item to retrieve
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
+     */
+    static async retrieve(retrieveDir, _, __, key) {
+        let paramArr = [];
+        if (key?.startsWith('id:')) {
+            paramArr = [key.slice(3)];
+        } else if (key) {
+            paramArr = [key];
+        }
+        if (!paramArr.length) {
+            // there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities
+            Util.logger.info(` - Caching dependent Metadata: automation`);
+            Automation.client = this.client;
+            Automation.buObject = this.buObject;
+            Automation.properties = this.properties;
+            Automation._skipNotificationRetrieve = true;
+            delete Automation._cachedMetadataMap;
+            const automationsMapObj = await Automation.retrieve();
+            delete Automation._skipNotificationRetrieve;
+            if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) {
+                if (!key) {
+                    // if we are not retrieving a single item, cache the automations for later use during retrieval of automations
+                    Automation._cachedMetadataMap = automationsMapObj?.metadata;
+                }
+                // automations found, lets iterate over their activities to find the dataVerification items
+                const dataVerificationIds = [];
+                for (const automation of Object.values(automationsMapObj.metadata)) {
+                    if (automation.steps) {
+                        for (const step of automation.steps) {
+                            for (const activity of step.activities) {
+                                if (
+                                    activity.objectTypeId === 1000 &&
+                                    activity.activityObjectId &&
+                                    activity.activityObjectId !==
+                                        '00000000-0000-0000-0000-000000000000'
+                                ) {
+                                    dataVerificationIds.push(activity.activityObjectId);
+                                }
+                            }
+                        }
+                    }
+                }
+                if (dataVerificationIds.length) {
+                    paramArr.push(...dataVerificationIds);
+                }
+            }
+        }
+        const results = {};
+        if (paramArr.length) {
+            const response = await this.retrieveRESTcollection(
+                paramArr.map((id) => ({ id, uri: '/automation/v1/dataverifications/' + id })),
+                undefined,
+                !key
+            );
+            if (response?.metadata) {
+                Object.assign(results, response.metadata);
+            }
+        }
+        if (retrieveDir) {
+            const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
+            Util.logger.info(
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
+                    Util.getKeysString(key)
+            );
+        }
+
+        return {
+            metadata: results,
+            type: this.definition.type,
+        };
+    }
+    /**
+     * helper for {@link this.retrieveRESTcollection}
+     *
+     * @param {Error} ex exception
+     * @param {string} id id or key of item
+     * @returns {null} -
+     */
+    static handleRESTErrors(ex, id) {
+        if (ex.message === 'Not Found' || ex.message === 'Request failed with status code 400') {
+            // if the ID is too short, the system will throw the 400 error
+            Util.logger.debug(
+                ` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`
+            );
+        } else {
+            // if we do get here, we should log the error and continue instead of failing to download all automations
+            Util.logger.error(
+                ` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`
+            );
+        }
+        return null;
+    }
+
+    /**
+     * Retrieves Metadata of  Data Extract Activity for caching
+     *
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
+     */
+    static async retrieveForCache() {
+        return this.retrieve();
+    }
+
+    /**
+     * Creates a single Data Extract
+     *
+     * @param {TYPE.VerificationItem} metadata a single Data Extract
+     * @returns {Promise} Promise
+     */
+    static create(metadata) {
+        return super.createREST(metadata, '/automation/v1/dataverifications/');
+    }
+
+    /**
+     * helper for {@link MetadataType.createREST}
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @param {TYPE.MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
+     * @returns {void}
+     */
+    static async postCreateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {
+        if (!apiResponse?.[this.definition.idField]) {
+            return;
+        }
+        Util.logger.warn(
+            ` - ${this.definition.type} ${
+                metadataEntryWithAllFields?.[this.definition.idField]
+            }: new key ${
+                apiResponse?.[this.definition.idField]
+            } automatically assigned during creation`
+        );
+        metadataEntry[this.definition.idField] = apiResponse?.[this.definition.idField];
+
+        // map structure: cred/bu --> type --> old key --> new key
+        const buName = this.buObject.credential + '/' + this.buObject.businessUnit;
+        Automation.createdKeyMap ||= {};
+        Automation.createdKeyMap[buName] ||= {};
+        Automation.createdKeyMap[buName][this.definition.type] ||= {};
+        Automation.createdKeyMap[buName][this.definition.type][
+            metadataEntryWithAllFields[this.definition.idField]
+        ] = metadataEntry[this.definition.idField];
+    }
+
+    /**
+     * Updates a single Data Extract
+     *
+     * @param {TYPE.VerificationItem} metadata a single Data Extract
+     * @returns {Promise} Promise
+     */
+    static update(metadata) {
+        return super.updateREST(
+            metadata,
+            '/automation/v1/dataverifications/' + metadata.dataVerificationDefinitionId
+        );
+    }
+
+    /**
+     * prepares a verification for deployment
+     *
+     * @param {TYPE.VerificationItem} metadata a single verification activity definition
+     * @returns {TYPE.VerificationItem} metadata object
+     */
+    static preDeployTasks(metadata) {
+        metadata.targetObjectId = cache.searchForField(
+            'dataExtension',
+            metadata.r__dataExtension_CustomerKey,
+            'CustomerKey',
+            'ObjectID'
+        );
+        delete metadata.r__dataExtension_CustomerKey;
+        return metadata;
+    }
+    /**
+     * parses retrieved Metadata before saving
+     *
+     * @param {TYPE.VerificationItem} metadata a single verification activity definition
+     * @returns {TYPE.VerificationItem} Array with one metadata object and one sql string
+     */
+    static postRetrieveTasks(metadata) {
+        try {
+            metadata.r__dataExtension_CustomerKey = cache.searchForField(
+                'dataExtension',
+                metadata.targetObjectId,
+                'ObjectID',
+                'CustomerKey'
+            );
+            delete metadata.targetObjectId;
+        } catch (ex) {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}`
+            );
+        }
+        return metadata;
+    }
+    /**
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {string} key Identifier of item
+     * @returns {Promise.<boolean>} deletion success status
+     */
+    static deleteByKey(key) {
+        return super.deleteByKeyREST('/automation/v1/dataverifications/' + key, key);
+    }
+}
+
+// Assign definition to static attributes
+Verification.definition = require('../MetadataTypeDefinitions').verification;
+
+module.exports = Verification;

package/lib/metadataTypes/definitions/AttributeGroup.definition.js

@@ -8,8 +8,8 @@ module.exports = {
     nameField: 'definitionName.value',
     restPagination: false, // Hub API does not support pagination and returns everything instead
     type: 'attributeGroup',
-    typeDescription: 'BETA: Groupings of Set Definitions (Data Extensions) in Data Designer.',
-    typeRetrieveByDefault: false,
+    typeDescription: 'Groupings of Attribute Sets (Data Extensions) in Data Designer.',
+    typeRetrieveByDefault: true,
     typeName: 'Data Designer Attribute Groups',
     fields: {
         applicationID: {