mcdev 5.2.0 → 5.3.0

package/lib/MetadataTypeDefinitions.js

@@ -38,6 +38,7 @@ const MetadataTypeDefinitions = {
     transactionalSMS: require('./metadataTypes/definitions/TransactionalSMS.definition'),
     triggeredSend: require('./metadataTypes/definitions/TriggeredSend.definition'),
     user: require('./metadataTypes/definitions/User.definition'),
+    verification: require('./metadataTypes/definitions/Verification.definition'),
 };
 
 module.exports = MetadataTypeDefinitions;

package/lib/MetadataTypeInfo.js

@@ -38,6 +38,7 @@ const MetadataTypeInfo = {
     transactionalSMS: require('./metadataTypes/TransactionalSMS'),
     triggeredSend: require('./metadataTypes/TriggeredSend'),
     user: require('./metadataTypes/User'),
+    verification: require('./metadataTypes/Verification'),
 };
 
 module.exports = MetadataTypeInfo;

package/lib/cli.js

@@ -45,7 +45,7 @@ yargs
         },
     })
     .command({
-        command: 'deploy [BU] [TYPE] [KEY] [--fromRetrieve] [--refresh]',
+        command: 'deploy [BU] [TYPE] [KEY]',
         aliases: ['d'],
         desc: 'deploys local metadata to a business unit',
         builder: (yargs) => {
@@ -97,6 +97,11 @@ yargs
                     group: 'Options for deploy:',
                     describe:
                         'optionally start existing schedule instead of running item once immediately (only works for automations)',
+                })
+                .option('fixShared', {
+                    group: 'Options for deploy:',
+                    describe:
+                        "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)",
                 });
         },
         handler: (argv) => {

package/lib/index.js

@@ -56,12 +56,14 @@ class Mcdev {
             'commitHistory',
             'execute',
             'filter',
+            'fixShared',
             'fromRetrieve',
             'json',
             'like',
             'noLogColors',
             'noLogFile',
             'refresh',
+            '_runningTest',
             'schedule',
             'skipInteraction',
         ];
@@ -310,7 +312,8 @@ class Mcdev {
                 triggeredSend: 'triggeredSendDefinition',
                 user: 'accountUser',
             };
-            Util.logger.info(`:: Retrieving ${cred}/${bu}\n`);
+            Util.logger.info('');
+            Util.logger.info(`:: Retrieving ${cred}/${bu}`);
             const retrieveTypesArr = [];
             if (selectedTypesArr) {
                 for (const selectedType of Array.isArray(selectedTypesArr)

package/lib/metadataTypes/AttributeSet.js

@@ -31,13 +31,7 @@ class AttributeSet extends MetadataType {
             const result = await AttributeGroup.retrieveForCache();
             cache.setMetadata('attributeGroup', result.metadata);
         }
-        return super.retrieveREST(
-            retrieveDir,
-            '/hub/v1/contacts/schema/setDefinitions',
-            null,
-            null,
-            key
-        );
+        return super.retrieveREST(retrieveDir, '/hub/v1/contacts/schema/setDefinitions', null, key);
     }
     /**
      * Retrieves Metadata of schema set definitions for caching.
@@ -47,6 +41,110 @@ class AttributeSet extends MetadataType {
     static retrieveForCache() {
         return super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
     }
+    /**
+     * used to identify updated shared data extensions that are used in attributeSets.
+     * helper for DataExtension.#fixShared_onBU
+     *
+     * @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
+     * @param {object} fixShared_fields DataExtensionField.fixShared_fields
+     * @returns {Promise.<string[]>} Promise of list of shared dataExtension IDs
+     */
+    static async fixShared_retrieve(sharedDataExtensionMap, fixShared_fields) {
+        if (!Object.keys(sharedDataExtensionMap).length) {
+            return [];
+        }
+        const result = await super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
+        const metadataMap = result?.metadata;
+        if (metadataMap && Object.keys(metadataMap).length) {
+            const sharedDeIds = Object.keys(metadataMap)
+                .filter(
+                    (asKey) =>
+                        metadataMap[asKey].storageLogicalType === 'ExactTargetSchema' ||
+                        metadataMap[asKey].storageLogicalType === 'DataExtension'
+                )
+                .filter((asKey) => {
+                    // check if dataExtension ID is found on any attributeSet of this BU
+                    if (sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]) {
+                        Util.logger.debug(
+                            ` shared dataExtension ID ${metadataMap[asKey].storageReferenceID.value} found in attributeSet ${asKey}`
+                        );
+                        return true;
+                    } else {
+                        return false;
+                    }
+                })
+                .filter((asKey) => {
+                    // check if any of the dataExtension fields dont exist on the attributeSet or are out of date
+                    const deKey =
+                        sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value];
+                    const asFields = metadataMap[asKey].valueDefinitions;
+                    const deFields = Object.values(fixShared_fields[deKey]);
+                    return deFields.some((deField) => {
+                        const search = asFields.filter((asf) => asf.name === deField.Name);
+                        if (!search.length) {
+                            Util.logger.debug(
+                                Util.getGrayMsg(
+                                    ` - Field ${deField.Name} not found in attributeSet; Note: only first recognized difference is printed to log`
+                                )
+                            );
+                            return true;
+                        }
+                        const asField = search[0];
+                        if (asField.dataType !== deField.FieldType) {
+                            Util.logger.debug(
+                                Util.getGrayMsg(
+                                    ` - Field ${deField.Name} FieldType changed (old: ${asField.dataType}; new: ${deField.FieldType}); Note: only first recognized difference is printed to log`
+                                )
+                            );
+                            return true;
+                        }
+                        asField.defaultValue ||= '';
+                        if (
+                            (asField.defaultValue && deField.DefaultValue === '') ||
+                            (deField.FieldType === 'Boolean' &&
+                                deField.DefaultValue !== '' &&
+                                (deField.DefaultValue
+                                    ? 'True'
+                                    : 'False' !== asField.defaultValue)) ||
+                            (deField.FieldType !== 'Boolean' &&
+                                deField.DefaultValue !== asField.defaultValue)
+                        ) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} DefaultValue changed (old: ${asField.defaultValue}; new: ${deField.DefaultValue}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        // some field types don't carry the length property. reset to 0 to ease comparison
+                        asField.length ||= 0;
+                        if (asField.length !== deField.MaxLength) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} MaxLength changed (old: ${asField.length}; new: ${deField.MaxLength}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        if (asField.isNullable !== deField.IsRequired) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} IsRequired changed (old: ${asField.isNullable}; new: ${deField.IsRequired}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        if (asField.isPrimaryKey !== deField.IsPrimaryKey) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} IsPrimaryKey changed (old: ${asField.isPrimaryKey}; new: ${deField.IsPrimaryKey}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        return false;
+                    });
+                })
+                .map((key) => metadataMap[key].storageReferenceID.value)
+                .filter(Boolean);
+            return sharedDeIds;
+        } else {
+            // nothing to do - return empty array
+            return [];
+        }
+    }
 
     /**
      * Builds map of metadata entries mapped to their keyfields
@@ -83,7 +181,7 @@ class AttributeSet extends MetadataType {
         switch (metadata.storageLogicalType) {
             case 'ExactTargetSchema': // synced / shared DEs
             case 'DataExtension': {
-                // local DEs
+                // shared / local DEs
                 try {
                     metadata.r__dataExtension_CustomerKey = cache.searchForField(
                         'dataExtension',
@@ -226,6 +324,12 @@ class AttributeSet extends MetadataType {
         // Member ID
         delete metadata.customObjectOwnerMID;
 
+        // remove duplicate ID fields (main field is definitionID)
+        delete metadata.setDefinitionID;
+        if (metadata.dataRetentionProperties?.setDefinitionID) {
+            delete metadata.dataRetentionProperties?.setDefinitionID;
+        }
+
         // connectingID.identifierType seems to be always set to 'FullyQualifiedName' - to be sure we check it here and remove it if it's the case
         if (metadata.connectingID?.identifierType === 'FullyQualifiedName') {
             // remove useless field
@@ -241,8 +345,11 @@ class AttributeSet extends MetadataType {
      * @returns {object[]} all system value definitions
      */
     static _getSystemValueDefinitions() {
-        if (!this.systemValueDefinitions) {
-            this.systemValueDefinitions = Object.values(cache.getCache()['attributeSet'])
+        this.systemValueDefinitions ||= {};
+        if (!this.systemValueDefinitions[this.buObject.mid]) {
+            this.systemValueDefinitions[this.buObject.mid] = Object.values(
+                cache.getCache()['attributeSet']
+            )
                 .flatMap((item) => {
                     if (item.isSystemDefined) {
                         return item.valueDefinitions;
@@ -250,7 +357,7 @@ class AttributeSet extends MetadataType {
                 })
                 .filter(Boolean);
         }
-        return this.systemValueDefinitions;
+        return this.systemValueDefinitions[this.buObject.mid];
     }
 }
 

package/lib/metadataTypes/Automation.js

@@ -25,71 +25,53 @@ class Automation extends MetadataType {
      * @returns {Promise.<TYPE.AutomationMapObj>} Promise of metadata
      */
     static async retrieve(retrieveDir, _, __, key) {
-        /** @type {TYPE.SoapRequestParams} */
-        let requestParams = null;
-        if (key) {
-            requestParams = {
-                filter: {
-                    leftOperand: 'CustomerKey',
-                    operator: 'equals',
-                    rightOperand: key,
-                },
-            };
-        }
-        const results = await this.client.soap.retrieveBulk('Program', ['ObjectID'], requestParams);
-        if (results.Results?.length && !key) {
-            // empty results will come back without "Results" defined
-            Util.logger.info(
-                Util.getGrayMsg(
-                    ` - ${results.Results?.length} automation${
-                        results.Results?.length === 1 ? '' : 's'
-                    } found. Retrieving details...`
-                )
+        let metadataMap;
+        if (key && this._cachedMetadataMap?.[key]) {
+            metadataMap = {};
+            metadataMap[key] = this._cachedMetadataMap[key];
+            delete this._cachedMetadataMap;
+        } else if (!key && this._cachedMetadataMap) {
+            metadataMap = this._cachedMetadataMap;
+            delete this._cachedMetadataMap;
+        } else {
+            /** @type {TYPE.SoapRequestParams} */
+            let requestParams = null;
+            if (key) {
+                requestParams = {
+                    filter: {
+                        leftOperand: 'CustomerKey',
+                        operator: 'equals',
+                        rightOperand: key,
+                    },
+                };
+            }
+            const results = await this.client.soap.retrieveBulk(
+                'Program',
+                ['ObjectID'],
+                requestParams
             );
-        }
-        // the API seems to handle 50 concurrent requests nicely
-        const rateLimit = pLimit(50);
-
-        const details = results.Results
-            ? await Promise.all(
-                  results.Results.map(async (item) =>
-                      rateLimit(async () => {
-                          try {
-                              return await this.client.rest.get(
-                                  '/automation/v1/automations/' + item.ObjectID
-                              );
-                          } catch (ex) {
-                              try {
-                                  if (ex.message == 'socket hang up') {
-                                      // one more retry; it's a rare case but retrying again should solve the issue gracefully
-                                      return await this.client.rest.get(
-                                          '/automation/v1/automations/' + item.ObjectID
-                                      );
-                                  }
-                              } catch {
-                                  // no extra action needed, handled below
-                              }
-                              // if we do get here, we should log the error and continue instead of failing to download all automations
-                              Util.logger.error(
-                                  ` ☇ skipping Automation ${item.ObjectID}: ${ex.message} ${ex.code}`
-                              );
-                              return null;
-                          }
-                      })
+            // the API seems to handle 50 concurrent requests nicely
+            const response = results?.Results?.length
+                ? await this.retrieveRESTcollection(
+                      results?.Results.map((item) => ({
+                          id: item.ObjectID,
+                          uri: '/automation/v1/automations/' + item.ObjectID,
+                      })),
+                      50,
+                      !key
                   )
-              )
-            : [];
-
-        // * if retrieving some automations fails, a null element would remain in the details-array for each of them that needs to be filtered to prevent it from causing issues elsewhere
-        let metadataMap = this.parseResponseBody({ items: details.filter(Boolean) });
+                : null;
+            metadataMap = response?.metadata || {};
+        }
 
-        if (Object.keys(metadataMap).length) {
+        if (!this._skipNotificationRetrieve && Object.keys(metadataMap).length) {
             // attach notification information to each automation that has any
             await this.#getAutomationNotificationsREST(metadataMap);
         }
 
         // * retrieveDir can be empty when we use it in the context of postDeployTasks
         if (retrieveDir) {
+            this.retrieveDir = retrieveDir;
             metadataMap = await this.saveResults(metadataMap, retrieveDir, null, null);
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(metadataMap).length})` +
@@ -101,6 +83,27 @@ class Automation extends MetadataType {
         return { metadata: metadataMap, type: this.definition.type };
     }
 
+    /**
+     * helper for {@link this.retrieveRESTcollection}
+     *
+     * @param {Error} ex exception
+     * @param {string} id id or key of item
+     * @returns {null} -
+     */
+    static async handleRESTErrors(ex, id) {
+        try {
+            if (ex.message == 'socket hang up') {
+                // one more retry; it's a rare case but retrying again should solve the issue gracefully
+                return await this.client.rest.get('/automation/v1/automations/' + id);
+            }
+        } catch {
+            // no extra action needed, handled below
+        }
+        // if we do get here, we should log the error and continue instead of failing to download all automations
+        Util.logger.error(` ☇ skipping Automation ${id}: ${ex.message} ${ex.code}`);
+        return null;
+    }
+
     /**
      * helper for {@link Automation.retrieve} to get Automation Notifications
      *
@@ -230,12 +233,18 @@ class Automation extends MetadataType {
      * @returns {Promise.<TYPE.AutomationMapObj>} Promise of metadata
      */
     static async retrieveForCache() {
-        // get automations for cache
-        const results = await this.client.soap.retrieveBulk('Program', [
-            'ObjectID',
-            'CustomerKey',
-            'Name',
-        ]);
+        let results = {};
+        if (this._cachedMetadataMap) {
+            results.Results = Object.values(this._cachedMetadataMap);
+            delete this._cachedMetadataMap;
+        } else {
+            // get automations for cache
+            results = await this.client.soap.retrieveBulk('Program', [
+                'ObjectID',
+                'CustomerKey',
+                'Name',
+            ]);
+        }
         const resultsConverted = {};
         if (Array.isArray(results?.Results)) {
             // get encodedAutomationID to retrieve notification information
@@ -252,16 +261,16 @@ class Automation extends MetadataType {
 
             // merge encodedAutomationID into results
             for (const m of results.Results) {
-                resultsConverted[m.CustomerKey] = {
-                    id: m.ObjectID,
-                    key: m.CustomerKey,
-                    name: m.Name,
-                    programId: automationsLegacy.metadata[m.CustomerKey]?.id,
-                    status: automationsLegacy.metadata[m.CustomerKey]?.status,
+                const key = m.CustomerKey || m.key;
+                resultsConverted[key] = {
+                    id: m.ObjectID || m.id,
+                    key: key,
+                    name: m.Name || m.name,
+                    programId: automationsLegacy.metadata[key]?.id,
+                    status: automationsLegacy.metadata[key]?.status,
                 };
             }
         }
-
         return { metadata: resultsConverted, type: this.definition.type };
     }
 
@@ -423,6 +432,7 @@ class Automation extends MetadataType {
                                 } due to missing activityObjectId: ${JSON.stringify(activity)}`
                             );
                             // empty if block
+                            continue;
                         } else if (!this.definition.dependencies.includes(activity.r__type)) {
                             Util.logger.debug(
                                 ` - skipping ${
@@ -433,9 +443,10 @@ class Automation extends MetadataType {
                                     activity.r__type
                                 } is not set up as a dependency for ${this.definition.type}`
                             );
+                            continue;
                         }
                         // / if managed by cache we can update references to support deployment
-                        else if (
+                        if (
                             Definitions[activity.r__type]?.['idField'] &&
                             cache.getCache(this.buObject.mid)[activity.r__type]
                         ) {
@@ -803,6 +814,7 @@ class Automation extends MetadataType {
             delete metadata.schedule;
             delete metadata.type;
             let i = 0;
+            const buName = this.buObject.credential + '/' + this.buObject.businessUnit;
             if (metadata.steps) {
                 for (const step of metadata.steps) {
                     let displayOrder = 0;
@@ -812,6 +824,23 @@ class Automation extends MetadataType {
                             activity.name &&
                             this.definition.dependencies.includes(activity.r__type)
                         ) {
+                            if (
+                                activity.r__type === 'verification' &&
+                                this.createdKeyMap?.[buName]?.verification?.[activity.name]
+                            ) {
+                                Util.logger.info(
+                                    Util.getGrayMsg(
+                                        ` - updated verification activity name from ${
+                                            activity.name
+                                        } to ${
+                                            this.createdKeyMap[buName].verification[activity.name]
+                                        }`
+                                    )
+                                );
+                                // map structure: cred/bu --> type --> old key --> new key
+                                activity.name =
+                                    this.createdKeyMap[buName].verification[activity.name];
+                            }
                             // automations can have empty placeholder for activities with only their type defined
                             activity.activityObjectId = cache.searchForField(
                                 activity.r__type,