mcdev 5.1.0 → 5.3.0

package/lib/metadataTypes/DataExtension.js

@@ -5,11 +5,13 @@ const TYPE = require('../../types/mcdev.d');
 const MetadataType = require('./MetadataType');
 const DataExtensionField = require('./DataExtensionField');
 const Folder = require('./Folder');
+const AttributeSet = require('./AttributeSet');
 const Util = require('../util/util');
 const File = require('../util/file');
 const auth = require('../util/auth');
 const cache = require('../util/cache');
 const pLimit = require('p-limit');
+const inquirer = require('inquirer');
 
 /**
  * DataExtension MetadataType
@@ -45,7 +47,7 @@ class DataExtension extends MetadataType {
                   };
         }
         Util.logger.info(` - Caching dependent Metadata: dataExtensionField`);
-        await this._attachFields(metadataMap, fieldOptions);
+        await this.#attachFields(metadataMap, fieldOptions);
 
         /** @type {object[]} */
         const metadataToCreate = [];
@@ -203,7 +205,7 @@ class DataExtension extends MetadataType {
         }
     }
     /**
-     * helper for {@link upsert}
+     * helper for {@link DataExtension.upsert}
      *
      * @private
      * @param {object} res -
@@ -283,7 +285,7 @@ class DataExtension extends MetadataType {
      * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
      * @returns {void}
      */
-    static postDeployTasks(upsertedMetadata, originalMetadata, createdUpdated) {
+    static async postDeployTasks(upsertedMetadata, originalMetadata, createdUpdated) {
         for (const key in upsertedMetadata) {
             const item = upsertedMetadata[key];
 
@@ -354,6 +356,376 @@ class DataExtension extends MetadataType {
                 DataExtensionField.postRetrieveTasks(field, true);
             }
         }
+        await this.#fixShared(upsertedMetadata, originalMetadata, createdUpdated);
+    }
+
+    /**
+     * takes care of updating attribute groups on child BUs after an update to Shared DataExtensions
+     * helper for {@link DataExtension.postDeployTasks}
+     * fixes an issue where shared data extensions are not visible in data designer on child BU; SF known issue: https://issues.salesforce.com/#q=W-11031095
+     *
+     * @param {TYPE.DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     * @param {TYPE.DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
+     * @returns {void}
+     */
+    static async #fixShared(upsertedMetadata, originalMetadata, createdUpdated) {
+        if (this.buObject.eid !== this.buObject.mid) {
+            // only if we were executing a deploy on parent bu could we be deploying shared data extensions
+            Util.logger.debug(`Skipping fixShared logic because we are not executing on Parent BU`);
+            return;
+        }
+        if (createdUpdated.updated === 0) {
+            // only if updates were made could the issue in https://issues.salesforce.com/#q=W-11031095 affect data designer
+            Util.logger.debug(`Skipping fixShared logic because nothing was updated`);
+            return;
+        }
+
+        // find all shared data extensions
+        if (!this.deployedSharedKeys.length) {
+            Util.logger.debug(
+                `Skipping fixShared logic because no Shared Data Extensions were updated`
+            );
+            return;
+        }
+
+        const sharedDataExtensionsKeys = this.deployedSharedKeys;
+        this.deployedSharedKeys = null;
+
+        if (Util.OPTIONS.fixShared) {
+            // select which BUs to run this for
+            const selectedBuNames = await this.#fixShared_getBUs();
+
+            // backup settings
+            const buObjectBak = this.buObject;
+            const clientBak = this.client;
+
+            // get dataExtension ID-Key relationship
+            const sharedDataExtensionMap = {};
+            for (const key of sharedDataExtensionsKeys) {
+                try {
+                    const id = cache.searchForField(
+                        'dataExtension',
+                        key,
+                        'CustomerKey',
+                        'ObjectID',
+                        this.buObject.eid
+                    );
+                    sharedDataExtensionMap[id] = key;
+                } catch {
+                    continue;
+                }
+            }
+
+            // run the fix-data-model logic
+            Util.logger.info(
+                `Fixing Shared Data Extensions details in data models of child BUs` +
+                    Util.getKeysString(sharedDataExtensionsKeys)
+            );
+
+            for (const buName of selectedBuNames) {
+                await this.#fixShared_onBU(buName, buObjectBak, clientBak, sharedDataExtensionMap);
+            }
+            Util.logger.info(`Finished fixing Shared Data Extensions details in data models`);
+
+            // restore settings
+            this.buObject = buObjectBak;
+            this.client = clientBak;
+        } else {
+            Util.logger.warn(
+                'Shared Data Extensions were updated but --fixShared option is not set. This can result in your changes not being visible in attribute groups on child BUs.'
+            );
+            Util.logger.info(
+                'We recommend to re-run your deployment with the --fixShared option unless you are sure your Shared Data Extension is not used in attribute groups on any child BU.'
+            );
+        }
+    }
+
+    /**
+     * helper for {@link DataExtension.#fixShared}
+     *
+     * @returns {string[]} list of selected BU names
+     */
+    static async #fixShared_getBUs() {
+        const buListObj = this.properties.credentials[this.buObject.credential].businessUnits;
+        const fixBuPreselected = [];
+        const availableBuNames = Object.keys(buListObj).filter(
+            (buName) => buName !== Util.parentBuName
+        );
+        if (typeof Util.OPTIONS.fixShared === 'string') {
+            if (Util.OPTIONS.fixShared === '*') {
+                // pre-select all BUs
+                fixBuPreselected.push(...availableBuNames);
+            } else {
+                // pre-select BUs from comma-separated list
+                fixBuPreselected.push(
+                    ...Util.OPTIONS.fixShared
+                        .split(',')
+                        .filter(Boolean)
+                        .map((bu) => bu.trim())
+                        .filter((bu) => availableBuNames.includes(bu))
+                );
+            }
+        }
+        if (Util.skipInteraction && fixBuPreselected.length) {
+            // assume programmatic use case or user that wants to skip the wizard. Use pre-selected BUs
+            return fixBuPreselected;
+        }
+
+        const buList = availableBuNames.map((name) => ({
+            name,
+            value: name,
+            checked: fixBuPreselected.includes(name),
+        }));
+        const questions = {
+            type: 'checkbox',
+            name: 'businessUnits',
+            message: 'Please select BUs that have access to the updated Shared Data Extensions:',
+            pageSize: 10,
+            choices: buList,
+        };
+        let responses = null;
+
+        try {
+            responses = await inquirer.prompt(questions);
+        } catch (ex) {
+            Util.logger.info(ex);
+        }
+        return responses.businessUnits;
+    }
+
+    /**
+     * helper for {@link DataExtension.#fixShared}
+     *
+     * @param {string} childBuName name of child BU to fix
+     * @param {TYPE.BuObject} buObjectParent bu object for parent BU
+     * @param {object} clientParent SDK for parent BU
+     * @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
+     * @returns {Promise.<string[]>} updated shared DE keys on BU
+     */
+    static async #fixShared_onBU(
+        childBuName,
+        buObjectParent,
+        clientParent,
+        sharedDataExtensionMap
+    ) {
+        /** @type {TYPE.BuObject} */
+        const buObjectChildBu = {
+            eid: this.properties.credentials[buObjectParent.credential].eid,
+            mid: this.properties.credentials[buObjectParent.credential].businessUnits[childBuName],
+            businessUnit: childBuName,
+            credential: this.buObject.credential,
+        };
+        const clientChildBu = auth.getSDK(buObjectChildBu);
+
+        try {
+            // check if shared data Extension is used in an attributeSet on current BU
+            AttributeSet.properties = this.properties;
+            AttributeSet.buObject = buObjectChildBu;
+            AttributeSet.client = clientChildBu;
+            const sharedDeIdsUsedOnBU = await AttributeSet.fixShared_retrieve(
+                sharedDataExtensionMap,
+                DataExtensionField.fixShared_fields
+            );
+            if (sharedDeIdsUsedOnBU.length) {
+                let sharedDataExtensionsKeys = sharedDeIdsUsedOnBU.map(
+                    (deId) => sharedDataExtensionMap[deId]
+                );
+                Util.logger.info(
+                    ` - Fixing dataExtensions on BU ${childBuName} ` +
+                        Util.getKeysString(sharedDataExtensionsKeys)
+                );
+
+                for (const deId of sharedDeIdsUsedOnBU) {
+                    // dont use Promise.all to ensure order of execution; otherwise, switched BU contexts in one step will affect the next
+                    const fixed = await this.#fixShared_item(
+                        deId,
+                        sharedDataExtensionMap[deId],
+                        buObjectChildBu,
+                        clientChildBu,
+                        buObjectParent,
+                        clientParent
+                    );
+                    if (!fixed) {
+                        // remove from list of shared DEs that were fixed
+                        sharedDataExtensionsKeys = sharedDataExtensionsKeys.filter(
+                            (key) => key !== sharedDataExtensionMap[deId]
+                        );
+                    }
+                }
+                if (sharedDataExtensionsKeys.length) {
+                    Util.logger.debug(
+                        ` - Fixed ${sharedDataExtensionsKeys.length}/${
+                            sharedDeIdsUsedOnBU.length
+                        }: ${sharedDataExtensionsKeys.join(', ')}`
+                    );
+                }
+                return sharedDataExtensionsKeys;
+            } else {
+                Util.logger.info(
+                    Util.getGrayMsg(
+                        ` - No matching attributeSet found for given Shared Data Extensions keys found on BU ${childBuName}`
+                    )
+                );
+                return [];
+            }
+        } catch (ex) {
+            Util.logger.error(ex.message);
+            return [];
+        }
+    }
+
+    /**
+     * method that actually takes care of triggering the update for a particular BU-sharedDe combo
+     * helper for {@link DataExtension.#fixShared_onBU}
+     *
+     * @param {string} deId data extension ObjectID
+     * @param {string} deKey dataExtension key
+     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {object} clientChildBu SDK for child BU
+     * @param {TYPE.BuObject} buObjectParent BU object for Parent BU
+     * @param {object} clientParent SDK for parent BU
+     * @returns {Promise.<boolean>} flag that signals if the fix was successful
+     */
+    static async #fixShared_item(
+        deId,
+        deKey,
+        buObjectChildBu,
+        clientChildBu,
+        buObjectParent,
+        clientParent
+    ) {
+        try {
+            // add field via child BU
+            const randomSuffix = await DataExtension.#fixShared_item_addField(
+                buObjectChildBu,
+                clientChildBu,
+                deKey,
+                deId
+            );
+
+            // get field ID from parent BU (it is not returned on child BU)
+            const fieldObjectID = await DataExtension.#fixShared_item_getFieldId(
+                randomSuffix,
+                buObjectParent,
+                clientParent,
+                deKey
+            );
+
+            // delete field via child BU
+            await DataExtension.#fixShared_item_deleteField(
+                randomSuffix,
+                buObjectChildBu,
+                clientChildBu,
+                deKey,
+                fieldObjectID
+            );
+
+            Util.logger.info(
+                `   - Fixed dataExtension ${deKey} on BU ${buObjectChildBu.businessUnit}`
+            );
+
+            return true;
+        } catch (ex) {
+            Util.logger.error(
+                `- error fixing dataExtension ${deKey} on BU ${buObjectChildBu.businessUnit}: ${ex.message}`
+            );
+            return false;
+        }
+    }
+
+    /**
+     * add a new field to the shared DE to trigger an update to the data model
+     * helper for {@link DataExtension.#fixShared_item}
+     *
+     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {object} clientChildBu SDK for child BU
+     * @param {string} deKey dataExtension key
+     * @param {string} deId dataExtension ObjectID
+     * @returns {Promise.<string>} randomSuffix
+     */
+    static async #fixShared_item_addField(buObjectChildBu, clientChildBu, deKey, deId) {
+        this.buObject = buObjectChildBu;
+        this.client = clientChildBu;
+        const randomSuffix = Util.OPTIONS._runningTest
+            ? '_randomNumber_'
+            : Math.floor(Math.random() * 9999999999);
+        // add a new field to the shared DE to trigger an update to the data model
+        const soapType = this.definition.soapType || this.definition.type;
+        await this.client.soap.update(
+            Util.capitalizeFirstLetter(soapType),
+            {
+                CustomerKey: deKey,
+                ObjectID: deId,
+                Fields: {
+                    Field: [
+                        {
+                            Name: 'TriggerUpdate' + randomSuffix,
+                            IsRequired: false,
+                            IsPrimaryKey: false,
+                            FieldType: 'Boolean',
+                            ObjectID: null,
+                        },
+                    ],
+                },
+            },
+            null
+        );
+        return randomSuffix;
+    }
+
+    /**
+     * get ID of the field added by {@link DataExtension.#fixShared_item_addField} on the shared DE via parent BU
+     * helper for {@link DataExtension.#fixShared_item}
+     *
+     * @param {string} randomSuffix -
+     * @param {TYPE.BuObject} buObjectParent BU object for Parent BU
+     * @param {object} clientParent SDK for parent BU
+     * @param {string} deKey dataExtension key
+     * @returns {Promise.<string>} fieldObjectID
+     */
+    static async #fixShared_item_getFieldId(randomSuffix, buObjectParent, clientParent, deKey) {
+        DataExtensionField.buObject = buObjectParent;
+        DataExtensionField.client = clientParent;
+        const fieldKey = `[${deKey}].[TriggerUpdate${randomSuffix}]`;
+        const fieldResponse = await DataExtensionField.retrieveForCache(
+            {
+                filter: {
+                    leftOperand: 'CustomerKey',
+                    operator: 'equals',
+                    rightOperand: fieldKey,
+                },
+            },
+            ['Name', 'ObjectID']
+        );
+        const fieldObjectID = fieldResponse.metadata[fieldKey]?.ObjectID;
+        return fieldObjectID;
+    }
+
+    /**
+     * delete the field added by {@link DataExtension.#fixShared_item_addField}
+     * helper for {@link DataExtension.#fixShared_item}
+     *
+     * @param {string} randomSuffix -
+     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {object} clientChildBu SDK for child BU
+     * @param {string} deKey dataExtension key
+     * @param {string} fieldObjectID field ObjectID
+     * @returns {Promise} -
+     */
+    static async #fixShared_item_deleteField(
+        randomSuffix,
+        buObjectChildBu,
+        clientChildBu,
+        deKey,
+        fieldObjectID
+    ) {
+        DataExtensionField.buObject = buObjectChildBu;
+        DataExtensionField.client = clientChildBu;
+        await DataExtensionField.deleteByKeySOAP(
+            deKey + '.TriggerUpdate' + randomSuffix,
+            fieldObjectID
+        );
     }
 
     /**
@@ -390,70 +762,10 @@ class DataExtension extends MetadataType {
         // in case of cache dont get fields
         if (metadata && retrieveDir) {
             // get fields from API
-            await this._attachFields(metadata, fieldOptions, additionalFields);
+            await this.#attachFields(metadata, fieldOptions, additionalFields);
         }
         if (!retrieveDir && this.buObject.eid !== this.buObject.mid) {
-            // for caching, we want to retrieve shared DEs as well from the instance parent BU
-            Util.logger.info(
-                ' - Caching dependent Metadata: dataExtension (shared via _ParentBU_)'
-            );
-            /** @type {TYPE.BuObject} */
-            const buObjectParentBu = {
-                eid: this.properties.credentials[this.buObject.credential].eid,
-                mid: this.properties.credentials[this.buObject.credential].eid,
-                businessUnit: Util.parentBuName,
-                credential: this.buObject.credential,
-            };
-            try {
-                this.client = auth.getSDK(buObjectParentBu);
-            } catch (ex) {
-                Util.logger.error(ex.message);
-                return;
-            }
-            const metadataParentBu = await this._retrieveAll(additionalFields);
-
-            // get shared folders to match our shared / synched Data Extensions
-            const subTypeArr = this.definition.dependencies
-                .filter((item) => item.startsWith('folder-'))
-                .map((item) => item.slice(7));
-            Util.logger.info(' - Caching dependent Metadata: folder (shared via _ParentBU_)');
-            Util.logSubtypes(subTypeArr);
-            Folder.client = this.client;
-            Folder.buObject = buObjectParentBu;
-            Folder.properties = this.properties;
-            const result = await Folder.retrieveForCache(null, subTypeArr);
-            cache.mergeMetadata('folder', result.metadata, this.buObject.eid);
-
-            // get the types and clean out non-shared ones
-            const folderTypesFromParent = require('../MetadataTypeDefinitions').folder
-                .folderTypesFromParent;
-            for (const metadataEntry in metadataParentBu) {
-                try {
-                    // get the data extension type from the folder
-                    const folderContentType = cache.searchForField(
-                        'folder',
-                        metadataParentBu[metadataEntry].CategoryID,
-                        'ID',
-                        'ContentType',
-                        this.buObject.eid
-                    );
-                    if (!folderTypesFromParent.includes(folderContentType)) {
-                        Util.logger.verbose(
-                            `removing ${metadataEntry} because r__folder_ContentType '${folderContentType}' identifies this DE as not being shared`
-                        );
-                        delete metadataParentBu[metadataEntry];
-                    }
-                } catch (ex) {
-                    Util.logger.debug(
-                        `removing ${metadataEntry} because of error while retrieving r__folder_ContentType: ${ex.message}`
-                    );
-                    delete metadataParentBu[metadataEntry];
-                }
-            }
-
-            // revert client to current default
-            this.client = auth.getSDK(this.buObject);
-            Folder.client = auth.getSDK(this.buObject);
+            const metadataParentBu = await this.retrieveSharedForCache(additionalFields);
 
             // make sure to overwrite parent bu DEs with local ones
             metadata = { ...metadataParentBu, ...metadata };
@@ -469,6 +781,82 @@ class DataExtension extends MetadataType {
         return { metadata: metadata, type: 'dataExtension' };
     }
 
+    /**
+     * get shared dataExtensions from parent BU and merge them into the cache
+     * helper for {@link DataExtension.retrieve} and for AttributeSet.fixShared_retrieve
+     *
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
+     */
+    static async retrieveSharedForCache(additionalFields = []) {
+        // for caching, we want to retrieve shared DEs as well from the instance parent BU
+        Util.logger.info(' - Caching dependent Metadata: dataExtension (shared via _ParentBU_)');
+        const buObjectBak = this.buObject;
+        const clientBak = this.client;
+        /** @type {TYPE.BuObject} */
+        const buObjectParentBu = {
+            eid: this.properties.credentials[this.buObject.credential].eid,
+            mid: this.properties.credentials[this.buObject.credential].eid,
+            businessUnit: Util.parentBuName,
+            credential: this.buObject.credential,
+        };
+        try {
+            this.buObject = buObjectParentBu;
+            this.client = auth.getSDK(buObjectParentBu);
+        } catch (ex) {
+            Util.logger.error(ex.message);
+            return;
+        }
+        const metadataParentBu = await this._retrieveAll(additionalFields);
+
+        // get shared folders to match our shared / synched Data Extensions
+        const subTypeArr = this.definition.dependencies
+            .filter((item) => item.startsWith('folder-'))
+            .map((item) => item.slice(7));
+        Util.logger.info(' - Caching dependent Metadata: folder (shared via _ParentBU_)');
+        Util.logSubtypes(subTypeArr);
+        Folder.client = this.client;
+        Folder.buObject = this.buObject;
+        Folder.properties = this.properties;
+        const result = await Folder.retrieveForCache(null, subTypeArr);
+        cache.mergeMetadata('folder', result.metadata, this.buObject.eid);
+
+        // get the types and clean out non-shared ones
+        const folderTypesFromParent = require('../MetadataTypeDefinitions').folder
+            .folderTypesFromParent;
+        for (const metadataEntry in metadataParentBu) {
+            try {
+                // get the data extension type from the folder
+                const folderContentType = cache.searchForField(
+                    'folder',
+                    metadataParentBu[metadataEntry].CategoryID,
+                    'ID',
+                    'ContentType',
+                    this.buObject.eid
+                );
+                if (!folderTypesFromParent.includes(folderContentType)) {
+                    // Util.logger.verbose(
+                    //     `removing ${metadataEntry} because r__folder_ContentType '${folderContentType}' identifies this DE as not being shared`
+                    // );
+                    delete metadataParentBu[metadataEntry];
+                }
+            } catch (ex) {
+                Util.logger.debug(
+                    `removing ${metadataEntry} because of error while retrieving r__folder_ContentType: ${ex.message}`
+                );
+                delete metadataParentBu[metadataEntry];
+            }
+        }
+
+        // revert client to current default
+        this.client = clientBak;
+        this.buObject = buObjectBak;
+        Folder.client = clientBak;
+        Folder.buObject = buObjectBak;
+
+        return metadataParentBu;
+    }
+
     /**
      * helper to retrieve all dataExtension fields and attach them to the dataExtension metadata
      *
@@ -478,7 +866,7 @@ class DataExtension extends MetadataType {
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @returns {Promise.<void>} -
      */
-    static async _attachFields(metadata, fieldOptions, additionalFields) {
+    static async #attachFields(metadata, fieldOptions, additionalFields) {
         const fieldsObj = await this._retrieveFields(fieldOptions, additionalFields);
         const fieldKeys = Object.keys(fieldsObj);
         // add fields to corresponding DE
@@ -607,7 +995,7 @@ class DataExtension extends MetadataType {
         metadata[customerKey].Fields = fieldArr;
     }
     /**
-     * helper for {@link super.updateREST} and {@link super.updateSOAP} that removes old files after the key was changed
+     * helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP} that removes old files after the key was changed
      *
      * @private
      * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
@@ -628,11 +1016,16 @@ class DataExtension extends MetadataType {
             throw new Error(`Cannot Upsert Strongly Typed Data Extensions`);
         }
         if (
+            !Util.OPTIONS._fixSharedOnBu &&
             this.buObject.eid !== this.buObject.mid &&
             metadata.r__folder_Path?.startsWith('Shared Items')
         ) {
             throw new Error(`Cannot Create/Update a Shared Data Extension from the Child BU`);
         }
+        if (metadata.r__folder_ContentType === 'shared_dataextension') {
+            this.deployedSharedKeys ||= [];
+            this.deployedSharedKeys.push(metadata.CustomerKey);
+        }
         if (metadata.r__folder_Path?.startsWith('Synchronized Data Extensions')) {
             throw new Error(
                 `Cannot Create/Update a Synchronized Data Extension. Please use Contact Builder to maintain these`
@@ -822,7 +1215,11 @@ class DataExtension extends MetadataType {
             for (const element of fieldsJson) {
                 const newJsonElement = {};
                 for (const field of fieldsToKeep) {
-                    newJsonElement[field] = element[field];
+                    if (field === 'MaxLength' && element.FieldType === 'Decimal') {
+                        newJsonElement.MaxLength = `${element.MaxLength},${element.Scale}`;
+                    } else {
+                        newJsonElement[field] = element[field];
+                    }
                 }
                 newJson.push(newJsonElement);
             }