mcdev 8.2.1 → 8.3.0

package/lib/metadataTypes/Filter.js

@@ -1,6 +1,16 @@
 'use strict';
 
 import MetadataType from './MetadataType.js';
+import { Util } from '../util/util.js';
+import cache from '../util/cache.js';
+import File from '../util/file.js';
+
+const dataTypes = {
+    1: 'List',
+    2: 'DataExtension',
+    3: 'Group Wizard',
+    4: 'Behavioral Data',
+};
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -13,6 +23,7 @@ import MetadataType from './MetadataType.js';
  * @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
  * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
  * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').FilterItem} FilterItem
  */
 
 /**
@@ -34,7 +45,310 @@ class Filter extends MetadataType {
      * @returns {Promise.<MetadataTypeMapObj>} Promise
      */
     static async retrieve(retrieveDir, _, __, key) {
-        return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null, key);
+        const objectId = key ? await this._getObjectIdForSingleRetrieve(key) : '';
+        return super.retrieveREST(retrieveDir, '/automation/v1/filters/' + objectId, null, key);
+    }
+    /**
+     * parses retrieved Metadata before saving
+     *
+     * @param {FilterItem} metadata a single record
+     * @returns {FilterItem} parsed metadata definition
+     */
+    static postRetrieveTasks(metadata) {
+        // folder
+        super.setFolderPath(metadata);
+
+        try {
+            // filterDefinition
+            metadata.r__dataFilter_key = cache.searchForField(
+                'dataFilter',
+                metadata.filterDefinitionId,
+                'id',
+                'key'
+            );
+            delete metadata.filterDefinitionId;
+        } catch {
+            try {
+                // filterDefinitionHidden - auto-generated for filtered dataExtensions
+                metadata.r__dataFilter_key = cache.searchForField(
+                    'dataFilterHidden',
+                    metadata.filterDefinitionId,
+                    'id',
+                    'key'
+                );
+                delete metadata.filterDefinitionId;
+            } catch {
+                // ignore
+            }
+        }
+
+        // rest API does not return description property when it's empty but we always want this as part of our json
+        metadata.description ||= '';
+
+        this._postRetrieve_dataTypeMapping('source', metadata);
+        this._postRetrieve_dataTypeMapping('destination', metadata);
+        return metadata;
+    }
+
+    /**
+     * helper for postRetrieveTasks to map data types
+     *
+     * @param {'source'|'destination'} target we are processing source and destinations
+     * @param {FilterItem} metadata single record
+     */
+    static _postRetrieve_dataTypeMapping(target, metadata) {
+        try {
+            switch (metadata[`${target}TypeId`]) {
+                // case 1: {
+                //     // List
+                //     // TODO
+                //     break;
+                case 2: {
+                    // dataExtension
+                    metadata[`r__${target}_dataExtension_key`] = cache.searchForField(
+                        'dataExtension',
+                        metadata[`${target}ObjectId`],
+                        'ObjectID',
+                        'CustomerKey'
+                    );
+                    delete metadata[`${target}ObjectId`];
+                    delete metadata[`${target}TypeId`];
+                    break;
+                }
+                // case 3: {
+                //     // GroupWizard
+                //     // TODO
+                //     break;
+                // }
+                // case 4: {
+                //     // BehavioralData
+                //     // TODO
+                //     break;
+                // }
+                default: {
+                    Util.logger.warn(
+                        ` - Filter '${metadata.name}' (${
+                            metadata.customerKey
+                        }): Unsupported ${target} type '${dataTypes[metadata[`${target}TypeId`]]}'`
+                    );
+                }
+            }
+        } catch (ex) {
+            Util.logger.warn(
+                ` - filter '${metadata.name}' (${metadata.customerKey}): ${target} not found (${ex.message})`
+            );
+        }
+    }
+    /**
+     * helper for preDeployTasks to map data types
+     *
+     * @param {'source'|'destination'} target we are processing source and destinations
+     * @param {FilterItem} metadata single record
+     * @param {FilterItem} cachedVersion cached version of the metadata
+     */
+    static _preDeploy_dataTypeMapping(target, metadata, cachedVersion) {
+        // TODO List / TypeId==1
+
+        // dataExtension
+        if (metadata[`r__${target}_dataExtension_key`]) {
+            metadata[`${target}TypeId`] = 2;
+
+            metadata[`${target}ObjectId`] = cache.searchForField(
+                'dataExtension',
+                metadata[`r__${target}_dataExtension_key`],
+                'CustomerKey',
+                'ObjectID'
+            );
+            if (
+                cachedVersion &&
+                cachedVersion[`${target}ObjectId`] !== metadata[`${target}ObjectId`]
+            ) {
+                throw new Error(
+                    `Updating r__${target}_dataExtension_key is not allowed. You need to delete and re-create the filter to change this.`
+                );
+                // metadata[`${target}ObjectId`] = cachedVersion[`${target}ObjectId`];
+            }
+            if (target === 'source') {
+                // this seems to be duplicated in update calls from the GUI
+                metadata.filterDefinitionSourceTypeId = metadata[`${target}TypeId`];
+                metadata.sourceId = null;
+            } else if (target === 'destination') {
+                const deItem = cache.getByKey(
+                    'dataExtension',
+                    metadata[`r__${target}_dataExtension_key`]
+                );
+                metadata.resultDEDescription = deItem?.Description || '';
+                metadata.resultDEName = deItem?.Name || '';
+                metadata.resultDEKey = deItem?.CustomerKey || '';
+
+                metadata.resultGroupFolderId = null;
+                metadata.resultGroupName = null;
+            }
+            delete metadata[`r__${target}_dataExtension_key`];
+        }
+
+        // TODO GroupWizard / TypeId==3
+
+        // TODO BehavioralData / TypeId==4
+    }
+
+    /**
+     * Creates a single item
+     * this uses soap API because the rest api does not allow hotlinking to an existing target DE
+     *
+     * @param {FilterItem} item a single item
+     * @returns {Promise} Promise
+     */
+    static async create(item) {
+        // create: POST automation/v1/filters/
+        // {"filterDefinitionId":"4b9adc79-8f56-488c-9910-ab7ad7afc13a","name":"jb_talent_flat_2_gui","description":"","categoryId":"5319","sourceId":null,"resultGroupFolderId":null,"resultGroupName":null,"filterDefinitionSourceTypeId":2,"folderLocationText":"Filter","customerKey":"jb_talent_flat_2_gui","resultDEName":"jb_talent_flat_2_gui","resultDEKey":"jb_talent_flat_2_gui","resultDEDescription":""}
+        // return super.createREST(item, '/automation/v1/filters/');
+
+        let response = null;
+        try {
+            response = await super.createSOAP(this.preCreateSOAPItem(item), true);
+
+            Util.logger.info(` - created ${Util.getTypeKeyName(this.definition, item)}`);
+        } catch (ex) {
+            this._handleSOAPErrors(ex, 'creating', item, false);
+        }
+        return this.postCreateTasks(item, response);
+    }
+
+    /**
+     * helper that converts the rest item into a soap item
+     *
+     * @param {FilterItem} item a single item
+     * @returns {object} SOAP formatted filter item
+     */
+    static preCreateSOAPItem(item) {
+        return {
+            FilterDefinitionID: item.filterDefinitionId,
+            Name: item.name,
+            Description: item.description || '',
+            // CategoryID: item.categoryId,
+            CustomerKey: item.customerKey,
+            DestinationTypeID: item.destinationTypeId,
+            DestinationObjectID: item.destinationObjectId,
+            SourceTypeID: item.sourceTypeId,
+            SourceObjectID: item.sourceObjectId,
+        };
+    }
+
+    /**
+     * helper that runs update on all create calls to ensure all fields are set
+     *
+     * @param {FilterItem} restItem original rest item
+     * @param {object} response SOAP response
+     * @returns {Promise.<FilterItem>} created item
+     */
+    static async postCreateTasks(restItem, response) {
+        Util.logger.debug(
+            ` - running post create tasks for ${Util.getTypeKeyName(this.definition, restItem)} to set the folder ID`
+        );
+        if (!response) {
+            return response;
+        }
+        const soapItem = response.Results?.[0].Object;
+        restItem.filterActivityId = soapItem.FilterActivityID;
+        return super.updateREST(
+            restItem,
+            '/automation/v1/filters/' + restItem[this.definition.idField],
+            undefined,
+            true
+        );
+    }
+
+    /**
+     * Updates a single item
+     *
+     * @param {MetadataTypeItem} item a single item
+     * @returns {Promise} Promise
+     */
+    static update(item) {
+        return super.updateREST(item, '/automation/v1/filters/' + item[this.definition.idField]);
+    }
+
+    /**
+     * prepares a record for deployment
+     *
+     * @param {FilterItem} metadata a single record
+     * @returns {Promise.<FilterItem>} Promise of updated single record
+     */
+    static async preDeployTasks(metadata) {
+        // folder
+        super.setFolderId(metadata);
+
+        // disable updates to r__dataFilter_key, r__source_dataExtension_key, r__destination_dataExtension_key
+        // technically, the API allows changing the reference to the dataFilter but the GUI does not.
+        // the API and GUI prevent changes to the source & destination data extensions.
+        // to avoid confusion and accidental changes, we will reset the values from cache before deployment
+        const normalizedKey = File.reverseFilterIllegalFilenames(
+            metadata[this.definition.keyField]
+        );
+        const cachedVersion = cache.getByKey(this.definition.type, normalizedKey);
+
+        // filterDefinition
+        if (metadata.r__dataFilter_key) {
+            metadata.filterDefinitionId = cache.searchForField(
+                'dataFilter',
+                metadata.r__dataFilter_key,
+                'key',
+                'id'
+            );
+            if (cachedVersion && cachedVersion.filterDefinitionId !== metadata.filterDefinitionId) {
+                throw new Error(
+                    `Updating r__dataFilter_key is not allowed. You need to delete and re-create the filter to change this.`
+                );
+            }
+            delete metadata.r__dataFilter_key;
+        }
+        if (!metadata.description) {
+            metadata.description = '';
+        }
+
+        metadata.resultGroupFolderId = null;
+        metadata.sourceId = null;
+
+        this._preDeploy_dataTypeMapping('source', metadata, cachedVersion);
+        this._preDeploy_dataTypeMapping('destination', metadata, cachedVersion);
+
+        return metadata;
+    }
+
+    /**
+     * helper to allow us to select single metadata entries via REST
+     *
+     * @private
+     * @param {string} key customer key
+     * @returns {Promise.<string>} objectId or empty string
+     */
+    static async _getObjectIdForSingleRetrieve(key) {
+        const name = key.startsWith('name:') ? key.slice(5) : null;
+        const response = await this.client.soap.retrieve(this.definition.soapType, ['ObjectID'], {
+            filter: {
+                leftOperand: name ? 'Name' : 'CustomerKey',
+                operator: 'equals',
+                rightOperand: name || key,
+            },
+        });
+        return response?.Results?.length ? response.Results[0].ObjectID : '';
+    }
+
+    /**
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {string} key Identifier of data extension
+     * @returns {Promise.<boolean>} deletion success flag
+     */
+    static async deleteByKey(key) {
+        // delete only works with the query's object id
+        const objectId = key ? await this._getObjectIdForSingleRetrieve(key) : null;
+        if (!objectId) {
+            await this.deleteNotFound(key);
+            return false;
+        }
+        return super.deleteByKeyREST('/automation/v1/filters/' + objectId, key);
     }
 }
 

package/lib/metadataTypes/ImportFile.js

@@ -57,12 +57,22 @@ class ImportFile extends MetadataType {
         );
         this.smsImports = {};
         if (smsImportResults.totalResults > 0 && smsImportResults.entry.length > 0) {
-            Util.logger.info(`Caching dependent Metadata: dataExtension (source for SMS imports)`);
+            if (smsImportResults.entry.filter((item) => !!item.sourceObjectId).length > 0) {
+                // imports from data extensions require additional information (item.sourceFile != '_CustomObject' and item.sourceObjectId is not set)
+                Util.logger.info(
+                    Util.getGrayMsg(
+                        ` - Caching dependent Metadata: dataExtension (source for SMS imports)`
+                    )
+                );
+            }
             const sourceObject = {};
             for (const item of smsImportResults.entry) {
                 // this api does not show the key but the name is assumed to be unique
                 this.smsImports[item.name] = item;
 
+                if (!item.sourceObjectId) {
+                    continue;
+                }
                 try {
                     if (!sourceObject[item.sourceObjectId]) {
                         sourceObject[item.sourceObjectId] = await this.client.rest.get(

package/lib/metadataTypes/Journey.js

@@ -2,6 +2,7 @@
 
 import MetadataType from './MetadataType.js';
 import TransactionalEmail from './TransactionalEmail.js';
+import DataExtensionField from './DataExtensionField.js';
 import TriggeredSend from './TriggeredSend.js';
 import Event from './Event.js';
 import { Util } from '../util/util.js';
@@ -512,7 +513,7 @@ class Journey extends MetadataType {
                 }
                 // ~~~ ACTIVITIES ~~~~
 
-                this._postRetrieveTasks_activities(metadata);
+                await this._postRetrieveTasks_activities(metadata);
 
                 // TODO: journey template id? / metaData.templateId
                 break;
@@ -581,7 +582,7 @@ class Journey extends MetadataType {
                                 delete activity.metaData.highThroughput.definitionKey;
                             }
 
-                            this._postRetrieveTasks_activities(metadata);
+                            await this._postRetrieveTasks_activities(metadata);
 
                             if (activity.metaData?.highThroughput?.dataExtensionId) {
                                 try {
@@ -639,7 +640,7 @@ class Journey extends MetadataType {
      * @private
      * @param {MetadataTypeItem} metadata a single item
      */
-    static _postRetrieveTasks_activities(metadata) {
+    static async _postRetrieveTasks_activities(metadata) {
         if (!metadata.activities) {
             return;
         }
@@ -1239,6 +1240,58 @@ class Journey extends MetadataType {
                     }
                     break;
                 }
+                case 'UPDATECONTACTDATA': {
+                    const contactFields =
+                        activity?.arguments?.activityData?.updateContactFields ?? [];
+                    let tempCachedFields = false;
+                    for (const contactField of contactFields) {
+                        try {
+                            contactField.r__dataExtension_key = cache.searchForField(
+                                'dataExtension',
+                                contactField.dataExtensionId,
+                                'ObjectID',
+                                'CustomerKey'
+                            );
+                            delete contactField.dataExtensionId;
+                        } catch (ex) {
+                            Util.logger.warn(
+                                ` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
+                                    metadata[this.definition.keyField]
+                                }) activity-key=${activity.key}: ${ex.message}`
+                            );
+                        }
+                        // Don't replace the field Id with the name if DE CustomerKey was not found
+                        try {
+                            if (!cache.getCache().dataExtensionField) {
+                                tempCachedFields = true;
+                                DataExtensionField.buObject = this.buObject;
+                                DataExtensionField.client = this.client;
+                                DataExtensionField.properties = this.properties;
+                                const fields = await DataExtensionField.retrieveFieldsForSingleDe(
+                                    contactField.r__dataExtension_key
+                                );
+                                cache.setMetadata('dataExtensionField', fields);
+                            }
+                            contactField.r__dataExtensionField_name = cache.searchForField(
+                                'dataExtensionField',
+                                contactField.field,
+                                'ObjectID',
+                                'Name'
+                            );
+                            delete contactField.field;
+                        } catch (ex) {
+                            Util.logger.warn(
+                                ` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
+                                    metadata[this.definition.keyField]
+                                }) activity-key=${activity.key}: ${ex.message}`
+                            );
+                        }
+                    }
+                    if (tempCachedFields) {
+                        // reset dataExtensionField caching to trigger re-caching
+                        cache.clearCache(this.buObject.mid, 'dataExtensionField');
+                    }
+                }
             }
         }
 
@@ -1310,7 +1363,7 @@ class Journey extends MetadataType {
 
                 // ~~~ ACTIVITIES ~~~~
 
-                this._preDeployTasks_activities(metadata);
+                await this._preDeployTasks_activities(metadata);
 
                 break;
             }
@@ -1338,7 +1391,7 @@ class Journey extends MetadataType {
                         cachedVersion.status = 'Paused';
                     } else {
                         throw new Error(
-                            `Cannot update transactional-send journey in Published status. Run depoy with --publish to auto-pause & republish the journey.`
+                            `Cannot update transactional-send journey in Published status. Run deploy with --publish to auto-pause & republish the journey.`
                         );
                     }
                 }
@@ -1404,7 +1457,7 @@ class Journey extends MetadataType {
                             delete activity.metaData.highThroughput.r__dataExtension_key;
                         }
 
-                        this._preDeployTasks_activities(metadata);
+                        await this._preDeployTasks_activities(metadata);
 
                         break;
                     }
@@ -1433,7 +1486,7 @@ class Journey extends MetadataType {
      * @private
      * @param {MetadataTypeItem} metadata a single item
      */
-    static _preDeployTasks_activities(metadata) {
+    static async _preDeployTasks_activities(metadata) {
         for (const activity of metadata.activities) {
             switch (activity.type) {
                 case 'EMAILV2': {
@@ -1666,6 +1719,57 @@ class Journey extends MetadataType {
 
                     break;
                 }
+                case 'UPDATECONTACTDATA': {
+                    const contactFields =
+                        activity?.arguments?.activityData?.updateContactFields ?? [];
+                    let tempCachedFields = false;
+                    for (const contactField of contactFields) {
+                        try {
+                            contactField.dataExtensionId = cache.searchForField(
+                                'dataExtension',
+                                contactField.r__dataExtension_key,
+                                'CustomerKey',
+                                'ObjectID'
+                            );
+                        } catch (ex) {
+                            Util.logger.warn(
+                                ` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
+                                    metadata[this.definition.keyField]
+                                }) activity-key=${activity.key}: ${ex.message}`
+                            );
+                        }
+                        try {
+                            if (!cache.getCache().dataExtensionField) {
+                                tempCachedFields = true;
+                                DataExtensionField.buObject = this.buObject;
+                                DataExtensionField.client = this.client;
+                                DataExtensionField.properties = this.properties;
+                                const fields = await DataExtensionField.retrieveFieldsForSingleDe(
+                                    contactField.r__dataExtension_key
+                                );
+                                cache.setMetadata('dataExtensionField', fields);
+                            }
+                            contactField.field = cache.searchForField(
+                                'dataExtensionField',
+                                `[${contactField.r__dataExtension_key}].[${contactField.r__dataExtensionField_name}]`,
+                                'CustomerKey',
+                                'ObjectID'
+                            );
+                            delete contactField.r__dataExtensionField_name;
+                            delete contactField.r__dataExtension_key;
+                        } catch (ex) {
+                            Util.logger.warn(
+                                ` - ${this.definition.type} '${metadata[this.definition.nameField]}'(${
+                                    metadata[this.definition.keyField]
+                                }) activity - key=${activity.key}: ${ex.message} `
+                            );
+                        }
+                    }
+                    if (tempCachedFields) {
+                        // reset dataExtensionField caching to trigger re-caching
+                        cache.clearCache(this.buObject.mid, 'dataExtensionField');
+                    }
+                }
             }
         }
     }

package/lib/metadataTypes/definitions/Automation.definition.js

@@ -28,6 +28,7 @@ export default {
         'emailSend',
         'fileTransfer',
         'fileLocation',
+        'filter',
         'folder-automations',
         'importFile',
         'query',