mcdev 8.3.0 → 8.4.0

package/@types/lib/metadataTypes/definitions/Script.definition.d.ts

@@ -146,6 +146,16 @@ declare namespace _default {
             let template_12: boolean;
             export { template_12 as template };
         }
+        namespace parentCategoryId {
+            let isCreateable_13: boolean;
+            export { isCreateable_13 as isCreateable };
+            let isUpdateable_13: boolean;
+            export { isUpdateable_13 as isUpdateable };
+            let retrieving_13: boolean;
+            export { retrieving_13 as retrieving };
+            let template_13: boolean;
+            export { template_13 as template };
+        }
         namespace r__folder_Path {
             let skipValidation: boolean;
         }

package/lib/cli.js

@@ -73,6 +73,12 @@ yargs(hideBin(process.argv))
                     type: 'boolean',
                     group: 'Options for retrieve:',
                     describe: 'deletes the relevant retrieve folder before retrieving',
+                })
+                .option('onlyPublished', {
+                    type: 'boolean',
+                    alias: 'op',
+                    group: 'Options for retrieve:',
+                    describe: 'only retrieve the version of the metadata that was published',
                 }),
         (argv) => {
             Mcdev.setOptions(argv);

package/lib/index.js

@@ -97,6 +97,7 @@ class Mcdev {
         'ignoreSfFields',
         'json',
         'keySuffix',
+        'onlyPublished',
         'like',
         'matchName',
         'noLogColors',

package/lib/metadataTypes/DataFilter.js

@@ -20,6 +20,8 @@ import File from '../util/file.js';
  * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
  * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
  * @typedef {import('../../types/mcdev.d.js').DataFilterItem} DataFilterItem
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldMap} DataExtensionFieldMap
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldItem} DataExtensionFieldItem
  * @typedef {import('../../types/mcdev.d.js').DataFilterMap} DataFilterMap
  * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeMap} MultiMetadataTypeMap
  * @typedef {import('../../types/mcdev.d.js').FilterConditionSet} FilterConditionSet
@@ -108,8 +110,10 @@ class DataFilter extends MetadataType {
      */
     static async _getFilterFolderIds(recached = false) {
         const fromCache =
-            this.cache.folderFilter || cache.getCache().folder
-                ? Object.values(this.cache.folderFilter || cache.getCache().folder)
+            this.cache[this.buObject.mid]?.folderFilter || cache.getCache().folder
+                ? Object.values(
+                      this.cache[this.buObject.mid]?.folderFilter || cache.getCache().folder
+                  )
                       .filter((item) => item.ContentType === 'filterdefinition')
                       .filter(
                           (item) =>
@@ -133,7 +137,10 @@ class DataFilter extends MetadataType {
         Folder.client = this.client;
         Folder.buObject = this.buObject;
         Folder.properties = this.properties;
-        this.cache.folderFilter = (await Folder.retrieveForCache(null, subTypeArr)).metadata;
+        this.cache[this.buObject.mid] ||= {};
+        this.cache[this.buObject.mid].folderFilter = (
+            await Folder.retrieveForCache(null, subTypeArr)
+        ).metadata;
         return this._getFilterFolderIds(true);
     }
     /**
@@ -143,9 +150,9 @@ class DataFilter extends MetadataType {
      */
     static async _getMeasureFolderIds() {
         const fromCache =
-            this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder
+            this.cache[this.buObject.mid]?.folderMeasure || cache.getCache().folder
                 ? Object.values(
-                      this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder
+                      this.cache[this.buObject.mid]?.folderMeasure || cache.getCache().folder
                   )
                       .filter((item) => item.ContentType === 'measure')
                       .map((item) => item.ID)
@@ -161,8 +168,9 @@ class DataFilter extends MetadataType {
         Folder.client = this.client;
         Folder.buObject = this.buObject;
         Folder.properties = this.properties;
-        this.cache.folderMeasure ||= {};
-        this.cache.folderMeasure[this.buObject.mid] = (
+        this.cache[this.buObject.mid] ||= {};
+        this.cache[this.buObject.mid].folderMeasure ||= {};
+        this.cache[this.buObject.mid].folderMeasure = (
             await Folder.retrieveForCache(null, subTypeArr)
         ).metadata;
         return this._getMeasureFolderIds();
@@ -201,12 +209,18 @@ class DataFilter extends MetadataType {
                 : Object.values(metadataTypeMap)
                       .filter((item) => item.r__source_dataExtension_key)
                       .map((item) => item.r__source_dataExtension_key)
+                      .filter(
+                          (deKey) =>
+                              !this.cache[this.buObject.mid]?.dataExtensionField ||
+                              !this.cache[this.buObject.mid]?.dataExtensionField[deKey]
+                      )
                       .filter(Boolean);
         if (deKeys.length) {
+            const deKeysUnique = [...new Set(deKeys)];
             Util.logger.info(' - Caching dependent Metadata: dataExtensionField');
             // only proceed with the download if we have dataExtension keys
             const fieldOptions = {};
-            for (const deKey of deKeys) {
+            for (const deKey of deKeysUnique) {
                 fieldOptions.filter = fieldOptions.filter
                     ? {
                           leftOperand: {
@@ -226,9 +240,38 @@ class DataFilter extends MetadataType {
             DataExtensionField.buObject = this.buObject;
             DataExtensionField.client = this.client;
             DataExtensionField.properties = this.properties;
-            this.dataExtensionFieldCache = (
-                await DataExtensionField.retrieveForCacheDE(fieldOptions, ['Name', 'ObjectID'])
-            ).metadata;
+            this.saveDataExtensionFieldCacheToMap(
+                (await DataExtensionField.retrieveForCacheDE(fieldOptions, ['Name', 'ObjectID']))
+                    .metadata
+            );
+        }
+    }
+    /**
+     * helper for {@link DataFilter._cacheDeFields}
+     *
+     * @param {DataExtensionFieldMap} deFieldCache -
+     */
+    static saveDataExtensionFieldCacheToMap(deFieldCache) {
+        this.cache[this.buObject.mid] ||= {};
+        this.cache[this.buObject.mid].dataExtensionField ||= {};
+
+        if (!deFieldCache) {
+            return;
+        }
+        for (const field of Object.values(deFieldCache)) {
+            if (!field?.DataExtension?.CustomerKey) {
+                continue;
+            }
+            if (
+                !this.cache[this.buObject.mid].dataExtensionField[field.DataExtension.CustomerKey]
+            ) {
+                /** @type {Object.<string, DataExtensionFieldItem[]>} */
+                this.cache[this.buObject.mid].dataExtensionField[field.DataExtension.CustomerKey] =
+                    [];
+            }
+            this.cache[this.buObject.mid].dataExtensionField[field.DataExtension.CustomerKey].push(
+                field
+            );
         }
     }
     /**
@@ -237,7 +280,7 @@ class DataFilter extends MetadataType {
      * @param {DataFilterMap} metadataTypeMap -
      */
     static async _cacheContactAttributes(metadataTypeMap) {
-        if (this.cache.contactAttributes?.[this.buObject.mid]) {
+        if (this.cache[this.buObject.mid]?.contactAttributes) {
             return;
         }
         const subscriberFilters = Object.values(metadataTypeMap)
@@ -248,8 +291,8 @@ class DataFilter extends MetadataType {
             const response = await this.client.rest.get('/email/v1/Contacts/Attributes/');
             const keyFieldBackup = this.definition.keyField;
             this.definition.keyField = 'id';
-            this.cache.contactAttributes ||= {};
-            this.cache.contactAttributes[this.buObject.mid] = this.parseResponseBody(response);
+            this.cache[this.buObject.mid] ||= {};
+            this.cache[this.buObject.mid].contactAttributes = this.parseResponseBody(response);
             this.definition.keyField = keyFieldBackup;
         }
     }
@@ -259,7 +302,7 @@ class DataFilter extends MetadataType {
      * @param {DataFilterMap} metadataTypeMap -
      */
     static async _cacheMeasures(metadataTypeMap) {
-        if (this.cache.measures?.[this.buObject.mid]) {
+        if (this.cache?.[this.buObject.mid]?.measures) {
             return;
         }
         const subscriberFilters = Object.values(metadataTypeMap)
@@ -281,8 +324,8 @@ class DataFilter extends MetadataType {
 
             const keyFieldBackup = this.definition.keyField;
             this.definition.keyField = 'measureID';
-            this.cache.measures ||= {};
-            this.cache.measures[this.buObject.mid] = this.parseResponseBody(response);
+            this.cache[this.buObject.mid] ||= {};
+            this.cache[this.buObject.mid].measures = this.parseResponseBody(response);
             this.definition.keyField = keyFieldBackup;
         }
     }
@@ -399,7 +442,7 @@ class DataFilter extends MetadataType {
      *
      * @param {DataFilterItem} metadata -
      * @param {'postRetrieve'|'preDeploy'} mode -
-     * @param {object[]} [fieldCache] -
+     * @param {DataExtensionFieldItem[]} [fieldCache] -
      * @param {FilterConditionSet} [filter] -
      * @returns {void}
      */
@@ -408,7 +451,9 @@ class DataFilter extends MetadataType {
             return this._resolveFields(
                 metadata,
                 mode,
-                Object.values(this.dataExtensionFieldCache),
+                this.cache[this.buObject.mid].dataExtensionField[
+                    metadata.r__source_dataExtension_key
+                ],
                 metadata.c__filterDefinition?.ConditionSet
             );
         }
@@ -418,11 +463,11 @@ class DataFilter extends MetadataType {
                 : [filter.Condition];
             if (mode === 'postRetrieve') {
                 for (const condition of conditionsArr) {
-                    this._postRetrieve_resolveFieldIdsCondition(condition, fieldCache);
+                    this._postRetrieve_resolveFieldIdsCondition(metadata, condition, fieldCache);
                 }
             } else if (mode === 'preDeploy') {
                 for (const condition of conditionsArr) {
-                    this._preDeploy_resolveFieldNamesCondition(condition, fieldCache);
+                    this._preDeploy_resolveFieldNamesCondition(metadata, condition, fieldCache);
                 }
             }
         }
@@ -438,16 +483,21 @@ class DataFilter extends MetadataType {
     /**
      * helper for {@link _resolveFields}
      *
+     * @param {DataFilterItem} metadata -
      * @param {FilterCondition} condition -
-     * @param {object[]} fieldCache -
+     * @param {DataExtensionFieldItem[]} fieldCache -
      * @returns {void}
      */
-    static _postRetrieve_resolveFieldIdsCondition(condition, fieldCache) {
+    static _postRetrieve_resolveFieldIdsCondition(metadata, condition, fieldCache) {
         condition.r__dataExtensionField_name = fieldCache.find(
             (field) => field.ObjectID === condition['@_ID']
         )?.Name;
         if (condition.r__dataExtensionField_name) {
             delete condition['@_ID'];
+        } else {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: could not resolve dataExtensionField with ID ${condition['@_ID']} in condition`
+            );
         }
         if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
             delete condition.Value;
@@ -456,16 +506,21 @@ class DataFilter extends MetadataType {
     /**
      * helper for {@link _resolveFields}
      *
+     * @param {DataFilterItem} metadata -
      * @param {FilterCondition} condition -
-     * @param {object[]} fieldCache -
+     * @param {DataExtensionFieldItem[]} fieldCache -
      * @returns {void}
      */
-    static _preDeploy_resolveFieldNamesCondition(condition, fieldCache) {
+    static _preDeploy_resolveFieldNamesCondition(metadata, condition, fieldCache) {
         condition['@_ID'] = fieldCache.find(
             (field) => field.Name === condition.r__dataExtensionField_name
         )?.ObjectID;
         if (condition['@_ID']) {
             delete condition.r__dataExtensionField_name;
+        } else {
+            throw new Error(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: could not resolve dataExtensionField with Name ${condition.r__dataExtensionField_name} in condition`
+            );
         }
         if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
             condition.Value ||= '';
@@ -489,8 +544,8 @@ class DataFilter extends MetadataType {
                 metadata.c__filterDefinition?.ConditionSet
             );
         }
-        const contactAttributes = this.cache.contactAttributes[this.buObject.mid];
-        const measures = this.cache.measures[this.buObject.mid];
+        const contactAttributes = this.cache[this.buObject.mid]?.contactAttributes;
+        const measures = this.cache[this.buObject.mid]?.measures;
         const conditionsArr = Array.isArray(filter.Condition)
             ? filter.Condition
             : [filter.Condition];

package/lib/metadataTypes/Event.js

@@ -1067,7 +1067,7 @@ class Event extends MetadataType {
             if (errors.length > 1) {
                 errors.unshift(``);
             }
-            throw new Error(errors.join('\n        · ')); // eslint-disable-line unicorn/error-message
+            throw new Error(errors.join('\n        · '));
         }
         if (warnings?.length) {
             // add a line break
@@ -1220,7 +1220,7 @@ class Event extends MetadataType {
             if (errors.length > 1) {
                 errors.unshift(``);
             }
-            throw new Error(errors.join('\n        · ')); // eslint-disable-line unicorn/error-message
+            throw new Error(errors.join('\n        · '));
         }
     }
 

package/lib/metadataTypes/FileLocation.js

@@ -22,6 +22,7 @@ import { Util } from '../util/util.js';
  * @augments MetadataType
  */
 class FileLocation extends MetadataType {
+    static cache = {};
     /**
      * Retrieves Metadata of FileLocation
      * Endpoint /automation/v1/ftplocations/ return all FileLocations
@@ -32,8 +33,32 @@ class FileLocation extends MetadataType {
      * @param {string} [key] customer key of single item to retrieve
      * @returns {Promise.<MetadataTypeMapObj>} Promise
      */
-    static retrieve(retrieveDir, _, __, key) {
-        return super.retrieveREST(retrieveDir, '/automation/v1/ftplocations/', null, key);
+    static async retrieve(retrieveDir, _, __, key) {
+        try {
+            const dataItems = await super.retrieveREST(
+                null,
+                '/data/v1/filetransferlocation' + (key ? '/' + encodeURIComponent(key) : 's'),
+                null,
+                key
+            );
+            this.cache[this.buObject.mid] ||= {};
+            this.cache[this.buObject.mid].dataItems = dataItems.metadata;
+        } catch (ex) {
+            if (ex.code === 'ERR_BAD_REQUEST') {
+                // if retrieve-by-key comes up empty, the data-endpoint returns a code 400
+                Util.logger.debug(ex.message);
+            } else {
+                Util.logger.warn(ex.message);
+            }
+        }
+        const items = await super.retrieveREST(
+            retrieveDir,
+            '/automation/v1/ftplocations/',
+            null,
+            key
+        );
+
+        return items;
     }
 
     /**
@@ -42,9 +67,156 @@ class FileLocation extends MetadataType {
      * @returns {Promise.<MetadataTypeMapObj>} Promise
      */
     static async retrieveForCache() {
-        return super.retrieveREST(null, '/automation/v1/ftplocations/');
+        return this.retrieve(null);
+    }
+
+    /**
+     * Creates a single item
+     *
+     * @param {MetadataTypeItem} metadata a single item
+     * @returns {Promise.<MetadataTypeItem>} Promise
+     */
+    static create(metadata) {
+        return this.createREST(metadata, '/data/v1/filetransferlocation');
+    }
+    /**
+     * Updates a single item
+     *
+     * @param {MetadataTypeItem} metadata a single item
+     * @returns {Promise.<MetadataTypeItem>} Promise
+     */
+    static update(metadata) {
+        return this.updateREST(
+            metadata,
+            '/data/v1/filetransferlocation/' +
+                encodeURIComponent(metadata[this.definition.keyField])
+        );
     }
 
+    /**
+     * helper for {@link MetadataType.parseResponseBody} that creates a custom key field for this type based on mobileCode and keyword
+     *
+     * @param {MetadataTypeItem} metadata single item
+     */
+    static createCustomKeyField(metadata) {
+        if (metadata.fileTransferLocation) {
+            const fileTransferLocation = metadata.fileTransferLocation;
+            for (const key of Object.keys(metadata)) {
+                delete metadata[key];
+            }
+            Object.assign(metadata, fileTransferLocation);
+        } else {
+            if (!metadata.customerKey && this.cache[this.buObject.mid]?.dataItems) {
+                const nameMatch = Object.values(this.cache[this.buObject.mid].dataItems).find(
+                    (item) => item.name === metadata.name
+                );
+                if (nameMatch) {
+                    Object.assign(metadata, nameMatch);
+                }
+            }
+            if (!this.definition.locationTypeIdMappingDeployable[metadata.locationTypeId]) {
+                // old file location types are only returned by the automation-endpoint which does not return customerKey field - but also these are not updatable and hence we can improvise here
+
+                metadata.customerKey ||= metadata.name;
+            }
+        }
+    }
+
+    /**
+     * Creates a single metadata entry via REST
+     *
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {string} uri rest endpoint for POST
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
+     * @returns {Promise.<object> | null} Promise of API response or null in case of an error
+     */
+    static async createREST(metadataEntry, uri, handleOutside) {
+        this.removeNotCreateableFields(metadataEntry);
+        const createPayload = { fileTransferLocation: metadataEntry };
+        try {
+            // set to empty object in case API returned nothing to be able to update it in helper classes
+            let response = (await this.client.rest.post(uri, createPayload)) || {};
+            response = await this.postCreateTasks(metadataEntry, response);
+            if (!handleOutside) {
+                Util.logger.info(
+                    ` - created ${Util.getTypeKeyName(this.definition, metadataEntry)}`
+                );
+            }
+            return response;
+        } catch (ex) {
+            const parsedErrors = this.getErrorsREST(ex);
+            Util.logger.error(
+                ` ☇ error creating ${Util.getTypeKeyName(this.definition, metadataEntry)}:`
+            );
+            if (parsedErrors.length) {
+                for (const msg of parsedErrors) {
+                    Util.logger.error('   • ' + msg);
+                }
+            } else if (ex?.message) {
+                Util.logger.debug(ex.message);
+            }
+            return null;
+        }
+    }
+
+    /**
+     * helper for {@link MetadataType.createREST}
+     *
+     * @param {MetadataTypeItem} _ a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @returns {Promise.<object>} apiResponse, potentially modified
+     */
+    static async postCreateTasks(_, apiResponse) {
+        return apiResponse?.fileTransferLocation || apiResponse;
+    }
+
+    /**
+     * Updates a single metadata entry via REST
+     *
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {string} uri rest endpoint for PATCH
+     * @param {'patch'|'post'|'put'} [httpMethod] defaults to 'patch'; some update requests require PUT instead of PATCH
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
+     * @returns {Promise.<object> | null} Promise of API response or null in case of an error
+     */
+    static async updateREST(metadataEntry, uri, httpMethod = 'patch', handleOutside) {
+        this.removeNotUpdateableFields(metadataEntry);
+        const updatePayload = { fileTransferLocation: metadataEntry };
+        try {
+            // set to empty object in case API returned nothing to be able to update it in helper classes
+            let response = (await this.client.rest[httpMethod](uri, updatePayload)) || {};
+            await this._postChangeKeyTasks(metadataEntry);
+            this.getErrorsREST(response);
+            response = await this.postUpdateTasks(metadataEntry, response);
+            // some times, e.g. automation dont return a key in their update response and hence we need to fall back to name
+            if (!handleOutside) {
+                Util.logger.info(
+                    ` - updated ${Util.getTypeKeyName(this.definition, metadataEntry)}`
+                );
+            }
+            return response;
+        } catch (ex) {
+            const parsedErrors = this.getErrorsREST(ex);
+            Util.logger.error(
+                ` ☇ error updating ${Util.getTypeKeyName(this.definition, metadataEntry)}:`
+            );
+            for (const msg of parsedErrors) {
+                Util.logger.error('   • ' + msg);
+            }
+            return null;
+        }
+    }
+
+    /**
+     * helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP}
+     *
+     * @param {MetadataTypeItem} _ a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @returns {Promise.<object>} apiResponse, potentially modified
+     */
+    static postUpdateTasks(_, apiResponse) {
+        return apiResponse?.fileTransferLocation || apiResponse;
+    }
     /**
      * prepares a import definition for deployment
      *
@@ -54,6 +226,14 @@ class FileLocation extends MetadataType {
     static async preDeployTasks(metadata) {
         if (metadata.c__locationType) {
             metadata.locationTypeId = this.definition.locationTypeMapping[metadata.c__locationType];
+            if (this.definition.locationTypeMappingDeployable[metadata.c__locationType]) {
+                metadata.locationType =
+                    this.definition.locationTypeMappingDeployable[metadata.c__locationType];
+            } else {
+                throw new Error(
+                    `Only FileLocations of types ${Object.keys(this.definition.locationTypeMappingDeployable).join(', ')} can be deployed via mcdev.`
+                );
+            }
         }
         return metadata;
     }
@@ -65,7 +245,7 @@ class FileLocation extends MetadataType {
      * @returns {MetadataTypeItem} parsed metadata
      */
     static postRetrieveTasks(metadata) {
-        if (metadata.locationTypeId) {
+        if (metadata.locationTypeId !== undefined) {
             try {
                 metadata.c__locationType = Util.inverseGet(
                     this.definition.locationTypeMapping,
@@ -78,9 +258,29 @@ class FileLocation extends MetadataType {
                         metadata.locationTypeId
                 );
             }
+        } else if (metadata.locationType) {
+            // assuming create/update of new types
+            metadata.c__locationType = Util.inverseGet(
+                this.definition.locationTypeMappingDeployable,
+                metadata.locationType
+            );
         }
         return metadata;
     }
+
+    /**
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {string} key Identifier of item
+     * @returns {Promise.<boolean>} deletion success flag
+     */
+    static async deleteByKey(key) {
+        return super.deleteByKeyREST(
+            '/data/v1/filetransferlocation/' + encodeURIComponent(key),
+            key,
+            400
+        );
+    }
 }
 
 // Assign definition to static attributes