mcdev 4.3.4 → 5.0.0

package/lib/metadataTypes/MetadataType.js

@@ -87,14 +87,16 @@ class MetadataType {
      * Returns fieldnames of Metadata Type. 'this.definition.fields' variable only set in child classes.
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @param {boolean} [isCaching] if true, then check if field should be skipped for caching
      * @returns {string[]} Fieldnames
      */
-    static getFieldNamesToRetrieve(additionalFields) {
+    static getFieldNamesToRetrieve(additionalFields, isCaching) {
         const fieldNames = [];
         for (const fieldName in this.definition.fields) {
             if (
                 additionalFields?.includes(fieldName) ||
-                this.definition.fields[fieldName].retrieving
+                (this.definition.fields[fieldName].retrieving &&
+                    !(isCaching && this.definition.fields[fieldName].skipCache))
             ) {
                 fieldNames.push(fieldName);
             }
@@ -112,11 +114,11 @@ class MetadataType {
      * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
      * @param {string} retrieveDir directory where metadata after deploy should be saved
+     * @param {boolean} [isRefresh] optional flag to indicate that triggeredSend should be refreshed after deployment of assets
      * @returns {Promise.<TYPE.MetadataTypeMap>} Promise of keyField => metadata map
      */
-    static async deploy(metadata, deployDir, retrieveDir) {
-        const upsertResults = await this.upsert(metadata, deployDir);
-        await this.postDeployTasks(upsertResults, metadata);
+    static async deploy(metadata, deployDir, retrieveDir, isRefresh) {
+        const upsertResults = await this.upsert(metadata, deployDir, isRefresh);
         const savedMetadata = await this.saveResults(upsertResults, retrieveDir, null);
         if (
             this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type) &&
@@ -132,11 +134,60 @@ class MetadataType {
     /**
      * Gets executed after deployment of metadata type
      *
-     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {TYPE.MetadataTypeMap} upsertResults metadata mapped by their keyField as returned by update/create
      * @param {TYPE.MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
+     * @param {boolean} [isRefresh] optional flag to indicate that triggeredSend should be refreshed after deployment of assets
+     * @returns {void}
+     */
+    static postDeployTasks(upsertResults, originalMetadata, createdUpdated, isRefresh) {}
+
+    /**
+     * helper for {@link createREST}
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
      * @returns {void}
      */
-    static postDeployTasks(metadata, originalMetadata) {}
+    static postCreateTasks(metadataEntry, apiResponse) {}
+
+    /**
+     * helper for {@link updateREST}
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @returns {void}
+     */
+    static postUpdateTasks(metadataEntry, apiResponse) {}
+
+    /**
+     * helper for {@link createREST} when legacy API endpoints as these do not return the created item but only their new id
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @returns {Promise.<void>} -
+     */
+    static async postDeployTasks_legacyApi(metadataEntry, apiResponse) {
+        if (!apiResponse?.[this.definition.idField] && !metadataEntry?.[this.definition.idField]) {
+            return;
+        }
+        const id =
+            apiResponse?.[this.definition.idField] || metadataEntry?.[this.definition.idField];
+        // re-retrieve created items because the API does not return any info for them except the new id (api key)
+        try {
+            const { metadata } = await this.retrieveForCache(null, null, 'id:' + id);
+            const item = Object.values(metadata)[0];
+            // ensure the "created item" cli log entry has the new auto-generated value
+            metadataEntry[this.definition.keyField] = item[this.definition.keyField];
+            // ensure postRetrieveTasks has the complete object in "apiResponse"
+            Object.assign(apiResponse, item);
+            // postRetrieveTasks will be run automatically on this via super.saveResult
+        } catch (ex) {
+            throw new Error(
+                `Could not get details for new ${this.definition.type} ${id} from server (${ex.message})`
+            );
+        }
+    }
 
     /**
      * Gets executed after retreive of metadata type
@@ -222,10 +273,11 @@ class MetadataType {
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {string[]} [subTypeArr] optionally limit to a single subtype
+     * @param {string} [key] customer key of single item to retrieve
      * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
      */
-    static async retrieveForCache(additionalFields, subTypeArr) {
-        return this.retrieve(null, additionalFields, subTypeArr);
+    static async retrieveForCache(additionalFields, subTypeArr, key) {
+        return this.retrieve(null, additionalFields, subTypeArr, key);
     }
     /**
      * Gets metadata cache with limited fields and does not store value to disk
@@ -246,6 +298,19 @@ class MetadataType {
         );
         return { metadata: null, type: this.definition.type };
     }
+    /**
+     * Retrieve a specific Script by Name
+     *
+     * @param {string} templateDir Directory where retrieved metadata directory will be saved
+     * @param {string} uri rest endpoint for GET
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {string} name name (not key) of the metadata item
+     * @returns {Promise.<{metadata: TYPE.MetadataTypeItem, type: string}>} Promise
+     */
+    static async retrieveTemplateREST(templateDir, uri, templateVariables, name) {
+        return this.retrieveREST(templateDir, uri, templateVariables, name);
+    }
+
     /**
      * Gets metadata cache with limited fields and does not store value to disk
      *
@@ -293,6 +358,26 @@ class MetadataType {
             }
             // return;
         }
+        if (this.definition.stringifyFieldsBeforeTemplate) {
+            // numeric fields are returned as numbers by the SDK/API. If we try to replace them in buildTemplate it would break the JSON format - but not if we stringify them first because then the {{{var}}} is in quotes
+            metadataStr = JSON.parse(metadataStr);
+            for (const field of this.definition.stringifyFieldsBeforeTemplate) {
+                if (metadataStr[field]) {
+                    if (Array.isArray(metadataStr[field])) {
+                        for (let i = 0; i < metadataStr[field].length; i++) {
+                            metadataStr[field][i] = metadataStr[field][i].toString();
+                        }
+                    } else if ('object' === typeof metadataStr[field]) {
+                        for (const subField in metadataStr[field]) {
+                            metadataStr[field][subField] = metadataStr[field][subField].toString();
+                        }
+                    } else {
+                        metadataStr[field] = metadataStr[field].toString();
+                    }
+                }
+            }
+            metadataStr = JSON.stringify(metadataStr);
+        }
         const metadata = JSON.parse(Util.replaceByObject(metadataStr, templateVariables));
         this.keepTemplateFields(metadata);
 
@@ -444,28 +529,30 @@ class MetadataType {
     /**
      * MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
      *
-     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {TYPE.MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
+     * @param {boolean} [isRefresh] optional flag to indicate that triggeredSend should be refreshed after deployment of assets
      * @returns {Promise.<TYPE.MetadataTypeMap>} keyField => metadata map
      */
-    static async upsert(metadata, deployDir) {
+    static async upsert(metadataMap, deployDir, isRefresh) {
+        const orignalMetadataMap = JSON.parse(JSON.stringify(metadataMap));
         const metadataToUpdate = [];
         const metadataToCreate = [];
         let filteredByPreDeploy = 0;
-        for (const metadataKey in metadata) {
+        for (const metadataKey in metadataMap) {
             let hasError = false;
             try {
                 // preDeployTasks parsing
                 let deployableMetadata;
                 try {
                     deployableMetadata = await this.preDeployTasks(
-                        metadata[metadataKey],
+                        metadataMap[metadataKey],
                         deployDir
                     );
                 } catch (ex) {
                     // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
                     hasError = true;
-                    deployableMetadata = metadata[metadataKey];
+                    deployableMetadata = metadataMap[metadataKey];
                     Util.logger.error(
                         ` ☇ skipping ${this.definition.type} ${
                             deployableMetadata[this.definition.keyField]
@@ -474,10 +561,10 @@ class MetadataType {
                 }
                 // if preDeploy returns nothing then it cannot be deployed so skip deployment
                 if (deployableMetadata) {
-                    metadata[metadataKey] = deployableMetadata;
+                    metadataMap[metadataKey] = deployableMetadata;
                     // create normalizedKey off of whats in the json rather than from "metadataKey" because preDeployTasks might have altered something (type asset)
-                    this.createOrUpdate(
-                        metadata,
+                    await this.createOrUpdate(
+                        metadataMap,
                         metadataKey,
                         hasError,
                         metadataToUpdate,
@@ -518,7 +605,7 @@ class MetadataType {
             `${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
                 (filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
         );
-
+        let upsertResults;
         if (this.definition.bodyIteratorField === 'Results') {
             // if Results then parse as SOAP
             // put in Retrieve Format for parsing
@@ -530,73 +617,172 @@ class MetadataType {
                 .filter((r) => r !== undefined && r !== null && Object.keys(r).length !== 0)
                 .flatMap((r) => r.Results)
                 .map((r) => r.Object);
-            return this.parseResponseBody({ Results: metadataResults });
+            upsertResults = this.parseResponseBody({ Results: metadataResults });
         } else {
             // likely comming from one of the many REST APIs
             // put in Retrieve Format for parsing
             // todo add handling when response does not contain items.
             // @ts-ignore
             const metadataResults = createResults.concat(updateResults).filter(Boolean);
-            return this.parseResponseBody(metadataResults);
+            upsertResults = this.parseResponseBody(metadataResults);
         }
+        await this.postDeployTasks(
+            upsertResults,
+            orignalMetadataMap,
+            { created: createResults.length, updated: updateResults.length },
+            isRefresh
+        );
+        return upsertResults;
     }
 
     /**
+     * helper for {@link MetadataType.upsert}
      *
-     * @param {TYPE.MetadataTypeItem} metadata single metadata itme
+     * @param {TYPE.MetadataTypeMap} metadataMap list of metadata
      * @param {string} metadataKey key of item we are looking at
      * @param {boolean} hasError error flag from previous code
      * @param {TYPE.MetadataTypeItemDiff[]} metadataToUpdate list of items to update
      * @param {TYPE.MetadataTypeItem[]} metadataToCreate list of items to create
-     * @returns {void}
+     * @returns {'create' | 'update' | 'skip'} action to take
      */
-    static createOrUpdate(metadata, metadataKey, hasError, metadataToUpdate, metadataToCreate) {
+    static createOrUpdate(metadataMap, metadataKey, hasError, metadataToUpdate, metadataToCreate) {
         const normalizedKey = File.reverseFilterIllegalFilenames(
-            metadata[metadataKey][this.definition.keyField]
+            metadataMap[metadataKey][this.definition.keyField]
         );
         // Update if it already exists; Create it if not
-        if (Util.logger.level === 'debug' && metadata[metadataKey][this.definition.idField]) {
+        if (
+            Util.logger.level === 'debug' &&
+            metadataMap[metadataKey][this.definition.idField] &&
+            this.definition.idField !== this.definition.keyField
+        ) {
             // TODO: re-evaluate in future releases if & when we managed to solve folder dependencies once and for all
             // only used if resource is excluded from cache and we still want to update it
             // needed e.g. to rewire lost folders
             Util.logger.warn(
                 ' - Hotfix for non-cachable resource found in deploy folder. Trying update:'
             );
-            Util.logger.warn(JSON.stringify(metadata[metadataKey]));
+            Util.logger.warn(JSON.stringify(metadataMap[metadataKey]));
             if (hasError) {
                 metadataToUpdate.push(null);
+                return 'skip';
             } else {
                 metadataToUpdate.push({
                     before: {},
-                    after: metadata[metadataKey],
+                    after: metadataMap[metadataKey],
                 });
+                return 'update';
             }
         } else if (cache.getByKey(this.definition.type, normalizedKey)) {
             // normal way of processing update files
             const cachedVersion = cache.getByKey(this.definition.type, normalizedKey);
-            if (!this.hasChanged(cachedVersion, metadata[metadataKey])) {
+            if (!this.hasChanged(cachedVersion, metadataMap[metadataKey])) {
                 hasError = true;
             }
+            if (Util.OPTIONS.changeKeyField) {
+                if (this.definition.keyField == this.definition.idField) {
+                    Util.logger.error(
+                        ` - --changeKeyField cannot be used for types that use their ID as key. Skipping change.`
+                    );
+                    hasError = true;
+                } else if (this.definition.keyIsFixed) {
+                    Util.logger.error(
+                        ` - type ${this.definition.type} does not support --changeKeyField and --changeKeyValue. Skipping change.`
+                    );
+                    hasError = true;
+                } else if (!metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
+                    Util.logger.error(
+                        ` - --changeKeyField is set to ${Util.OPTIONS.changeKeyField} but no value was found in the metadata. Skipping change.`
+                    );
+                    hasError = true;
+                } else if (Util.OPTIONS.changeKeyField === this.definition.keyField) {
+                    // simple issue, used WARN to avoid signaling a problem to ci/cd and don't fail deploy
+                    Util.logger.warn(
+                        ` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.`
+                    );
+                } else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
+                    // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
+                    const newKey = (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '')
+                        .trim()
+                        .slice(0, 36)
+                        .trim();
+                    if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '' > 36) {
+                        Util.logger.warn(
+                            `Customer Keys may not exceed 36 characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}`
+                        );
+                    }
+                    if (metadataKey == newKey) {
+                        Util.logger.warn(
+                            ` - --changeKeyField(${Util.OPTIONS.changeKeyField}) is providing the current value of the key (${metadataKey}). Skipping change.`
+                        );
+                    } else {
+                        Util.logger.info(
+                            ` - Changing ${this.definition.type} key from ${metadataKey} to ${newKey} via --changeKeyField=${Util.OPTIONS.changeKeyField}`
+                        );
+                        metadataMap[metadataKey][this.definition.keyField] = newKey;
+
+                        // ensure we can delete the old file(s) after the successful update
+                        Util.changedKeysMap[this.definition.type] ||= {};
+                        Util.changedKeysMap[this.definition.type][newKey] = metadataKey;
+                    }
+                }
+            } else if (Util.OPTIONS.changeKeyValue) {
+                // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
+                const newKey = Util.OPTIONS.changeKeyValue.trim().slice(0, 36).trim();
+                if (Util.OPTIONS.changeKeyValue.trim().length > 36) {
+                    Util.logger.warn(
+                        `Customer Keys may not exceed 36 characters. Truncated your value to ${newKey}`
+                    );
+                }
+                if (this.definition.keyField == this.definition.idField) {
+                    Util.logger.error(
+                        ` - --changeKeyValue cannot be used for types that use their ID as key. Skipping change.`
+                    );
+                    hasError = true;
+                } else if (this.definition.keyIsFixed) {
+                    Util.logger.error(
+                        ` - type ${this.definition.type} does not support --changeKeyField and --changeKeyValue. Skipping change.`
+                    );
+                    hasError = true;
+                } else if (metadataKey == newKey) {
+                    Util.logger.warn(
+                        ` - --changeKeyValue is providing the current value of the key (${metadataKey}). Skipping change.`
+                    );
+                } else {
+                    Util.logger.info(
+                        ` - Changing ${this.definition.type} key from ${metadataKey} to ${newKey} via --changeKeyValue`
+                    );
+                    metadataMap[metadataKey][this.definition.keyField] = newKey;
+
+                    // ensure we can delete the old file(s) after the successful update
+                    Util.changedKeysMap[this.definition.type] ||= {};
+                    Util.changedKeysMap[this.definition.type][newKey] = metadataKey;
+                }
+            }
 
             if (hasError) {
                 // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
                 metadataToUpdate.push(null);
+                return 'skip';
             } else {
                 // add ObjectId to allow actual update
-                metadata[metadataKey][this.definition.idField] =
+                metadataMap[metadataKey][this.definition.idField] =
                     cachedVersion[this.definition.idField];
 
                 metadataToUpdate.push({
-                    before: cache.getByKey(this.definition.type, normalizedKey),
-                    after: metadata[metadataKey],
+                    before: cachedVersion,
+                    after: metadataMap[metadataKey],
                 });
+
+                return 'update';
             }
         } else {
             if (hasError) {
                 // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
                 metadataToCreate.push(null);
+                return 'skip';
             } else {
-                metadataToCreate.push(metadata[metadataKey]);
+                metadataToCreate.push(metadataMap[metadataKey]);
+                return 'create';
             }
         }
     }
@@ -611,7 +797,9 @@ class MetadataType {
     static async createREST(metadataEntry, uri) {
         this.removeNotCreateableFields(metadataEntry);
         try {
-            const response = await this.client.rest.post(uri, metadataEntry);
+            // set to empty object in case API returned nothing to be able to update it in helper classes
+            const response = (await this.client.rest.post(uri, metadataEntry)) || {};
+            await this.postCreateTasks(metadataEntry, response);
             Util.logger.info(
                 ` - created ${this.definition.type}: ${
                     metadataEntry[this.definition.keyField] ||
@@ -627,8 +815,12 @@ class MetadataType {
                     metadataEntry[this.definition.nameField]
                 } / ${metadataEntry[this.definition.nameField]}:`
             );
-            for (const msg of parsedErrors) {
-                Util.logger.error('   • ' + msg);
+            if (parsedErrors.length) {
+                for (const msg of parsedErrors) {
+                    Util.logger.error('   • ' + msg);
+                }
+            } else if (ex?.message) {
+                Util.logger.debug(ex.message);
             }
             return null;
         }
@@ -638,16 +830,15 @@ class MetadataType {
      * Creates a single metadata entry via fuel-soap (generic lib not wrapper)
      *
      * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
-     * @param {string} [overrideType] can be used if the API type differs from the otherwise used type identifier
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static async createSOAP(metadataEntry, overrideType, handleOutside) {
+    static async createSOAP(metadataEntry, handleOutside) {
+        const soapType = this.definition.soapType || this.definition.type;
+        this.removeNotCreateableFields(metadataEntry);
         try {
-            this.removeNotCreateableFields(metadataEntry);
             const response = await this.client.soap.create(
-                overrideType ||
-                    this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
+                soapType.charAt(0).toUpperCase() + soapType.slice(1),
                 metadataEntry,
                 null
             );
@@ -671,16 +862,17 @@ class MetadataType {
      *
      * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {string} uri rest endpoint for PATCH
-     * @param {boolean} [usePut] some update requests require PUT instead of PATCH
+     * @param {'patch'|'post'|'put'} [httpMethod='patch'] defaults to 'patch'; some update requests require PUT instead of PATCH
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static async updateREST(metadataEntry, uri, usePut) {
+    static async updateREST(metadataEntry, uri, httpMethod = 'patch') {
         this.removeNotUpdateableFields(metadataEntry);
         try {
-            const response = usePut
-                ? await this.client.rest.put(uri, metadataEntry)
-                : await this.client.rest.patch(uri, metadataEntry);
+            // set to empty object in case API returned nothing to be able to update it in helper classes
+            const response = (await this.client.rest[httpMethod](uri, metadataEntry)) || {};
+            await this._postChangeKeyTasks(metadataEntry);
             this.checkForErrors(response);
+            await this.postUpdateTasks(metadataEntry, response);
             // some times, e.g. automation dont return a key in their update response and hence we need to fall back to name
             Util.logger.info(
                 ` - updated ${this.definition.type}: ${
@@ -704,20 +896,54 @@ class MetadataType {
         }
     }
 
+    /**
+     * helper for {@link updateREST} and {@link updateSOAP} that removes old files after the key was changed
+     *
+     * @private
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {boolean} [keepMap] some types require to check the old-key new-key relationship in their postDeployTasks; currently used by dataExtension only
+     * @returns {void}
+     */
+    static async _postChangeKeyTasks(metadataEntry, keepMap = false) {
+        if (
+            (Util.OPTIONS.changeKeyField || Util.OPTIONS.changeKeyValue) &&
+            Util.changedKeysMap?.[this.definition.type]?.[metadataEntry[this.definition.keyField]]
+        ) {
+            const oldKey =
+                Util.changedKeysMap?.[this.definition.type]?.[
+                    metadataEntry[this.definition.keyField]
+                ];
+
+            // delete file(s) of old key
+            await this.postDeleteTasks(oldKey);
+
+            // fix key in cache
+            const typeCache = cache.getCache()[this.definition.type];
+            typeCache[metadataEntry[this.definition.keyField]] = typeCache[oldKey];
+            delete typeCache[oldKey];
+
+            if (!keepMap) {
+                // clean entry from to-do list
+                delete Util.changedKeysMap?.[this.definition.type]?.[
+                    metadataEntry[this.definition.keyField]
+                ];
+            }
+        }
+    }
+
     /**
      * Updates a single metadata entry via fuel-soap (generic lib not wrapper)
      *
      * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
-     * @param {string} [overrideType] can be used if the API type differs from the otherwise used type identifier
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static async updateSOAP(metadataEntry, overrideType, handleOutside) {
+    static async updateSOAP(metadataEntry, handleOutside) {
+        const soapType = this.definition.soapType || this.definition.type;
+        this.removeNotUpdateableFields(metadataEntry);
         try {
-            this.removeNotUpdateableFields(metadataEntry);
             const response = await this.client.soap.update(
-                overrideType ||
-                    this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
+                soapType.charAt(0).toUpperCase() + soapType.slice(1),
                 metadataEntry,
                 null
             );
@@ -728,6 +954,7 @@ class MetadataType {
                     } / ${metadataEntry[this.definition.nameField]}`
                 );
             }
+            await this._postChangeKeyTasks(metadataEntry);
             return response;
         } catch (ex) {
             this._handleSOAPErrors(ex, 'updating', metadataEntry, handleOutside);
@@ -766,19 +993,17 @@ class MetadataType {
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {TYPE.SoapRequestParams} [requestParams] required for the specific request (filter for example)
+     * @param {string|number} [singleRetrieve] key of single item to filter by
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of item map
      */
-    static async retrieveSOAP(retrieveDir, requestParams, additionalFields) {
-        requestParams = requestParams || {};
-        const fields = this.getFieldNamesToRetrieve(additionalFields);
+    static async retrieveSOAP(retrieveDir, requestParams, singleRetrieve, additionalFields) {
+        requestParams ||= {};
+        const fields = this.getFieldNamesToRetrieve(additionalFields, !retrieveDir);
+        const soapType = this.definition.soapType || this.definition.type;
         let response;
         try {
-            response = await this.client.soap.retrieveBulk(
-                this.definition.type,
-                fields,
-                requestParams
-            );
+            response = await this.client.soap.retrieveBulk(soapType, fields, requestParams);
         } catch (ex) {
             this._handleSOAPErrors(ex, 'retrieving');
             return {};
@@ -788,7 +1013,8 @@ class MetadataType {
         if (retrieveDir) {
             const savedMetadata = await this.saveResults(metadata, retrieveDir, null);
             Util.logger.info(
-                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
+                    Util.getKeysString(singleRetrieve)
             );
             if (
                 this.buObject &&
@@ -805,12 +1031,11 @@ class MetadataType {
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string} uri rest endpoint for GET
-     * @param {string} [overrideType] force a metadata type (mainly used for Folders)
      * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
      * @param {string|number} [singleRetrieve] key of single item to filter by
      * @returns {Promise.<{metadata: (TYPE.MetadataTypeMap | TYPE.MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
      */
-    static async retrieveREST(retrieveDir, uri, overrideType, templateVariables, singleRetrieve) {
+    static async retrieveREST(retrieveDir, uri, templateVariables, singleRetrieve) {
         const response =
             this.definition.restPagination && !singleRetrieve
                 ? await this.client.rest.getBulk(uri, this.definition.restPageSize || 500)
@@ -831,19 +1056,18 @@ class MetadataType {
             const savedMetadata = await this.saveResults(
                 results,
                 retrieveDir,
-                overrideType,
+                null,
                 templateVariables
             );
             Util.logger.info(
-                `Downloaded: ${overrideType || this.definition.type} (${
-                    Object.keys(savedMetadata).length
-                })` + Util.getKeysString(singleRetrieve)
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
+                    Util.getKeysString(singleRetrieve)
             );
         }
 
         return {
             metadata: templateVariables ? Object.values(results)[0] : results,
-            type: overrideType || this.definition.type,
+            type: this.definition.type,
         };
     }
 
@@ -1535,8 +1759,8 @@ class MetadataType {
      * @returns {string[] | void} formatted Error Message
      */
     static checkForErrors(ex) {
+        const errors = [];
         if (ex?.response?.status >= 400 && ex?.response?.status < 600) {
-            const errors = [];
             if (ex.response.data.errors) {
                 for (const errMsg of ex.response.data.errors) {
                     errors.push(
@@ -1562,8 +1786,8 @@ class MetadataType {
             }
             Util.logger.debug(JSON.stringify(ex.config));
             Util.logger.debug(JSON.stringify(ex.response.data));
-            return errors;
         }
+        return errors;
     }
 
     /**
@@ -1593,38 +1817,44 @@ class MetadataType {
      * clean up after deleting a metadata item
      *
      * @param {string} customerKey Identifier of metadata item
-     * @returns {void}
+     * @param {string[]} [additionalExtensions] additional file extensions to delete on top of `${this.definition.type}-meta.json`
+     * @returns {Promise.<void>} - Promise
      */
-    static async postDeleteTasks(customerKey) {
-        // delete local copy: retrieve/cred/bu/type/...json
-        const jsonFile = File.normalizePath([
-            this.properties.directories.retrieve,
-            this.buObject.credential,
-            this.buObject.businessUnit,
-            this.definition.type,
-            `${customerKey}.${this.definition.type}-meta.json`,
-        ]);
-        await File.remove(jsonFile);
+    static async postDeleteTasks(customerKey, additionalExtensions) {
+        // delete local copy: retrieve/cred/bu/type/...json + whatever additional extensions were passed
+        const extArr = [`${this.definition.type}-meta.json`, ...(additionalExtensions || [])];
+        for (const ext of extArr) {
+            const jsonFile = File.normalizePath([
+                this.properties.directories.retrieve,
+                this.buObject.credential,
+                this.buObject.businessUnit,
+                this.definition.type,
+                `${customerKey}.${ext}`,
+            ]);
+            await File.remove(jsonFile);
+        }
     }
 
     /**
      * Delete a data extension from the specified business unit
      *
      * @param {string} customerKey Identifier of metadata
+     * @param {string} [overrideKeyField] optionally change the name of the key field if the api uses a different name
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {boolean} deletion success flag
      */
-    static async deleteByKeySOAP(customerKey, handleOutside) {
-        const keyObj = {};
-        keyObj[this.definition.keyField] = customerKey;
+    static async deleteByKeySOAP(customerKey, overrideKeyField, handleOutside) {
+        const metadata = {};
+        metadata[overrideKeyField || this.definition.keyField] = customerKey;
+        const soapType = this.definition.soapType || this.definition.type;
         try {
-            this.client.soap.delete(
-                this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
-                keyObj,
+            await this.client.soap.delete(
+                soapType.charAt(0).toUpperCase() + soapType.slice(1),
+                metadata,
                 null
             );
             if (!handleOutside) {
-                Util.logger.info(`- deleted ${this.definition.type}: ${customerKey}`);
+                Util.logger.info(` - deleted ${this.definition.type}: ${customerKey}`);
             }
             this.postDeleteTasks(customerKey);
 
@@ -1653,12 +1883,10 @@ class MetadataType {
      * @returns {boolean} deletion success flag
      */
     static async deleteByKeyREST(url, key, handleOutside) {
-        const keyObj = {};
-        keyObj[this.definition.keyField] = key;
         try {
             await this.client.rest.delete(url);
             if (!handleOutside) {
-                Util.logger.info(`- deleted ${this.definition.type}: ${key}`);
+                Util.logger.info(` - deleted ${this.definition.type}: ${key}`);
             }
             this.postDeleteTasks(key);
 
@@ -1682,7 +1910,7 @@ class MetadataType {
      * @returns {object} Metadata of BU in local directory
      */
     static readBUMetadataForType(readDir, listBadKeys, buMetadata) {
-        buMetadata = buMetadata || {};
+        buMetadata ||= {};
         readDir = File.normalizePath([readDir, this.definition.type]);
         try {
             if (File.pathExistsSync(readDir)) {