mcdev 3.1.1 → 4.0.0

package/lib/metadataTypes/MetadataType.js

@@ -7,32 +7,13 @@
  * in the generic version of the method
  */
 
-/**
- * @typedef {Object.<string, any>} MetadataTypeItem
- *
- * @typedef {Object} CodeExtractItem
- * @property {MetadataTypeItem} json metadata of one item w/o code
- * @property {CodeExtract[]} codeArr list of code snippets in this item
- * @property {string[]} subFolder mostly set to null, otherwise list of subfolders
- *
-* @typedef {Object} CodeExtract
- * @property {string[]} subFolder mostly set to null, otherwise subfolders path split into elements
- * @property {string} fileName name of file w/o extension
- * @property {string} fileExt file extension
- * @property {string} content file content
- * @property {'base64'} [encoding] optional for binary files
-
- *
- * @typedef {Object.<string, MetadataTypeItem>} MetadataTypeMap
- *
- * @typedef {Object.<string, MetadataTypeMap>} MultiMetadataTypeMap
- */
-
+const TYPE = require('../../types/mcdev.d');
 const Util = require('../util/util');
 const File = require('../util/file');
+const cache = require('../util/cache');
 
-// @ts-ignore
-const Promise = require('bluebird');
+const deepEqual = require('deep-equal');
+const pLimit = require('p-limit');
 const Mustache = require('mustache');
 
 /**
@@ -41,34 +22,18 @@ const Mustache = require('mustache');
  *
  */
 class MetadataType {
-    /**
-     * Instantiates a metadata constructor to avoid passing variables.
-     *
-     * @param {Util.ET_Client} client client for sfmc fuelsdk
-     * @param {string} businessUnit Name of business unit (corresponding to their keys in 'properties.json' file). Used to access correct directories
-     * @param {Object} cache metadata cache
-     * @param {Object} properties mcdev config
-     * @param {string} [subType] limit retrieve to specific subType
-     */
-    constructor(client, businessUnit, cache, properties, subType) {
-        this.client = client;
-        this.businessUnit = businessUnit;
-        this.cache = cache;
-        this.properties = properties;
-        this.subType = subType;
-    }
-
     /**
      * Returns file contents mapped to their filename without '.json' ending
+     *
      * @param {string} dir directory that contains '.json' files to be read
      * @param {boolean} [listBadKeys=false] do not print errors, used for badKeys()
-     * @returns {Object} fileName => fileContent map
+     * @returns {TYPE.MetadataTypeMap} fileName => fileContent map
      */
     static getJsonFromFS(dir, listBadKeys) {
         const fileName2FileContent = {};
         try {
             const files = File.readdirSync(dir);
-            files.forEach((fileName) => {
+            for (const fileName of files) {
                 try {
                     if (fileName.endsWith('.json')) {
                         const fileContent = File.readJSONFile(dir, fileName, true, false);
@@ -77,10 +42,11 @@ class MetadataType {
                         );
                         // We always store the filename using the External Key (CustomerKey or key) to avoid duplicate names.
                         // to ensure any changes are done to both the filename and external key do a check here
-                        if (
-                            fileContent[this.definition.keyField] === fileNameWithoutEnding ||
-                            listBadKeys
-                        ) {
+                        // ! convert numbers to string to allow numeric keys to be checked properly
+                        const key = Number.isInteger(fileContent[this.definition.keyField])
+                            ? fileContent[this.definition.keyField].toString()
+                            : fileContent[this.definition.keyField];
+                        if (key === fileNameWithoutEnding || listBadKeys) {
                             fileName2FileContent[fileNameWithoutEnding] = fileContent;
                         } else {
                             Util.metadataLogger(
@@ -90,7 +56,7 @@ class MetadataType {
                                 'Name of the Metadata and the External Identifier must match',
                                 JSON.stringify({
                                     Filename: fileNameWithoutEnding,
-                                    ExternalIdentifier: fileContent[this.definition.keyField],
+                                    ExternalIdentifier: key,
                                 })
                             );
                         }
@@ -99,7 +65,7 @@ class MetadataType {
                     // by catching this in the loop we gracefully handle the issue and move on to the next file
                     Util.metadataLogger('debug', this.definition.type, 'getJsonFromFS', ex);
                 }
-            });
+            }
         } catch (ex) {
             // this will catch issues with readdirSync
             Util.metadataLogger('debug', this.definition.type, 'getJsonFromFS', ex);
@@ -110,6 +76,7 @@ class MetadataType {
 
     /**
      * Returns fieldnames of Metadata Type. 'this.definition.fields' variable only set in child classes.
+     *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @returns {string[]} Fieldnames
      */
@@ -117,7 +84,7 @@ class MetadataType {
         const fieldNames = [];
         for (const fieldName in this.definition.fields) {
             if (
-                (additionalFields && additionalFields.includes(fieldName)) ||
+                additionalFields?.includes(fieldName) ||
                 this.definition.fields[fieldName].retrieving
             ) {
                 fieldNames.push(fieldName);
@@ -132,18 +99,23 @@ class MetadataType {
 
     /**
      * Deploys metadata
-     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
+     *
+     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
      * @param {string} retrieveDir directory where metadata after deploy should be saved
-     * @param {Util.BuObject} buObject properties for auth
-     * @returns {Promise<Object>} Promise of keyField => metadata map
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @returns {Promise.<object>} Promise of keyField => metadata map
      */
     static async deploy(metadata, deployDir, retrieveDir, buObject) {
         const upsertResults = await this.upsert(metadata, deployDir, buObject);
         await this.postDeployTasks(upsertResults, metadata);
         const savedMetadata = await this.saveResults(upsertResults, retrieveDir, null);
-        if (this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
+        if (
+            this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type) &&
+            !this.definition.documentInOneFile
+        ) {
             // * do not await here as this might take a while and has no impact on the deploy
+            // * this should only be run if documentation is on a per metadata record level. Types that document an overview into a single file will need a full retrieve to work instead
             this.document(buObject, savedMetadata, true);
         }
         return upsertResults;
@@ -151,86 +123,69 @@ class MetadataType {
 
     /**
      * Gets executed after deployment of metadata type
-     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
-     * @param {MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
+     *
+     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {TYPE.MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
      * @returns {void}
      */
     static postDeployTasks(metadata, originalMetadata) {}
 
     /**
      * Gets executed after retreive of metadata type
-     * @param {MetadataTypeItem} metadata a single item
+     *
+     * @param {TYPE.MetadataTypeItem} metadata a single item
      * @param {string} targetDir folder where retrieves should be saved
      * @param {boolean} [isTemplating] signals that we are retrieving templates
-     * @returns {MetadataTypeItem} cloned metadata
+     * @returns {TYPE.MetadataTypeItem} cloned metadata
      */
     static postRetrieveTasks(metadata, targetDir, isTemplating) {
         return JSON.parse(JSON.stringify(metadata));
     }
-    /**
-     * used to synchronize name and external key during retrieveAsTemplate
-     * @param {MetadataTypeItem} metadata a single item
-     * @param {string} [warningMsg] optional msg to show the user
-     * @returns {void}
-     */
-    static overrideKeyWithName(metadata, warningMsg) {
-        if (
-            this.definition.nameField &&
-            this.definition.keyField &&
-            metadata[this.definition.nameField] !== metadata[this.definition.keyField]
-        ) {
-            Util.logger.warn(
-                `Reset external key of ${this.definition.type} ${
-                    metadata[this.definition.nameField]
-                } to its name (${metadata[this.definition.keyField]})`
-            );
-            if (warningMsg) {
-                Util.logger.warn(warningMsg);
-            }
-            // do this after printing to cli or we lost the info
-            metadata[this.definition.keyField] = metadata[this.definition.nameField];
-        }
-    }
     /**
      * Gets metadata from Marketing Cloud
+     *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Util.BuObject} buObject properties for auth
+     * @param {TYPE.BuObject} buObject properties for auth
      * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     * @param {string} [key] customer key of single item to retrieve
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
      */
-    static retrieve(retrieveDir, additionalFields, buObject, subType) {
+    static retrieve(retrieveDir, additionalFields, buObject, subType, key) {
         Util.metadataLogger('error', this.definition.type, 'retrieve', `Not Supported`);
         const metadata = {};
         return { metadata: null, type: this.definition.type };
     }
     /**
      * Gets metadata from Marketing Cloud
+     *
+     * @param {TYPE.BuObject} [buObject] properties for auth
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Util.BuObject} buObject properties for auth
      * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
      */
-    static retrieveChangelog(additionalFields, buObject, subType) {
-        return this.retrieve(null, additionalFields, buObject, subType);
+    static retrieveChangelog(buObject, additionalFields, subType) {
+        return this.retrieveForCache(buObject, subType);
     }
 
     /**
      * Gets metadata cache with limited fields and does not store value to disk
-     * @param {Util.BuObject} buObject properties for auth
+     *
+     * @param {TYPE.BuObject} buObject properties for auth
      * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
      */
     static async retrieveForCache(buObject, subType) {
         return this.retrieve(null, null, buObject, subType);
     }
     /**
      * Gets metadata cache with limited fields and does not store value to disk
+     *
      * @param {string} templateDir Directory where retrieved metadata directory will be saved
      * @param {string} name name of the metadata file
-     * @param {Util.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
      * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     * @returns {Promise.<TYPE.MetadataTypeItemObj>} metadata
      */
     static retrieveAsTemplate(templateDir, name, templateVariables, subType) {
         Util.logger.error('retrieveAsTemplate is not supported yet for ' + this.definition.type);
@@ -242,60 +197,235 @@ class MetadataType {
         );
         return { metadata: null, type: this.definition.type };
     }
+    /**
+     * Gets metadata cache with limited fields and does not store value to disk
+     *
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {string} templateDir (List of) Directory where built definitions will be saved
+     * @param {string} key name of the metadata file
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {Promise.<TYPE.MetadataTypeItemObj>} single metadata
+     */
+    static async buildTemplate(retrieveDir, templateDir, key, templateVariables) {
+        // retrieve metadata template
+        let metadataStr;
+        const typeDirArr = [this.definition.type];
+        const subType = await this.findSubType(retrieveDir, key);
+        if (subType) {
+            typeDirArr.push(subType);
+        }
+        const suffix = subType ? `-${subType}-meta` : '-meta';
+        const fileName = key + '.' + this.definition.type + suffix;
+        try {
+            // ! do not load via readJSONFile to ensure we get a string, not parsed JSON
+            // templated files might contain illegal json before the conversion back to the file that shall be saved
+            metadataStr = await File.readFilteredFilename(
+                [retrieveDir, ...typeDirArr],
+                fileName,
+                'json'
+            );
+        } catch (ex) {
+            try {
+                metadataStr = await this.readSecondaryFolder(
+                    retrieveDir,
+                    typeDirArr,
+                    key,
+                    fileName,
+                    ex
+                );
+            } catch {
+                throw new Error(
+                    `${this.definition.type}:: Could not find ./${File.normalizePath([
+                        retrieveDir,
+                        ...typeDirArr,
+                        fileName + '.json',
+                    ])}.`
+                );
+            }
+            // return;
+        }
+        const metadata = JSON.parse(Util.replaceByObject(metadataStr, templateVariables));
+        this.keepTemplateFields(metadata);
+
+        // handle extracted code
+        // templating to extracted content is applied inside of buildTemplateForNested()
+        await this.buildTemplateForNested(
+            retrieveDir,
+            templateDir,
+            metadata,
+            templateVariables,
+            key
+        );
+
+        try {
+            // write to file
+            await File.writeJSONToFile(
+                [templateDir, ...typeDirArr],
+                key + '.' + this.definition.type + suffix,
+                metadata
+            );
+            Util.logger.info(
+                `- templated ${this.definition.type}: ${key} (${
+                    metadata[this.definition.nameField]
+                })`
+            );
+
+            return { metadata: metadata, type: this.definition.type };
+        } catch (ex) {
+            throw new Error(`${this.definition.type}:: ${ex.message}`);
+        }
+    }
 
     /**
      * Gets executed before deploying metadata
-     * @param {MetadataTypeItem} metadata a single metadata item
+     *
+     * @param {TYPE.MetadataTypeItem} metadata a single metadata item
      * @param {string} deployDir folder where files for deployment are stored
-     * @returns {Promise<MetadataTypeItem>} Promise of a single metadata item
+     * @param {TYPE.BuObject} buObject buObject properties for auth
+     * @returns {Promise.<TYPE.MetadataTypeItem>} Promise of a single metadata item
      */
-    static async preDeployTasks(metadata, deployDir) {
+    static async preDeployTasks(metadata, deployDir, buObject) {
         return metadata;
     }
 
     /**
      * Abstract create method that needs to be implemented in child metadata type
-     * @param {MetadataTypeItem} metadata single metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadata single metadata entry
      * @param {string} deployDir directory where deploy metadata are saved
      * @returns {void}
      */
     static create(metadata, deployDir) {
-        Util.metadataLogger('error', this.definition.type, 'create', 'create not supported');
+        Util.logger.error(
+            ` ☇ skipping ${this.definition.type} ${metadata[this.definition.keyField]} / ${
+                metadata[this.definition.nameField]
+            }: create is not supported yet for ${this.definition.type}`
+        );
         return;
     }
 
     /**
      * Abstract update method that needs to be implemented in child metadata type
-     * @param {MetadataTypeItem} metadata single metadata entry
-     * @param {MetadataTypeItem} [metadataBefore] metadata mapped by their keyField
+     *
+     * @param {TYPE.MetadataTypeItem} metadata single metadata entry
+     * @param {TYPE.MetadataTypeItem} [metadataBefore] metadata mapped by their keyField
      * @returns {void}
      */
     static update(metadata, metadataBefore) {
-        Util.metadataLogger('error', this.definition.type, 'update', 'update not supported');
+        Util.logger.error(
+            ` ☇ skipping ${this.definition.type} ${metadata[this.definition.keyField]} / ${
+                metadata[this.definition.nameField]
+            }: update is not supported yet for ${this.definition.type}`
+        );
         return;
     }
 
+    /**
+     * test if metadata was actually changed or not to potentially skip it during deployment
+     *
+     * @param {TYPE.MetadataTypeItem} cachedVersion cached version from the server
+     * @param {TYPE.MetadataTypeItem} metadata item to upload
+     * @param {string} [fieldName] optional field name to use for identifying the record in logs
+     * @returns {boolean} true if metadata was changed
+     */
+    static hasChanged(cachedVersion, metadata, fieldName) {
+        // should be set up type by type but the *_generic version is likely a good start for many types
+        return true;
+    }
+    /**
+     * test if metadata was actually changed or not to potentially skip it during deployment
+     *
+     * @param {TYPE.MetadataTypeItem} cachedVersion cached version from the server
+     * @param {TYPE.MetadataTypeItem} metadata item to upload
+     * @param {string} [fieldName] optional field name to use for identifying the record in logs
+     * @param {boolean} [silent] optionally suppress logging
+     * @returns {boolean} true on first identified deviation or false if none are found
+     */
+    static hasChangedGeneric(cachedVersion, metadata, fieldName, silent) {
+        if (!cachedVersion) {
+            return true;
+        }
+        // we do need the full set in other places and hence need to work with a clone here
+        const clonedMetada = JSON.parse(JSON.stringify(metadata));
+        this.removeNotUpdateableFields(clonedMetada);
+        // iterate over what we want to upload rather than what we cached to avoid false positives
+        for (const prop in clonedMetada) {
+            if (this.definition.ignoreFieldsForUpdateCheck?.includes(prop)) {
+                continue;
+            }
+            if (
+                clonedMetada[prop] === null ||
+                ['string', 'number', 'boolean'].includes(typeof clonedMetada[prop])
+            ) {
+                // check simple variables directly
+                // check should ignore types to bypass string/number auto-conversions caused by SFMC-SDK
+                if (clonedMetada[prop] != cachedVersion[prop]) {
+                    Util.logger.debug(
+                        `${this.definition.type}:: ${
+                            clonedMetada[fieldName || this.definition.keyField]
+                        }.${prop} changed: '${cachedVersion[prop]}' to '${clonedMetada[prop]}'`
+                    );
+                    return true;
+                }
+            } else if (deepEqual(clonedMetada[prop], cachedVersion[prop])) {
+                // test complex objects here
+                Util.logger.debug(
+                    `${this.definition.type}:: ${
+                        clonedMetada[fieldName || this.definition.keyField]
+                    }.${prop} changed: '${cachedVersion[prop]}' to '${clonedMetada[prop]}'`
+                );
+                return true;
+            }
+        }
+        if (!silent) {
+            Util.logger.verbose(
+                ` ☇ skipping ${this.definition.type} ${clonedMetada[this.definition.keyField]} / ${
+                    clonedMetada[fieldName || this.definition.nameField]
+                }: no change detected`
+            );
+        }
+        return false;
+    }
     /**
      * MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
-     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
+     *
+     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
-     * @param {Util.BuObject} [buObject] properties for auth
-     * @returns {Promise<MetadataTypeMap>} keyField => metadata map
+     * @param {TYPE.BuObject} [buObject] properties for auth
+     * @returns {Promise.<TYPE.MetadataTypeMap>} keyField => metadata map
      */
     static async upsert(metadata, deployDir, buObject) {
         const metadataToUpdate = [];
         const metadataToCreate = [];
+        let filteredByPreDeploy = 0;
         for (const metadataKey in metadata) {
+            let hasError = false;
             try {
                 // preDeployTasks parsing
-                const deployableMetadata = await this.preDeployTasks(
-                    metadata[metadataKey],
-                    deployDir
-                );
+                let deployableMetadata;
+                try {
+                    deployableMetadata = await this.preDeployTasks(
+                        metadata[metadataKey],
+                        deployDir,
+                        buObject
+                    );
+                } catch (ex) {
+                    // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
+                    hasError = true;
+                    deployableMetadata = metadata[metadataKey];
+                    Util.logger.error(
+                        ` ☇ skipping ${this.definition.type} ${
+                            deployableMetadata[this.definition.keyField]
+                        } / ${deployableMetadata[this.definition.nameField]}: ${ex.message}`
+                    );
+                }
                 // if preDeploy returns nothing then it cannot be deployed so skip deployment
                 if (deployableMetadata) {
                     metadata[metadataKey] = deployableMetadata;
-                    const normalizedKey = File.reverseFilterIllegalFilenames(metadataKey);
+                    // create normalizedKey off of whats in the json rather than from "metadataKey" because preDeployTasks might have altered something (type asset)
+                    const normalizedKey = File.reverseFilterIllegalFilenames(
+                        deployableMetadata[this.definition.keyField]
+                    );
                     // Update if it already exists; Create it if not
                     if (
                         Util.logger.level === 'debug' &&
@@ -305,52 +435,79 @@ class MetadataType {
                         // only used if resource is excluded from cache and we still want to update it
                         // needed e.g. to rewire lost folders
                         Util.logger.warn(
-                            'Hotfix for non-cachable resource found in deploy folder. Trying update:'
+                            ' - Hotfix for non-cachable resource found in deploy folder. Trying update:'
                         );
                         Util.logger.warn(JSON.stringify(metadata[metadataKey]));
-                        metadataToUpdate.push({
-                            before: {},
-                            after: metadata[metadataKey],
-                        });
-                    } else if (this.cache[this.definition.type][normalizedKey]) {
+                        if (hasError) {
+                            metadataToUpdate.push(null);
+                        } else {
+                            metadataToUpdate.push({
+                                before: {},
+                                after: metadata[metadataKey],
+                            });
+                        }
+                    } else if (cache.getByKey(this.definition.type, normalizedKey)) {
                         // normal way of processing update files
-                        metadata[metadataKey][this.definition.idField] =
-                            this.cache[this.definition.type][normalizedKey][
-                                this.definition.idField
-                            ];
-                        metadataToUpdate.push({
-                            before: this.cache[this.definition.type][normalizedKey],
-                            after: metadata[metadataKey],
-                        });
+                        const cachedVersion = cache.getByKey(this.definition.type, normalizedKey);
+                        if (!this.hasChanged(cachedVersion, metadata[metadataKey])) {
+                            hasError = true;
+                        }
+
+                        if (hasError) {
+                            // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
+                            metadataToUpdate.push(null);
+                        } else {
+                            // add ObjectId to allow actual update
+                            metadata[metadataKey][this.definition.idField] =
+                                cachedVersion[this.definition.idField];
+
+                            metadataToUpdate.push({
+                                before: cache.getByKey(this.definition.type, normalizedKey),
+                                after: metadata[metadataKey],
+                            });
+                        }
                     } else {
-                        metadataToCreate.push(metadata[metadataKey]);
+                        if (hasError) {
+                            // do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
+                            metadataToCreate.push(null);
+                        } else {
+                            metadataToCreate.push(metadata[metadataKey]);
+                        }
                     }
+                } else {
+                    filteredByPreDeploy++;
                 }
             } catch (ex) {
-                Util.metadataLogger('error', this.definition.type, 'upsert', ex, metadataKey);
+                Util.logger.errorStack(
+                    ex,
+                    `Upserting ${this.definition.type} failed: ${ex.message}`
+                );
             }
         }
-
+        const createLimit = pLimit(10);
         const createResults = (
-            await Promise.map(
-                metadataToCreate,
-                (metadataEntry) => this.create(metadataEntry, deployDir),
-                { concurrency: 10 }
+            await Promise.all(
+                metadataToCreate
+                    .filter((r) => r !== undefined && r !== null)
+                    .map((metadataEntry) =>
+                        createLimit(() => this.create(metadataEntry, deployDir))
+                    )
             )
         ).filter((r) => r !== undefined && r !== null);
+        const updateLimit = pLimit(10);
         const updateResults = (
-            await Promise.map(
-                metadataToUpdate,
-                (metadataEntry) => this.update(metadataEntry.after, metadataEntry.before),
-                { concurrency: 10 }
+            await Promise.all(
+                metadataToUpdate
+                    .filter((r) => r !== undefined && r !== null)
+                    .map((metadataEntry) =>
+                        updateLimit(() => this.update(metadataEntry.after, metadataEntry.before))
+                    )
             )
         ).filter((r) => r !== undefined && r !== null);
         // Logging
-        Util.metadataLogger(
-            'info',
-            this.definition.type,
-            'upsert',
-            `${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated`
+        Util.logger.info(
+            `${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
+                (filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
         );
 
         // if Results then parse as SOAP
@@ -360,60 +517,57 @@ class MetadataType {
             // @ts-ignore
             const metadataResults = createResults
                 .concat(updateResults)
-                .filter((r) => r !== undefined && r !== null)
-                .map((r) => r.body.Results)
-                .flat()
+                // TODO remove Object.keys check after create/update SOAP methods stop returning empty objects instead of null
+                .filter((r) => r !== undefined && r !== null && Object.keys(r).length !== 0)
+                .flatMap((r) => r.Results)
                 .map((r) => r.Object);
             return this.parseResponseBody({ Results: metadataResults });
         } else {
             // put in Retrieve Format for parsing
             // todo add handling when response does not contain items.
             // @ts-ignore
-            const metadataResults = createResults
-                .concat(updateResults)
-                .filter((r) => r !== undefined && r !== null && r.res && r.res.body)
-                .map((r) => r.res.body);
+            const metadataResults = createResults.concat(updateResults).filter(Boolean);
             return this.parseResponseBody({ items: metadataResults });
         }
     }
 
     /**
      * Creates a single metadata entry via REST
-     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {string} uri rest endpoint for POST
      * @returns {Promise} Promise
      */
     static async createREST(metadataEntry, uri) {
         this.removeNotCreateableFields(metadataEntry);
-        const options = {
-            uri: uri,
-            json: metadataEntry,
-            headers: {},
-        };
         try {
-            let response;
-            await Util.retryOnError(
-                `Retrying ${this.definition.type}: ${metadataEntry[this.definition.nameField]}`,
-                async () => (response = await this.client.RestClient.post(options))
-            );
-            this.checkForErrors(response);
+            const response = await this.client.rest.post(uri, metadataEntry);
             Util.logger.info(
-                `- created ${this.definition.type}: ${metadataEntry[this.definition.keyField]}`
+                ` - created ${this.definition.type}: ${
+                    metadataEntry[this.definition.keyField] ||
+                    metadataEntry[this.definition.nameField]
+                } / ${metadataEntry[this.definition.nameField]}`
             );
             return response;
         } catch (ex) {
+            const parsedErrors = this.checkForErrors(ex);
             Util.logger.error(
-                `- error creating ${this.definition.type}: ${
-                    metadataEntry[this.definition.keyField]
-                } (${ex.message})`
+                ` ☇ error creating ${this.definition.type} ${
+                    metadataEntry[this.definition.keyField] ||
+                    metadataEntry[this.definition.nameField]
+                } / ${metadataEntry[this.definition.nameField]}:`
             );
+            for (const msg of parsedErrors) {
+                Util.logger.error('   • ' + msg);
+            }
             return null;
         }
     }
 
     /**
      * Creates a single metadata entry via fuel-soap (generic lib not wrapper)
-     * @param {MetadataTypeItem} metadataEntry single metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
      * @param {string} [overrideType] can be used if the API type differs from the otherwise used type identifier
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise} Promise
@@ -421,39 +575,29 @@ class MetadataType {
     static async createSOAP(metadataEntry, overrideType, handleOutside) {
         try {
             this.removeNotCreateableFields(metadataEntry);
-            let response;
-            await Util.retryOnError(
-                `Retrying to create ${this.definition.type}: ${
-                    metadataEntry[this.definition.nameField]
-                }`,
-                async () =>
-                    (response = await new Promise((resolve, reject) => {
-                        this.client.SoapClient.create(
-                            overrideType ||
-                                this.definition.type.charAt(0).toUpperCase() +
-                                    this.definition.type.slice(1),
-                            metadataEntry,
-                            null,
-                            (error, response) => (error ? reject(error) : resolve(response))
-                        );
-                    }))
+            const response = await this.client.soap.create(
+                overrideType ||
+                    this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
+                metadataEntry,
+                null
             );
+
             if (!handleOutside) {
                 Util.logger.info(
-                    `- created ${this.definition.type}: ${metadataEntry[this.definition.keyField]}`
+                    ` - created ${this.definition.type}: ${
+                        metadataEntry[this.definition.keyField]
+                    } / ${metadataEntry[this.definition.nameField]}`
                 );
             }
             return response;
         } catch (ex) {
             if (!handleOutside) {
-                let errorMsg;
-                if (ex.results && ex.results.length) {
-                    errorMsg = `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`;
-                } else {
-                    errorMsg = ex.message;
-                }
+                const errorMsg =
+                    ex.results && ex.results.length
+                        ? `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`
+                        : ex.message;
                 Util.logger.error(
-                    `- error creating ${this.definition.type} '${
+                    ` ☇ error creating ${this.definition.type} '${
                         metadataEntry[this.definition.keyField]
                     }': ${errorMsg}`
                 );
@@ -467,85 +611,71 @@ class MetadataType {
 
     /**
      * Updates a single metadata entry via REST
-     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {string} uri rest endpoint for PATCH
      * @returns {Promise} Promise
      */
     static async updateREST(metadataEntry, uri) {
         this.removeNotUpdateableFields(metadataEntry);
-        const options = {
-            uri: uri,
-            json: metadataEntry,
-            headers: {},
-        };
         try {
-            let response;
-            await Util.retryOnError(
-                `Retrying ${this.definition.type}: ${metadataEntry[this.definition.nameField]}`,
-                async () => (response = await this.client.RestClient.patch(options))
-            );
+            const response = await this.client.rest.patch(uri, metadataEntry);
             this.checkForErrors(response);
             // some times, e.g. automation dont return a key in their update response and hence we need to fall back to name
             Util.logger.info(
-                `- updated ${this.definition.type}: ${
+                ` - updated ${this.definition.type}: ${
                     metadataEntry[this.definition.keyField] ||
                     metadataEntry[this.definition.nameField]
-                }`
+                } / ${metadataEntry[this.definition.nameField]}`
             );
             return response;
         } catch (ex) {
+            const parsedErrors = this.checkForErrors(ex);
             Util.logger.error(
-                `- error updating ${this.definition.type}: ${
-                    metadataEntry[this.definition.keyField]
-                } (${ex.message})`
+                ` ☇ error updating ${this.definition.type} ${
+                    metadataEntry[this.definition.keyField] ||
+                    metadataEntry[this.definition.nameField]
+                } / ${metadataEntry[this.definition.nameField]}:`
             );
+            for (const msg of parsedErrors) {
+                Util.logger.error('   • ' + msg);
+            }
             return null;
         }
     }
 
     /**
      * Updates a single metadata entry via fuel-soap (generic lib not wrapper)
-     * @param {MetadataTypeItem} metadataEntry single metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
      * @param {string} [overrideType] can be used if the API type differs from the otherwise used type identifier
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise} Promise
      */
     static async updateSOAP(metadataEntry, overrideType, handleOutside) {
-        let response;
         try {
             this.removeNotUpdateableFields(metadataEntry);
-            await Util.retryOnError(
-                `Retrying to update ${this.definition.type}: ${
-                    metadataEntry[this.definition.nameField]
-                }`,
-                async () =>
-                    (response = await new Promise((resolve, reject) => {
-                        this.client.SoapClient.update(
-                            overrideType ||
-                                this.definition.type.charAt(0).toUpperCase() +
-                                    this.definition.type.slice(1),
-                            metadataEntry,
-                            null,
-                            (error, response) => (error ? reject(error) : resolve(response))
-                        );
-                    }))
+            const response = await this.client.soap.update(
+                overrideType ||
+                    this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
+                metadataEntry,
+                null
             );
             if (!handleOutside) {
                 Util.logger.info(
-                    `- updated ${this.definition.type}: ${metadataEntry[this.definition.keyField]}`
+                    ` - updated ${this.definition.type}: ${
+                        metadataEntry[this.definition.keyField]
+                    } /  ${metadataEntry[this.definition.nameField]}`
                 );
             }
             return response;
         } catch (ex) {
             if (!handleOutside) {
-                let errorMsg;
-                if (ex.results && ex.results.length) {
-                    errorMsg = `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`;
-                } else {
-                    errorMsg = ex.message;
-                }
+                const errorMsg = ex?.json?.Results.length
+                    ? `${ex.json.Results[0].StatusMessage} (Code ${ex.json.Results[0].ErrorCode})`
+                    : ex.message;
                 Util.logger.error(
-                    `- error updating ${this.definition.type} '${
+                    ` ☇ error updating ${this.definition.type} '${
                         metadataEntry[this.definition.keyField]
                     }': ${errorMsg}`
                 );
@@ -558,23 +688,23 @@ class MetadataType {
     }
     /**
      * Retrieves SOAP via generic fuel-soap wrapper based metadata of metadata type into local filesystem. executes callback with retrieved metadata
+     *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
-     * @param {Util.BuObject} buObject properties for auth
-     * @param {Object} [options] required for the specific request (filter for example)
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @param {TYPE.SoapRequestParams} [requestParams] required for the specific request (filter for example)
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {string} [overrideType] can be used if the API type differs from the otherwise used type identifier
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} Promise of item map
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of item map
      */
-    static async retrieveSOAPgeneric(
-        retrieveDir,
-        buObject,
-        options,
-        additionalFields,
-        overrideType
-    ) {
+    static async retrieveSOAP(retrieveDir, buObject, requestParams, additionalFields) {
+        requestParams = requestParams || {};
         const fields = this.getFieldNamesToRetrieve(additionalFields);
+        const response = await this.client.soap.retrieveBulk(
+            this.definition.type,
+            fields,
+            requestParams
+        );
+        const metadata = this.parseResponseBody(response);
 
-        const metadata = await this.retrieveSOAPBody(fields, options, overrideType);
         if (retrieveDir) {
             const savedMetadata = await this.saveResults(metadata, retrieveDir, null);
             Util.logger.info(
@@ -589,107 +719,31 @@ class MetadataType {
         }
         return { metadata: metadata, type: this.definition.type };
     }
-    /**
-     * helper that handles batched retrieve via SOAP
-     * @param {string[]} fields list of fields that we want to see retrieved
-     * @param {Object} [options] required for the specific request (filter for example)
-     * @param {string} [type] optionally overwrite the API type of the metadata here
-     * @returns {Promise<MetadataTypeMap>} keyField => metadata map
-     */
-    static async retrieveSOAPBody(fields, options, type) {
-        let status;
-        let batchCounter = 1;
-        const defaultBatchSize = 2500; // 2500 is the typical batch size
-        options = options || {};
-        let metadata = {};
-        do {
-            let resultsBatch;
-            await Util.retryOnError(`Retrying ${this.definition.type}`, async () => {
-                resultsBatch = await new Promise((resolve, reject) => {
-                    this.client.SoapClient.retrieve(
-                        type || this.definition.type,
-                        fields,
-                        options || {},
-                        (error, response) => {
-                            if (error) {
-                                Util.logger.debug(`SOAP.retrieve Error: ${error.message}`);
-                                reject(error);
-                            }
-                            if (response) {
-                                resolve(response.body);
-                            } else {
-                                // fallback, lets make sure surrounding methods know we got an empty result back
-                                resolve({});
-                            }
-                        }
-                    );
-                });
-            });
-            status = resultsBatch.OverallStatus;
-            if (status === 'MoreDataAvailable') {
-                options.continueRequest = resultsBatch.RequestID;
-                Util.logger.info(
-                    `-  more than ${batchCounter * defaultBatchSize} ${
-                        this.definition.typeName
-                    }s found in Business Unit - loading next batch...`
-                );
-                batchCounter++;
-            }
-            const metadataBatch = this.parseResponseBody(resultsBatch);
-
-            metadata = { ...metadata, ...metadataBatch };
-        } while (status === 'MoreDataAvailable');
-
-        return metadata;
-    }
 
     /**
      * Retrieves Metadata for Rest Types
+     *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string} uri rest endpoint for GET
      * @param {string} [overrideType] force a metadata type (mainly used for Folders)
-     * @param {Util.TemplateMap} [templateVariables] variables to be replaced in the metadata
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} Promise of item map
+     * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
+     * @param {string|number} [singleRetrieve] key of single item to filter by
+     * @returns {Promise.<{metadata: (TYPE.MetadataTypeMap | TYPE.MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
      */
-    static async retrieveREST(retrieveDir, uri, overrideType, templateVariables) {
-        const options = {
-            uri: uri,
-            headers: {},
-        };
-        let moreResults;
-        let lastPage = null;
-        let results = {};
-        do {
-            options.uri = this.paginate(options.uri, lastPage);
-            let response;
-            await Util.retryOnError(`Retrying ${this.definition.type}`, async () => {
-                response = await this.client.RestClient.get(options);
-            });
-            const metadata = this.parseResponseBody(response.body);
-            results = Object.assign(results, metadata);
-            if (
-                this.definition.restPagination &&
-                Object.keys(metadata).length > 0 &&
-                response.body.page * response.body.pageSize < response.body.count
-            ) {
-                lastPage = Number(response.body.page);
-                moreResults = true;
-            } else {
-                moreResults = false;
-            }
-        } while (moreResults);
+    static async retrieveREST(retrieveDir, uri, overrideType, templateVariables, singleRetrieve) {
+        const response =
+            this.definition.restPagination && !singleRetrieve
+                ? await this.client.rest.getBulk(uri)
+                : await this.client.rest.get(uri);
+        const results = this.parseResponseBody(response, singleRetrieve);
         // get extended metadata if applicable
         if (this.definition.hasExtended) {
-            const extended = await Promise.all(
-                Object.keys(results).map((key) =>
-                    this.client.RestClient.get({
-                        uri: uri + results[key][this.definition.idField],
-                    })
-                )
+            const extended = await this.client.rest.getCollection(
+                Object.keys(results).map((key) => uri + results[key][this.definition.idField])
             );
             for (const ext of extended) {
-                const key = ext.body[this.definition.keyField];
-                results[key] = Object.assign(results[key], ext.body);
+                const key = ext[this.definition.keyField];
+                results[key] = Object.assign(results[key], ext);
             }
         }
 
@@ -707,21 +761,27 @@ class MetadataType {
             );
         }
 
-        return { metadata: results, type: overrideType || this.definition.type };
+        return {
+            metadata: templateVariables ? Object.values(results)[0] : results,
+            type: overrideType || this.definition.type,
+        };
     }
 
     /**
      * Builds map of metadata entries mapped to their keyfields
-     * @param {Object} body json of response body
-     * @returns {Promise<MetadataTypeMap>} keyField => metadata map
+     *
+     * @param {object} body json of response body
+     * @param {string|number} [singleRetrieve] key of single item to filter by
+     * @returns {TYPE.MetadataTypeMap} keyField => metadata map
      */
-    static parseResponseBody(body) {
+    static parseResponseBody(body, singleRetrieve) {
         const bodyIteratorField = this.definition.bodyIteratorField;
         const keyField = this.definition.keyField;
         const metadataStructure = {};
+
         if (body !== null) {
-            // in some cases data is just an array
-            if (Array.isArray(bodyIteratorField)) {
+            if (Array.isArray(body)) {
+                // in some cases data is just an array
                 for (const item of body) {
                     const key = item[keyField];
                     metadataStructure[key] = item;
@@ -731,6 +791,20 @@ class MetadataType {
                     const key = item[keyField];
                     metadataStructure[key] = item;
                 }
+            } else if (singleRetrieve) {
+                // some types will return a single item intead of an array if the key is supported by their api
+                metadataStructure[singleRetrieve] = body;
+
+                return metadataStructure;
+            }
+            if (
+                metadataStructure[singleRetrieve] &&
+                (typeof singleRetrieve === 'string' || typeof singleRetrieve === 'number')
+            ) {
+                // in case we really just wanted one entry but couldnt do so in the api call, filter it here
+                const single = {};
+                single[singleRetrieve] = metadataStructure[singleRetrieve];
+                return single;
             }
         }
         return metadataStructure;
@@ -738,10 +812,11 @@ class MetadataType {
 
     /**
      * Deletes a field in a metadata entry if the selected definition property equals false.
+     *
      * @example
      * Removes field (or nested fields childs) that are not updateable
      * deleteFieldByDefinition(metadataEntry, 'CustomerKey', 'isUpdateable');
-     * @param {MetadataTypeItem} metadataEntry One entry of a metadataType
+     * @param {TYPE.MetadataTypeItem} metadataEntry One entry of a metadataType
      * @param {string} fieldPath field path to be checked if it conforms to the definition (dot seperated if nested): 'fuu.bar'
      * @param {'isCreateable'|'isUpdateable'|'retrieving'|'templating'} definitionProperty delete field if definitionProperty equals false for specified field. Options: [isCreateable | isUpdateable]
      * @param {string} origin string of parent object, required when using arrays as these are parsed slightly differently.
@@ -752,30 +827,20 @@ class MetadataType {
         let fieldContent;
         try {
             fieldContent = fieldPath.split('.').reduce((field, key) => field[key], metadataEntry);
-        } catch (e) {
+        } catch {
             // when we hit fields that have dots in their name (e.g. interarction, metaData['simulation.id']) then this will fail
             // decided to skip these cases for now entirely
             return;
         }
-        let originHelper;
-
         // revert back placeholder to dots
-        if (origin) {
-            originHelper = origin + '.' + fieldPath;
-        } else {
-            originHelper = fieldPath;
-        }
+        const originHelper = origin ? origin + '.' + fieldPath : fieldPath;
 
-        if (
-            this.definition.fields[originHelper] &&
-            this.definition.fields[originHelper].skipValidation
-        ) {
+        if (this.definition.fields[originHelper]?.skipValidation) {
             // skip if current field should not be validated
             return;
         } else if (
             Array.isArray(fieldContent) &&
-            this.definition.fields[originHelper] &&
-            this.definition.fields[originHelper][definitionProperty] === true
+            this.definition.fields[originHelper]?.[definitionProperty] === true
         ) {
             for (const subObject of fieldContent) {
                 // for simple arrays skip, only process object or array arrays further
@@ -823,7 +888,8 @@ class MetadataType {
     }
     /**
      * Remove fields from metadata entry that are not createable
-     * @param {MetadataTypeItem} metadataEntry metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static removeNotCreateableFields(metadataEntry) {
@@ -834,7 +900,8 @@ class MetadataType {
 
     /**
      * Remove fields from metadata entry that are not updateable
-     * @param {MetadataTypeItem} metadataEntry metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static removeNotUpdateableFields(metadataEntry) {
@@ -845,7 +912,8 @@ class MetadataType {
 
     /**
      * Remove fields from metadata entry that are not needed in the template
-     * @param {MetadataTypeItem} metadataEntry metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static keepTemplateFields(metadataEntry) {
@@ -856,7 +924,8 @@ class MetadataType {
 
     /**
      * Remove fields from metadata entry that are not needed in the stored metadata
-     * @param {MetadataTypeItem} metadataEntry metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static keepRetrieveFields(metadataEntry) {
@@ -867,8 +936,9 @@ class MetadataType {
 
     /**
      * checks if the current metadata entry should be saved on retrieve or not
+     *
      * @static
-     * @param {MetadataTypeItem} metadataEntry metadata entry
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
      * @param {boolean} [include=false] true: use definition.include / options.include; false=exclude: use definition.filter / options.exclude
      * @returns {boolean} true: skip saving == filtered; false: continue with saving
      * @memberof MetadataType
@@ -911,14 +981,15 @@ class MetadataType {
     }
     /**
      * optionally filter by what folder something is in
+     *
      * @static
-     * @param {Object} metadataEntry metadata entry
+     * @param {object} metadataEntry metadata entry
      * @param {boolean} [include=false] true: use definition.include / options.include; false=exclude: use definition.filter / options.exclude
      * @returns {boolean} true: filtered == do NOT save; false: not filtered == do save
      * @memberof MetadataType
      */
     static isFilteredFolder(metadataEntry, include) {
-        if (metadataEntry.json && metadataEntry.json.r__folder_Path) {
+        if (metadataEntry.json?.r__folder_Path) {
             // r__folder_Path found in sub-object
             metadataEntry = metadataEntry.json;
         } else if (!metadataEntry.r__folder_Path) {
@@ -979,8 +1050,9 @@ class MetadataType {
     }
     /**
      * internal helper
+     *
      * @private
-     * @param {Object} myFilter include/exclude filter object
+     * @param {object} myFilter include/exclude filter object
      * @param {string} r__folder_Path already determined folder path
      * @returns {?boolean} true: filter value found; false: filter value not found; null: no filter defined
      */
@@ -1008,9 +1080,10 @@ class MetadataType {
     }
     /**
      * internal helper
+     *
      * @private
-     * @param {Object} myFilter include/exclude filter object
-     * @param {Object} metadataEntry metadata entry
+     * @param {object} myFilter include/exclude filter object
+     * @param {object} metadataEntry metadata entry
      * @returns {?boolean} true: filter value found; false: filter value not found; null: no filter defined
      */
     static _filterOther(myFilter, metadataEntry) {
@@ -1037,69 +1110,59 @@ class MetadataType {
         return false;
     }
 
-    /**
-     * Paginates a URL
-     * @param {string} url url of the request
-     * @param {number} last Number of the page of the last request
-     * @returns {string} new url with pagination
-     */
-    static paginate(url, last) {
-        if (this.definition.restPagination) {
-            const baseUrl = url.split('?')[0];
-            const queryParams = new URLSearchParams(url.split('?')[1]);
-            // if no page add page
-            if (!queryParams.has('$page')) {
-                queryParams.append('$page', (1).toString());
-            }
-            // if there is a page and a last value, then add to it.
-            else if (queryParams.has('$page') && last) {
-                queryParams.set('$page', (Number(last) + 1).toString());
-            }
-            return baseUrl + '?' + decodeURIComponent(queryParams.toString());
-        } else {
-            return url;
-        }
-    }
     /**
      * Helper for writing Metadata to disk, used for Retrieve and deploy
-     * @param {MetadataTypeMap} results metadata results from deploy
+     *
+     * @param {TYPE.MetadataTypeMap} results metadata results from deploy
      * @param {string} retrieveDir directory where metadata should be stored after deploy/retrieve
      * @param {string} [overrideType] for use when there is a subtype (such as folder-queries)
-     * @param {Util.TemplateMap} [templateVariables] variables to be replaced in the metadata
-     * @returns {Promise<MetadataTypeMap>} Promise of saved metadata
+     * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
+     * @returns {Promise.<TYPE.MetadataTypeMap>} Promise of saved metadata
      */
     static async saveResults(results, retrieveDir, overrideType, templateVariables) {
         const savedResults = {};
-        const subtypeExtension = '.' + (overrideType || this.definition.type) + '-meta';
         let filterCounter = 0;
-        for (const metadataEntry in results) {
+        let subtypeExtension;
+        for (const originalKey in results) {
+            if (this.definition.type === 'asset') {
+                overrideType =
+                    this.definition.type +
+                    '-' +
+                    Object.keys(this.definition.extendedSubTypes).find((type) =>
+                        this.definition.extendedSubTypes[type].includes(
+                            results[originalKey].assetType.name
+                        )
+                    );
+            }
+            subtypeExtension = '.' + (overrideType || this.definition.type) + '-meta';
+
             try {
                 if (
-                    this.isFiltered(results[metadataEntry], true) ||
-                    this.isFiltered(results[metadataEntry], false)
+                    this.isFiltered(results[originalKey], true) ||
+                    this.isFiltered(results[originalKey], false)
                 ) {
                     // if current metadata entry is filtered don't save it
                     filterCounter++;
                     continue;
                 }
+
                 // define directory into which the current metdata shall be saved
                 const baseDir = [retrieveDir, ...(overrideType || this.definition.type).split('-')];
-
-                results[metadataEntry] = await this.postRetrieveTasks(
-                    results[metadataEntry],
+                results[originalKey] = await this.postRetrieveTasks(
+                    results[originalKey],
                     retrieveDir,
                     templateVariables ? true : false
                 );
-                if (!results[metadataEntry] || results[metadataEntry] === null) {
+                if (!results[originalKey] || results[originalKey] === null) {
                     // we encountered a situation in our postRetrieveTasks that made us want to filter this record
-                    delete results[metadataEntry];
+                    delete results[originalKey];
                     filterCounter++;
                     continue;
                 }
 
                 if (
-                    this.isFilteredFolder(results[metadataEntry], true) ||
-                    this.isFilteredFolder(results[metadataEntry], false)
+                    this.isFilteredFolder(results[originalKey], true) ||
+                    this.isFilteredFolder(results[originalKey], false)
                 ) {
                     // if current metadata entry is filtered don't save it
                     filterCounter++;
@@ -1107,20 +1170,20 @@ class MetadataType {
                 }
 
                 // for complex types like asset, script, query we need to save the scripts that were extracted from the JSON
-                if (results[metadataEntry].json && results[metadataEntry].codeArr) {
+                if (results[originalKey].json && results[originalKey].codeArr) {
                     // replace market values with template variable placeholders (do not do it on .codeArr)
                     if (templateVariables) {
-                        results[metadataEntry].json = Util.replaceByObject(
-                            results[metadataEntry].json,
+                        results[originalKey].json = Util.replaceByObject(
+                            results[originalKey].json,
                             templateVariables
                         );
-                        results[metadataEntry].subFolder = Util.replaceByObject(
-                            results[metadataEntry].subFolder,
+                        results[originalKey].subFolder = Util.replaceByObject(
+                            results[originalKey].subFolder,
                             templateVariables
                         );
                     }
 
-                    const postRetrieveData = results[metadataEntry];
+                    const postRetrieveData = results[originalKey];
                     if (postRetrieveData.subFolder) {
                         // very complex types have their own subfolder
                         baseDir.push(...postRetrieveData.subFolder);
@@ -1141,13 +1204,13 @@ class MetadataType {
                         );
                     }
                     // normalize results[metadataEntry]
-                    results[metadataEntry] = postRetrieveData.json;
+                    results[originalKey] = postRetrieveData.json;
                 } else {
                     // not a complex type, run the the entire JSON through templating
                     // replace market values with template variable placeholders
                     if (templateVariables) {
-                        results[metadataEntry] = Util.replaceByObject(
-                            results[metadataEntry],
+                        results[originalKey] = Util.replaceByObject(
+                            results[originalKey],
                             templateVariables
                         );
                     }
@@ -1156,7 +1219,7 @@ class MetadataType {
                 // we dont store Id on local disk, but we need it for caching logic,
                 // so its in retrieve but not in save. Here we put into the clone so that the original
                 // object used for caching doesnt have the Id removed.
-                const saveClone = JSON.parse(JSON.stringify(results[metadataEntry]));
+                const saveClone = JSON.parse(JSON.stringify(results[originalKey]));
                 if (!this.definition.keepId) {
                     delete saveClone[this.definition.idField];
                 }
@@ -1166,45 +1229,71 @@ class MetadataType {
                 } else {
                     this.keepRetrieveFields(saveClone);
                 }
-                savedResults[metadataEntry] = saveClone;
-                File.writeJSONToFile(
+                savedResults[originalKey] = saveClone;
+                await File.writeJSONToFile(
                     // manage subtypes
                     baseDir,
-                    saveClone[this.definition.keyField] + subtypeExtension,
+                    originalKey + subtypeExtension,
                     saveClone
                 );
+                if (templateVariables) {
+                    Util.logger.info(
+                        `- templated ${this.definition.type}: ${
+                            saveClone[this.definition.nameField]
+                        }`
+                    );
+                }
             } catch (ex) {
-                console.log(ex.stack);
-                Util.metadataLogger(
-                    'error',
-                    this.definition.type,
-                    'saveResults',
+                Util.logger.errorStack(
                     ex,
-                    metadataEntry
+                    ` - Saving ${this.definition.type} ${originalKey} failed`
                 );
             }
         }
-        if (filterCounter) {
-            if (this.definition.type !== 'asset') {
-                // interferes with progress bar in assets and is printed 1-by-1 otherwise
-                Util.logger.info(
-                    `Filtered ${this.definition.type}: ${filterCounter} (downloaded but not saved to disk)`
-                );
-            }
+        if (filterCounter && this.definition.type !== 'asset') {
+            // interferes with progress bar in assets and is printed 1-by-1 otherwise
+            Util.logger.info(
+                ` - Filtered ${this.definition.type}: ${filterCounter} (downloaded but not saved to disk)`
+            );
         }
         return savedResults;
     }
+    /**
+     * helper for buildDefinitionForNested
+     * searches extracted file for template variable names and applies the market values
+     *
+     * @param {string} code code from extracted code
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {string} code with markets applied
+     */
+    static applyTemplateValues(code, templateVariables) {
+        // replace template variables with their values
+        return Mustache.render(code, templateVariables);
+    }
+    /**
+     * helper for buildTemplateForNested
+     * searches extracted file for template variable values and applies the market variable names
+     *
+     * @param {string} code code from extracted code
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {string} code with markets applied
+     */
+    static applyTemplateNames(code, templateVariables) {
+        // replace template variables with their values
+        return Util.replaceByObject(code, templateVariables);
+    }
     /**
      * helper for buildDefinition
      * handles extracted code if any are found for complex types (e.g script, asset, query)
+     *
      * @param {string} templateDir Directory where metadata templates are stored
      * @param {string} targetDir Directory where built definitions will be saved
-     * @param {MetadataTypeItem} metadata main JSON file that was read from file system
-     * @param {Util.TemplateMap} variables variables to be replaced in the metadata
+     * @param {TYPE.MetadataTypeItem} metadata main JSON file that was read from file system
+     * @param {TYPE.TemplateMap} variables variables to be replaced in the metadata
      * @param {string} templateName name of the template to be built
-     * @returns {Promise<void>} Promise
+     * @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
      */
-    static async buildDefinitionForExtracts(
+    static async buildDefinitionForNested(
         templateDir,
         targetDir,
         metadata,
@@ -1214,23 +1303,46 @@ class MetadataType {
         // generic version here does nothing. actual cases handled in type classes
         return null;
     }
+    /**
+     * helper for buildTemplate
+     * handles extracted code if any are found for complex types
+     *
+     * @param {string} templateDir Directory where metadata templates are stored
+     * @param {string|string[]} targetDir (List of) Directory where built definitions will be saved
+     * @param {TYPE.MetadataTypeItem} metadata main JSON file that was read from file system
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {string} templateName name of the template to be built
+     * @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
+     */
+    static buildTemplateForNested(
+        templateDir,
+        targetDir,
+        metadata,
+        templateVariables,
+        templateName
+    ) {
+        // generic version here does nothing. actual cases handled in type classes
+        return null;
+    }
     /**
      * check template directory for complex types that open subfolders for their subtypes
+     *
      * @param {string} templateDir Directory where metadata templates are stored
      * @param {string} templateName name of the metadata file
-     * @returns {string} subtype name
+     * @returns {Promise.<string>} subtype name
      */
-    static findSubType(templateDir, templateName) {
+    static async findSubType(templateDir, templateName) {
         return null;
     }
     /**
      * optional method used for some types to try a different folder structure
+     *
      * @param {string} templateDir Directory where metadata templates are stored
      * @param {string[]} typeDirArr current subdir for this type
      * @param {string} templateName name of the metadata template
      * @param {string} fileName name of the metadata template file w/o extension
      * @param {Error} ex error from first attempt
-     * @returns {Object} metadata
+     * @returns {object} metadata
      */
     static async readSecondaryFolder(templateDir, typeDirArr, templateName, fileName, ex) {
         // we just want to push the method into the catch here
@@ -1240,17 +1352,18 @@ class MetadataType {
      * Builds definition based on template
      * NOTE: Most metadata files should use this generic method, unless custom
      * parsing is required (for example scripts & queries)
+     *
      * @param {string} templateDir Directory where metadata templates are stored
-     * @param {String|String[]} targetDir (List of) Directory where built definitions will be saved
+     * @param {string | string[]} targetDir (List of) Directory where built definitions will be saved
      * @param {string} templateName name of the metadata file
-     * @param {Util.TemplateMap} variables variables to be replaced in the metadata
-     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} Promise of item map
+     * @param {TYPE.TemplateMap} variables variables to be replaced in the metadata
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of item map
      */
     static async buildDefinition(templateDir, targetDir, templateName, variables) {
         // retrieve metadata template
         let metadataStr;
         let typeDirArr = [this.definition.type];
-        const subType = this.findSubType(templateDir, templateName);
+        const subType = await this.findSubType(templateDir, templateName);
         if (subType) {
             typeDirArr.push(subType);
         }
@@ -1259,7 +1372,11 @@ class MetadataType {
         try {
             // ! do not load via readJSONFile to ensure we get a string, not parsed JSON
             // templated files might contain illegal json before the conversion back to the file that shall be saved
-            metadataStr = await File.readFile([templateDir, ...typeDirArr], fileName, 'json');
+            metadataStr = await File.readFilteredFilename(
+                [templateDir, ...typeDirArr],
+                fileName,
+                'json'
+            );
         } catch (ex) {
             try {
                 metadataStr = await this.readSecondaryFolder(
@@ -1269,7 +1386,7 @@ class MetadataType {
                     fileName,
                     ex
                 );
-            } catch (ex) {
+            } catch {
                 throw new Error(
                     `${this.definition.type}:: Could not find ./${File.normalizePath([
                         templateDir,
@@ -1286,7 +1403,7 @@ class MetadataType {
             // update all initial variables & create metadata object
             metadata = JSON.parse(Mustache.render(metadataStr, variables));
             typeDirArr = typeDirArr.map((el) => Mustache.render(el, variables));
-        } catch (ex) {
+        } catch {
             throw new Error(
                 `${this.definition.type}:: Error applying template variables on ${
                     templateName + '.' + this.definition.type
@@ -1296,8 +1413,8 @@ class MetadataType {
 
         // handle extracted code
         // run after metadata was templated and converted into JS-object
-        // templating to extracted content is applied inside of buildDefinitionForExtracts()
-        await this.buildDefinitionForExtracts(
+        // templating to extracted content is applied inside of buildDefinitionForNested()
+        await this.buildDefinitionForNested(
             templateDir,
             targetDir,
             metadata,
@@ -1307,24 +1424,19 @@ class MetadataType {
 
         try {
             // write to file
-            let targetDirArr;
-            if (!Array.isArray(targetDir)) {
-                targetDirArr = [targetDir];
-            } else {
-                targetDirArr = targetDir;
-            }
+            const targetDirArr = !Array.isArray(targetDir) ? [targetDir] : targetDir;
+
             for (const targetDir of targetDirArr) {
-                File.writeJSONToFile(
+                await File.writeJSONToFile(
                     [targetDir, ...typeDirArr],
                     metadata[this.definition.keyField] + '.' + this.definition.type + suffix,
                     metadata
                 );
             }
             Util.logger.info(
-                'MetadataType[' +
-                    this.definition.type +
-                    '].buildDefinition:: Complete - ' +
+                `- prepared deployment definition of ${this.definition.type}: ${
                     metadata[this.definition.keyField]
+                }`
             );
 
             return { metadata: metadata, type: this.definition.type };
@@ -1333,38 +1445,48 @@ class MetadataType {
         }
     }
     /**
+     *  Standardizes a check for multiple messages
      *
-     * @param {Object} response response payload from REST API
-     * @returns {void}
+     * @param {object} ex response payload from REST API
+     * @returns {string[]} formatted Error Message
      */
-    static checkForErrors(response) {
-        if (response && response.res.statusCode >= 400 && response.res.statusCode < 600) {
+    static checkForErrors(ex) {
+        if (ex?.response?.status >= 400 && ex?.response?.status < 600) {
             const errors = [];
-            if (response.body.errors) {
-                for (const errMsg of response.body.errors) {
-                    errors.push(errMsg.message.split('<br />').join(''));
+            if (ex.response.data.errors) {
+                for (const errMsg of ex.response.data.errors) {
+                    errors.push(
+                        ...errMsg.message
+                            .split('<br />')
+                            .map((el) => el.trim())
+                            .filter(Boolean)
+                    );
                 }
-            } else if (response.body.validationErrors) {
-                for (const errMsg of response.body.validationErrors) {
-                    errors.push(errMsg.message.split('<br />').join(''));
+            } else if (ex.response.data.validationErrors) {
+                for (const errMsg of ex.response.data.validationErrors) {
+                    errors.push(
+                        ...errMsg.message
+                            .split('<br />')
+                            .map((el) => el.trim())
+                            .filter(Boolean)
+                    );
                 }
-            } else if (response.body.message) {
-                errors.push(response.body.message);
+            } else if (ex.response.data.message) {
+                errors.push(ex.response.data.message);
             } else {
-                errors.push(`Undefined Errors: ${JSON.stringify(response.body)}`);
+                errors.push(`Undefined Errors: ${JSON.stringify(ex.response.data)}`);
             }
-            throw new Error(
-                `Errors on upserting metadata at ${response.res.request.path}: ${errors.join(
-                    '<br />'
-                )}`
-            );
+            Util.logger.debug(JSON.stringify(ex.config));
+            Util.logger.debug(JSON.stringify(ex.response.data));
+            return errors;
         }
     }
 
     /**
      * Gets metadata cache with limited fields and does not store value to disk
-     * @param {Util.BuObject} [buObject] properties for auth
-     * @param {MetadataTypeMap} [metadata] a list of type definitions
+     *
+     * @param {TYPE.BuObject} [buObject] properties for auth
+     * @param {TYPE.MetadataTypeMap} [metadata] a list of type definitions
      * @param {boolean} [isDeploy] used to skip non-supported message during deploy
      * @returns {void}
      */
@@ -1375,26 +1497,86 @@ class MetadataType {
     }
 
     /**
-     * Delete a data extension from the specified business unit
-     * @param {Util.BuObject} buObject references credentials
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {TYPE.BuObject} buObject references credentials
      * @param {string} customerKey Identifier of data extension
-     * @returns {void} -
+     * @returns {boolean} deletion success status
      */
     static deleteByKey(buObject, customerKey) {
-        Util.logger.error(`Deleting type ${this.definition.type} is not supported.`);
+        Util.logger.error(`Deletion is not yet supported for ${this.definition.typeName}!`);
+        return false;
+    }
+    /**
+     * clean up after deleting a metadata item
+     *
+     * @param {TYPE.BuObject} buObject references credentials
+     * @param {string} customerKey Identifier of metadata item
+     * @returns {void}
+     */
+    static async postDeleteTasks(buObject, customerKey) {
+        // delete local copy: retrieve/cred/bu/type/...json
+        const jsonFile = File.normalizePath([
+            this.properties.directories.retrieve,
+            buObject.credential,
+            buObject.businessUnit,
+            this.definition.type,
+            `${customerKey}.${this.definition.type}-meta.json`,
+        ]);
+        await File.remove(jsonFile);
+    }
+
+    /**
+     * Delete a data extension from the specified business unit
+     *
+     * @param {TYPE.BuObject} buObject references credentials
+     * @param {string} customerKey Identifier of metadata
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
+     * @returns {boolean} deletion success flag
+     */
+    static async deleteByKeySOAP(buObject, customerKey, handleOutside) {
+        const keyObj = {};
+        keyObj[this.definition.keyField] = customerKey;
+        try {
+            this.client.soap.delete(
+                this.definition.type.charAt(0).toUpperCase() + this.definition.type.slice(1),
+                keyObj,
+                null
+            );
+            if (!handleOutside) {
+                Util.logger.info(`- deleted ${this.definition.type}: ${customerKey}`);
+            }
+            this.postDeleteTasks(buObject, customerKey);
+
+            return true;
+        } catch (ex) {
+            if (!handleOutside) {
+                const errorMsg = ex?.results?.length
+                    ? `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`
+                    : ex.message;
+                Util.logger.error(
+                    `- error deleting ${this.definition.type} '${customerKey}': ${errorMsg}`
+                );
+            } else {
+                throw ex;
+            }
+
+            return false;
+        }
     }
     /**
      * Returns metadata of a business unit that is saved locally
+     *
      * @param {string} readDir root directory of metadata.
      * @param {boolean} [listBadKeys=false] do not print errors, used for badKeys()
-     * @param {Object} [buMetadata] Metadata of BU in local directory
-     * @returns {Object} Metadata of BU in local directory
+     * @param {object} [buMetadata] Metadata of BU in local directory
+     * @returns {object} Metadata of BU in local directory
      */
     static readBUMetadataForType(readDir, listBadKeys, buMetadata) {
         buMetadata = buMetadata || {};
         readDir = File.normalizePath([readDir, this.definition.type]);
         try {
-            if (File.existsSync(readDir)) {
+            if (File.pathExistsSync(readDir)) {
                 // check if folder name is a valid metadataType, then check if the user limited to a certain type in the command params
                 buMetadata[this.definition.type] = this.getJsonFromFS(readDir, listBadKeys);
                 return buMetadata;
@@ -1405,6 +1587,25 @@ class MetadataType {
             throw new Error(ex.message);
         }
     }
+    /**
+     * should return only the json for all but asset, query and script that are saved as multiple files
+     * additionally, the documentation for dataExtension and automation should be returned
+     *
+     * @param {string[]} keyArr customerkey of the metadata
+     * @returns {Promise.<string[]>} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
+     */
+    static getFilesToCommit(keyArr) {
+        const typeExtension = '.' + this.definition.type + '-meta.json';
+        const path = File.normalizePath([
+            this.properties.directories.retrieve,
+            this.buObject.credential,
+            this.buObject.businessUnit,
+            this.definition.type,
+        ]);
+
+        const fileList = keyArr.map((key) => File.normalizePath([path, key + typeExtension]));
+        return fileList;
+    }
 }
 
 MetadataType.definition = {
@@ -1418,13 +1619,20 @@ MetadataType.definition = {
     type: '',
 };
 /**
- * @type {Util.ET_Client}
+ * @type {TYPE.SDK}
  */
 MetadataType.client = undefined;
 /**
- * @type {MultiMetadataTypeMap}
+ * @type {TYPE.Mcdevrc}
  */
-MetadataType.cache = {};
 MetadataType.properties = null;
+/**
+ * @type {string}
+ */
+MetadataType.subType = null;
+/**
+ * @type {TYPE.BuObject}
+ */
+MetadataType.buObject = null;
 
 module.exports = MetadataType;