mcdev 6.0.2 → 7.0.0

package/lib/metadataTypes/MetadataType.js

@@ -7,14 +7,31 @@
  * in the generic version of the method
  */
 
-import TYPE from '../../types/mcdev.d.js';
-
 import { Util } from '../util/util.js';
 import File from '../util/file.js';
 import cache from '../util/cache.js';
 import deepEqual from 'deep-equal';
 import pLimit from 'p-limit';
 import Mustache from 'mustache';
+
+/**
+ * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
+ * @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
+ * @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
+ * @typedef {import('../../types/mcdev.d.js').Mcdevrc} Mcdevrc
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
+ * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
+ * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').SDK} SDK
+ * @typedef {import('../../types/mcdev.d.js').SDKError} SDKError
+ * @typedef {import('../../types/mcdev.d.js').SOAPError} SOAPError
+ * @typedef {import('../../types/mcdev.d.js').RestError} RestError
+ */
+
 /**
  * ensure that Mustache does not escape any characters
  *
@@ -36,16 +53,17 @@ class MetadataType {
      *
      * @param {string} dir directory that contains '.json' files to be read
      * @param {boolean} [listBadKeys] do not print errors, used for badKeys()
-     * @returns {TYPE.MetadataTypeMap} fileName => fileContent map
+     * @param {string[]} [selectedSubType] asset, message, ...
+     * @returns {Promise.<MetadataTypeMap>} fileName => fileContent map
      */
-    static getJsonFromFS(dir, listBadKeys) {
+    static async getJsonFromFS(dir, listBadKeys, selectedSubType) {
         const fileName2FileContent = {};
         try {
-            const files = File.readdirSync(dir);
+            const files = await File.readdir(dir);
             for (const fileName of files) {
                 try {
                     if (fileName.endsWith('.json')) {
-                        const fileContent = File.readJSONFile(dir, fileName, true, false);
+                        const fileContent = await File.readJSONFile(dir, fileName, false);
                         const fileNameWithoutEnding = File.reverseFilterIllegalFilenames(
                             fileName.split(/\.(\w|-)+-meta.json/)[0]
                         );
@@ -105,10 +123,10 @@ class MetadataType {
     /**
      * Deploys metadata
      *
-     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
+     * @param {MetadataTypeMap} metadata metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
      * @param {string} retrieveDir directory where metadata after deploy should be saved
-     * @returns {Promise.<TYPE.MetadataTypeMap>} Promise of keyField => metadata map
+     * @returns {Promise.<MetadataTypeMap>} Promise of keyField => metadata map
      */
     static async deploy(metadata, deployDir, retrieveDir) {
         const upsertResults = await this.upsert(metadata, deployDir);
@@ -127,8 +145,8 @@ class MetadataType {
     /**
      * Gets executed after deployment of metadata type
      *
-     * @param {TYPE.MetadataTypeMap} upsertResults metadata mapped by their keyField as returned by update/create
-     * @param {TYPE.MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {MetadataTypeMap} upsertResults metadata mapped by their keyField as returned by update/create
+     * @param {MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
      * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
      * @returns {void}
      */
@@ -137,9 +155,9 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.createREST}
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {object} apiResponse varies depending on the API call
-     * @param {TYPE.MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
+     * @param {MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
      * @returns {void}
      */
     static postCreateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {}
@@ -147,7 +165,7 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.updateREST}
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {object} apiResponse varies depending on the API call
      * @returns {void}
      */
@@ -156,7 +174,7 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.createREST} when legacy API endpoints as these do not return the created item but only their new id
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {object} apiResponse varies depending on the API call
      * @returns {Promise.<void>} -
      */
@@ -185,18 +203,18 @@ class MetadataType {
     /**
      * Gets executed after retreive of metadata type
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single item
+     * @param {MetadataTypeItem} metadata a single item
      * @param {string} targetDir folder where retrieves should be saved
      * @param {boolean} [isTemplating] signals that we are retrieving templates
-     * @returns {TYPE.MetadataTypeItem} cloned metadata
+     * @returns {MetadataTypeItem} cloned metadata
      */
     static postRetrieveTasks(metadata, targetDir, isTemplating) {
-        return JSON.parse(JSON.stringify(metadata));
+        return structuredClone(metadata);
     }
     /**
      * generic script that retrieves the folder path from cache and updates the given metadata with it after retrieve
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single item
+     * @param {MetadataTypeItem} metadata a single item
      */
     static setFolderPath(metadata) {
         if (!this.definition.folderIdField) {
@@ -221,7 +239,7 @@ class MetadataType {
     /**
      * generic script that retrieves the folder ID from cache and updates the given metadata with it before deploy
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single item
+     * @param {MetadataTypeItem} metadata a single item
      */
     static setFolderId(metadata) {
         if (!this.definition.folderIdField) {
@@ -243,19 +261,19 @@ class MetadataType {
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {string[]} [subTypeArr] optionally limit to a single subtype
      * @param {string} [key] customer key of single item to retrieve
-     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
+     * @returns {Promise.<MetadataTypeMapObj>} metadata
      */
-    static retrieve(retrieveDir, additionalFields, subTypeArr, key) {
+    static async retrieve(retrieveDir, additionalFields, subTypeArr, key) {
         Util.metadataLogger('error', this.definition.type, 'retrieve', `Not Supported`);
         const metadata = {};
-        return { metadata: null, type: this.definition.type };
+        return { metadata: {}, type: this.definition.type };
     }
     /**
      * Gets metadata from Marketing Cloud
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {string[]} [subTypeArr] optionally limit to a single subtype
-     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
+     * @returns {Promise.<MetadataTypeMapObj>} metadata
      */
     static retrieveChangelog(additionalFields, subTypeArr) {
         return this.retrieveForCache(additionalFields, subTypeArr);
@@ -266,21 +284,22 @@ class MetadataType {
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {string[]} [subTypeArr] optionally limit to a single subtype
-     * @returns {Promise.<TYPE.MetadataTypeMapObj>} metadata
+     * @param {string} [key] customer key of single item to retrieve
+     * @returns {Promise.<MetadataTypeMapObj>} metadata
      */
-    static async retrieveForCache(additionalFields, subTypeArr) {
-        return this.retrieve(null, additionalFields, subTypeArr);
+    static async retrieveForCache(additionalFields, subTypeArr, key) {
+        return this.retrieve(null, additionalFields, subTypeArr, key);
     }
     /**
      * Gets metadata cache with limited fields and does not store value to disk
      *
      * @param {string} templateDir Directory where retrieved metadata directory will be saved
      * @param {string} name name of the metadata file
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
      * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise.<TYPE.MetadataTypeItemObj>} metadata
+     * @returns {Promise.<MetadataTypeItemObj>} metadata
      */
-    static retrieveAsTemplate(templateDir, name, templateVariables, subType) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables, subType) {
         Util.logger.error('retrieveAsTemplate is not supported yet for ' + this.definition.type);
         Util.metadataLogger(
             'debug',
@@ -295,9 +314,9 @@ class MetadataType {
      *
      * @param {string} templateDir Directory where retrieved metadata directory will be saved
      * @param {string} uri rest endpoint for GET
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
      * @param {string} name name (not key) of the metadata item
-     * @returns {Promise.<{metadata: TYPE.MetadataTypeItem, type: string}>} Promise
+     * @returns {Promise.<{metadata: MetadataTypeItem, type: string}>} Promise
      */
     static async retrieveTemplateREST(templateDir, uri, templateVariables, name) {
         return this.retrieveREST(templateDir, uri, templateVariables, name);
@@ -309,8 +328,8 @@ class MetadataType {
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string} templateDir (List of) Directory where built definitions will be saved
      * @param {string} key name of the metadata file
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
-     * @returns {Promise.<TYPE.MetadataTypeItemObj>} single metadata
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {Promise.<MetadataTypeItemObj>} single metadata
      */
     static async buildTemplate(retrieveDir, templateDir, key, templateVariables) {
         // retrieve metadata template
@@ -330,6 +349,9 @@ class MetadataType {
                 fileName,
                 'json'
             );
+            if (!metadataStr) {
+                throw new Error('File not found');
+            }
         } catch (ex) {
             try {
                 metadataStr = await this.readSecondaryFolder(
@@ -340,16 +362,17 @@ class MetadataType {
                     ex
                 );
             } catch {
-                throw new Error(
-                    `${this.definition.type}:: Could not find ./${File.normalizePath([
-                        retrieveDir,
-                        ...typeDirArr,
-                        fileName + '.json',
-                    ])}.`
+                // only happening for types that use readSecondaryFolder (e.g. asset)
+                // if we still have no metadataStr then we have to skip this metadata for all types and hence handle it outside of this catch
+            }
+            if (!metadataStr) {
+                Util.logger.warn(
+                    Util.getGrayMsg(`- skipped ${this.definition.type} ${key}: not found`)
                 );
+                return;
             }
-            // return;
         }
+
         if (this.definition.stringifyFieldsBeforeTemplate) {
             // numeric fields are returned as numbers by the SDK/API. If we try to replace them in buildTemplate it would break the JSON format - but not if we stringify them first because then the {{{var}}} is in quotes
             metadataStr = JSON.parse(metadataStr);
@@ -370,6 +393,12 @@ class MetadataType {
             }
             metadataStr = JSON.stringify(metadataStr);
         }
+        if (this.definition.type === 'asset') {
+            // turn memberId integer to string to ensure it does not get replaced with a templateVariable
+            const temp = JSON.parse(metadataStr);
+            temp.memberId = temp.memberId.toString();
+            metadataStr = JSON.stringify(temp);
+        }
         const metadata = JSON.parse(Util.replaceByObject(metadataStr, templateVariables));
         this.keepTemplateFields(metadata);
 
@@ -401,9 +430,9 @@ class MetadataType {
     /**
      * Gets executed before deploying metadata
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single metadata item
+     * @param {MetadataTypeItem} metadata a single metadata item
      * @param {string} deployDir folder where files for deployment are stored
-     * @returns {Promise.<TYPE.MetadataTypeItem>} Promise of a single metadata item
+     * @returns {Promise.<MetadataTypeItem>} Promise of a single metadata item
      */
     static async preDeployTasks(metadata, deployDir) {
         return metadata;
@@ -412,11 +441,11 @@ class MetadataType {
     /**
      * Abstract create method that needs to be implemented in child metadata type
      *
-     * @param {TYPE.MetadataTypeItem} metadata single metadata entry
+     * @param {MetadataTypeItem} metadata single metadata entry
      * @param {string} deployDir directory where deploy metadata are saved
-     * @returns {void}
+     * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static create(metadata, deployDir) {
+    static async create(metadata, deployDir) {
         Util.logger.error(
             ` ☇ skipping ${this.definition.type} ${metadata[this.definition.keyField]} / ${
                 metadata[this.definition.nameField]
@@ -428,11 +457,11 @@ class MetadataType {
     /**
      * Abstract update method that needs to be implemented in child metadata type
      *
-     * @param {TYPE.MetadataTypeItem} metadata single metadata entry
-     * @param {TYPE.MetadataTypeItem} [metadataBefore] metadata mapped by their keyField
-     * @returns {void}
+     * @param {MetadataTypeItem} metadata single metadata entry
+     * @param {MetadataTypeItem} [metadataBefore] metadata mapped by their keyField
+     * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static update(metadata, metadataBefore) {
+    static async update(metadata, metadataBefore) {
         Util.logger.error(
             ` ☇ skipping ${this.definition.type} ${metadata[this.definition.keyField]} / ${
                 metadata[this.definition.nameField]
@@ -452,12 +481,86 @@ class MetadataType {
         return;
     }
 
+    /**
+     * this iterates over all items found in the retrieve folder and executes the type-specific method for replacing references
+     *
+     * @param {MetadataTypeMap} metadataMap list of metadata (keyField => metadata)
+     * @param {string} retrieveDir retrieve dir including cred and bu
+     * @returns {Promise.<string[]>} Returns list of keys for which references were replaced
+     */
+    static async replaceCbReferenceLoop(metadataMap, retrieveDir) {
+        const keysForDeploy = [];
+        if (!metadataMap) {
+            // if a type was skipped e.g. because it shall only be looked at on the parent then we would expect metadataMap to be undefined
+            return keysForDeploy;
+        }
+
+        const fromDescription = Util.OPTIONS.referenceFrom
+            .map((from) => 'ContentBlockBy' + Util.capitalizeFirstLetter(from))
+            .join(' and ');
+
+        if (Object.keys(metadataMap).length) {
+            Util.logger.info(`Searching ${this.definition.type} for ${fromDescription}:`);
+            const baseDir = [retrieveDir, ...this.definition.type.split('-')];
+            const deployMap = {};
+
+            for (const key in metadataMap) {
+                const item = metadataMap[key];
+                if (this.isFiltered(item, true) || this.isFiltered(item, false)) {
+                    // we would not have saved these items to disk but they exist in the cache and hence need to be skipped here
+
+                    continue;
+                }
+
+                try {
+                    // add key but make sure to turn it into string or else numeric keys will be filtered later
+                    deployMap[key] = await this.replaceCbReference(item, retrieveDir);
+                    await this.saveToDisk(deployMap, key, baseDir);
+
+                    keysForDeploy.push(key + '');
+                    Util.logger.info(` - added ${this.definition.type} to update queue: ${key}`);
+                } catch (ex) {
+                    if (ex.code !== 200) {
+                        // dont print error if we simply did not find relevant content blocks
+                        Util.logger.errorStack(ex, ex.message);
+                    }
+                    Util.logger.info(
+                        Util.getGrayMsg(
+                            ` ☇ skipping ${this.definition.type} ${
+                                item[this.definition.keyField]
+                            }: no ${fromDescription} found`
+                        )
+                    );
+                }
+            }
+
+            Util.logger.info(
+                `Found ${keysForDeploy.length} ${this.definition.type}${keysForDeploy.length === 1 ? '' : 's'} to update`
+            );
+        }
+        return keysForDeploy;
+    }
     /**
      * Abstract execute method that needs to be implemented in child metadata type
      *
-     * @returns {void}
+     * @param {MetadataTypeItem} item single metadata item
+     * @param {string} [retrieveDir] directory where metadata is saved
+     * @returns {Promise.<MetadataTypeItem | CodeExtractItem>} key of the item that was updated
+     */
+    static async replaceCbReference(item, retrieveDir) {
+        Util.logger.error(
+            ` ☇ skipping ${this.definition.type}: replaceCbReference is not supported yet for ${this.definition.type}`
+        );
+        return [];
+    }
+
+    /**
+     * Abstract execute method that needs to be implemented in child metadata type
+     *
+     * @param {string[]} keyArr customerkey of the metadata
+     * @returns {Promise.<string[]>} Returns list of keys that were executed
      */
-    static execute() {
+    static async execute(keyArr) {
         Util.logger.error(
             ` ☇ skipping ${this.definition.type}: execute is not supported yet for ${this.definition.type}`
         );
@@ -466,9 +569,10 @@ class MetadataType {
     /**
      * Abstract pause method that needs to be implemented in child metadata type
      *
-     * @returns {void}
+     * @param {string[]} keyArr customerkey of the metadata
+     * @returns {Promise.<string[]>} Returns list of keys that were paused
      */
-    static pause() {
+    static async pause(keyArr) {
         Util.logger.error(
             ` ☇ skipping ${this.definition.type}: pause is not supported yet for ${this.definition.type}`
         );
@@ -478,8 +582,8 @@ class MetadataType {
     /**
      * test if metadata was actually changed or not to potentially skip it during deployment
      *
-     * @param {TYPE.MetadataTypeItem} cachedVersion cached version from the server
-     * @param {TYPE.MetadataTypeItem} metadata item to upload
+     * @param {MetadataTypeItem} cachedVersion cached version from the server
+     * @param {MetadataTypeItem} metadata item to upload
      * @param {string} [fieldName] optional field name to use for identifying the record in logs
      * @returns {boolean} true if metadata was changed
      */
@@ -490,8 +594,8 @@ class MetadataType {
     /**
      * test if metadata was actually changed or not to potentially skip it during deployment
      *
-     * @param {TYPE.MetadataTypeItem} cachedVersion cached version from the server
-     * @param {TYPE.MetadataTypeItem} metadata item to upload
+     * @param {MetadataTypeItem} cachedVersion cached version from the server
+     * @param {MetadataTypeItem} metadata item to upload
      * @param {string} [fieldName] optional field name to use for identifying the record in logs
      * @param {boolean} [silent] optionally suppress logging
      * @returns {boolean} true on first identified deviation or false if none are found
@@ -501,7 +605,7 @@ class MetadataType {
             return true;
         }
         // we do need the full set in other places and hence need to work with a clone here
-        const clonedMetada = JSON.parse(JSON.stringify(metadata));
+        const clonedMetada = structuredClone(metadata);
         this.removeNotUpdateableFields(clonedMetada);
         // iterate over what we want to upload rather than what we cached to avoid false positives
         for (const prop in clonedMetada) {
@@ -544,12 +648,12 @@ class MetadataType {
     /**
      * MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
      *
-     * @param {TYPE.MetadataTypeMap} metadataMap metadata mapped by their keyField
+     * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @param {string} deployDir directory where deploy metadata are saved
-     * @returns {Promise.<TYPE.MetadataTypeMap>} keyField => metadata map
+     * @returns {Promise.<MetadataTypeMap>} keyField => metadata map
      */
     static async upsert(metadataMap, deployDir) {
-        const orignalMetadataMap = JSON.parse(JSON.stringify(metadataMap));
+        const orignalMetadataMap = structuredClone(metadataMap);
         const metadataToUpdate = [];
         const metadataToCreate = [];
         let filteredByPreDeploy = 0;
@@ -624,7 +728,6 @@ class MetadataType {
             // if Results then parse as SOAP
             // put in Retrieve Format for parsing
             // todo add handling when response does not contain items.
-            // @ts-ignore
             const metadataResults = createResults
                 .concat(updateResults)
                 // TODO remove Object.keys check after create/update SOAP methods stop returning empty objects instead of null
@@ -636,7 +739,6 @@ class MetadataType {
             // likely comming from one of the many REST APIs
             // put in Retrieve Format for parsing
             // todo add handling when response does not contain items.
-            // @ts-ignore
             const metadataResults = createResults.concat(updateResults).filter(Boolean);
             upsertResults = this.parseResponseBody(metadataResults);
         }
@@ -650,19 +752,39 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.upsert}
      *
-     * @param {TYPE.MetadataTypeMap} metadataMap list of metadata
+     * @param {MetadataTypeMap} metadataMap list of metadata
      * @param {string} metadataKey key of item we are looking at
      * @param {boolean} hasError error flag from previous code
-     * @param {TYPE.MetadataTypeItemDiff[]} metadataToUpdate list of items to update
-     * @param {TYPE.MetadataTypeItem[]} metadataToCreate list of items to create
-     * @returns {'create' | 'update' | 'skip'} action to take
-     */
-    static createOrUpdate(metadataMap, metadataKey, hasError, metadataToUpdate, metadataToCreate) {
-        const normalizedKey = File.reverseFilterIllegalFilenames(
+     * @param {MetadataTypeItemDiff[]} metadataToUpdate list of items to update
+     * @param {MetadataTypeItem[]} metadataToCreate list of items to create
+     * @returns {Promise.<'create' | 'update' | 'skip'>} action to take
+     */
+    static async createOrUpdate(
+        metadataMap,
+        metadataKey,
+        hasError,
+        metadataToUpdate,
+        metadataToCreate
+    ) {
+        let normalizedKey = File.reverseFilterIllegalFilenames(
             metadataMap[metadataKey][this.definition.keyField]
         );
+
         // Update if it already exists; Create it if not
         const maxKeyLength = this.definition.maxKeyLength || 36;
+
+        // make sure keySuffix always has a string value
+        Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : '';
+        if (Util.OPTIONS.keySuffix && !cache.getByKey(this.definition.type, normalizedKey)) {
+            // to ensure we go into update mode like we would with appending the MID to asset-keys, recreated the normalized key here
+            const newKey = this.getNewKey(
+                this.definition.keyField,
+                metadataMap[metadataKey],
+                maxKeyLength
+            );
+
+            normalizedKey = File.reverseFilterIllegalFilenames(newKey);
+        }
         if (
             Util.logger.level === 'debug' &&
             metadataMap[metadataKey][this.definition.idField] &&
@@ -691,6 +813,7 @@ class MetadataType {
             if (!this.hasChanged(cachedVersion, metadataMap[metadataKey])) {
                 hasError = true;
             }
+
             if (Util.OPTIONS.changeKeyField) {
                 if (this.definition.keyField == this.definition.idField) {
                     Util.logger.error(
@@ -713,12 +836,18 @@ class MetadataType {
                         ` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.`
                     );
                 } else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
-                    // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
-                    const newKey = (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '')
-                        .trim()
-                        .slice(0, maxKeyLength)
-                        .trim();
-                    if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '' > maxKeyLength) {
+                    const newKey = this.getNewKey(
+                        Util.OPTIONS.changeKeyField,
+                        metadataMap[metadataKey],
+                        maxKeyLength
+                    );
+
+                    if (
+                        metadataMap[metadataKey][Util.OPTIONS.changeKeyField] +
+                            '' +
+                            Util.OPTIONS.keySuffix >
+                        maxKeyLength
+                    ) {
                         Util.logger.warn(
                             `${this.definition.type} ${this.definition.keyField} may not exceed ${maxKeyLength} characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}`
                         );
@@ -739,6 +868,9 @@ class MetadataType {
                     }
                 }
             } else if (Util.OPTIONS.changeKeyValue) {
+                if (Util.OPTIONS.keySuffix !== '') {
+                    Util.logger.warn(`Ignoring --keySuffix as --changeKeyValue is set.`);
+                }
                 // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
                 const newKey = Util.OPTIONS.changeKeyValue.trim().slice(0, maxKeyLength).trim();
                 if (Util.OPTIONS.changeKeyValue.trim().length > maxKeyLength) {
@@ -770,6 +902,21 @@ class MetadataType {
                     Util.changedKeysMap[this.definition.type] ||= {};
                     Util.changedKeysMap[this.definition.type][newKey] = metadataKey;
                 }
+            } else if (Util.OPTIONS.keySuffix && Util.OPTIONS.keySuffix !== '') {
+                // assume we simply want to append a suffix
+                const newKey = this.getNewKey(
+                    this.definition.keyField,
+                    metadataMap[metadataKey],
+                    maxKeyLength
+                );
+                Util.logger.info(
+                    ` - Changing ${this.definition.type} key from ${metadataKey} to ${newKey} via --keySuffix=${Util.OPTIONS.keySuffix}`
+                );
+                metadataMap[metadataKey][this.definition.keyField] = newKey;
+
+                // ensure we can delete the old file(s) after the successful update
+                Util.changedKeysMap[this.definition.type] ||= {};
+                Util.changedKeysMap[this.definition.type][newKey] = metadataKey;
             }
 
             if (hasError) {
@@ -794,6 +941,19 @@ class MetadataType {
                 metadataToCreate.push(null);
                 return 'skip';
             } else {
+                if (Util.OPTIONS.keySuffix && Util.OPTIONS.keySuffix !== '') {
+                    // assume we simply want to append a suffix
+                    const newKey = this.getNewKey(
+                        this.definition.keyField,
+                        metadataMap[metadataKey],
+                        maxKeyLength
+                    );
+                    Util.logger.info(
+                        ` - Changing ${this.definition.type} key from ${metadataKey} to ${newKey} via --keySuffix=${Util.OPTIONS.keySuffix}`
+                    );
+                    metadataMap[metadataKey][this.definition.keyField] = newKey;
+                }
+
                 metadataToCreate.push(metadataMap[metadataKey]);
                 return 'create';
             }
@@ -803,23 +963,26 @@ class MetadataType {
     /**
      * Creates a single metadata entry via REST
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {string} uri rest endpoint for POST
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
-    static async createREST(metadataEntry, uri) {
-        const metadataClone = JSON.parse(JSON.stringify(metadataEntry));
+    static async createREST(metadataEntry, uri, handleOutside) {
+        const metadataClone = structuredClone(metadataEntry);
         this.removeNotCreateableFields(metadataEntry);
         try {
             // set to empty object in case API returned nothing to be able to update it in helper classes
             const response = (await this.client.rest.post(uri, metadataEntry)) || {};
             await this.postCreateTasks(metadataEntry, response, metadataClone);
-            Util.logger.info(
-                ` - created ${this.definition.type}: ${
-                    metadataEntry[this.definition.keyField] ||
-                    metadataEntry[this.definition.nameField]
-                } / ${metadataEntry[this.definition.nameField]}`
-            );
+            if (!handleOutside) {
+                Util.logger.info(
+                    ` - created ${this.definition.type}: ${
+                        metadataEntry[this.definition.keyField] ||
+                        metadataEntry[this.definition.nameField]
+                    } / ${metadataEntry[this.definition.nameField]}`
+                );
+            }
             return response;
         } catch (ex) {
             const parsedErrors = this.getErrorsREST(ex);
@@ -843,7 +1006,7 @@ class MetadataType {
     /**
      * Creates a single metadata entry via fuel-soap (generic lib not wrapper)
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
+     * @param {MetadataTypeItem} metadataEntry single metadata entry
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
@@ -874,7 +1037,7 @@ class MetadataType {
     /**
      * Updates a single metadata entry via REST
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {string} uri rest endpoint for PATCH
      * @param {'patch'|'post'|'put'} [httpMethod] defaults to 'patch'; some update requests require PUT instead of PATCH
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
@@ -913,10 +1076,9 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP} that removes old files after the key was changed
      *
-     * @private
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
      * @param {boolean} [keepMap] some types require to check the old-key new-key relationship in their postDeployTasks; currently used by dataExtension only
-     * @returns {void}
+     * @returns {Promise.<void>} -
      */
     static async _postChangeKeyTasks(metadataEntry, keepMap = false) {
         if (
@@ -948,7 +1110,7 @@ class MetadataType {
     /**
      * Updates a single metadata entry via fuel-soap (generic lib not wrapper)
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
+     * @param {MetadataTypeItem} metadataEntry single metadata entry
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise.<object> | null} Promise of API response or null in case of an error
      */
@@ -977,9 +1139,9 @@ class MetadataType {
     }
     /**
      *
-     * @param {Error} ex error that occured
-     * @param {'creating'|'updating'} msg what to print in the log
-     * @param {TYPE.MetadataTypeItem} [metadataEntry] single metadata entry
+     * @param {SOAPError} ex error that occured
+     * @param {'creating'|'updating'|'retrieving'|'executing'|'pausing'} msg what to print in the log
+     * @param {MetadataTypeItem} [metadataEntry] single metadata entry
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      */
     static _handleSOAPErrors(ex, msg, metadataEntry, handleOutside) {
@@ -994,7 +1156,7 @@ class MetadataType {
     /**
      * helper for {@link MetadataType._handleSOAPErrors}
      *
-     * @param {Error} ex error that occured
+     * @param {SOAPError} ex error that occured
      * @returns {string} error message
      */
     static getSOAPErrorMsg(ex) {
@@ -1010,11 +1172,11 @@ class MetadataType {
     /**
      * Retrieves SOAP via generic fuel-soap wrapper based metadata of metadata type into local filesystem. executes callback with retrieved metadata
      *
-     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
-     * @param {TYPE.SoapRequestParams} [requestParams] required for the specific request (filter for example)
-     * @param {string|number} [singleRetrieve] key of single item to filter by
+     * @param {string} [retrieveDir] Directory where retrieved metadata directory will be saved
+     * @param {SoapRequestParams} [requestParams] required for the specific request (filter for example)
+     * @param {string} [singleRetrieve] key of single item to filter by
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of item map
+     * @returns {Promise.<MetadataTypeMapObj>} Promise of item map
      */
     static async retrieveSOAP(retrieveDir, requestParams, singleRetrieve, additionalFields) {
         requestParams ||= {};
@@ -1029,7 +1191,7 @@ class MetadataType {
             );
         } catch (ex) {
             this._handleSOAPErrors(ex, 'retrieving');
-            return {};
+            return;
         }
         const metadata = this.parseResponseBody(response);
 
@@ -1049,9 +1211,9 @@ class MetadataType {
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string} uri rest endpoint for GET
-     * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
-     * @param {string|number} [singleRetrieve] key of single item to filter by
-     * @returns {Promise.<{metadata: (TYPE.MetadataTypeMap | TYPE.MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
+     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
+     * @param {string} [singleRetrieve] key of single item to filter by
+     * @returns {Promise.<{metadata: (MetadataTypeMap | MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
      */
     static async retrieveREST(retrieveDir, uri, templateVariables, singleRetrieve) {
         const response =
@@ -1063,7 +1225,13 @@ class MetadataType {
         if (this.definition.hasExtended) {
             Util.logger.debug(' - retrieving extended metadata');
             const extended = await this.client.rest.getCollection(
-                Object.keys(results).map((key) => uri + results[key][this.definition.idField])
+                Object.keys(results)
+                    .map((key) =>
+                        uri.endsWith(results[key][this.definition.idField])
+                            ? null
+                            : uri + results[key][this.definition.idField]
+                    )
+                    .filter(Boolean)
             );
             for (const ext of extended) {
                 const key = ext[this.definition.keyField];
@@ -1095,7 +1263,7 @@ class MetadataType {
      * @param {object[]} urlArray {uri: string, id: string} combo of URL and ID/key of metadata
      * @param {number} [concurrentRequests] optionally set a different amount of concurrent requests
      * @param {boolean} [logAmountOfUrls] if true, prints an info message about to-be loaded amount of metadata
-     * @returns {Promise.<{metadata: (TYPE.MetadataTypeMap | TYPE.MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
+     * @returns {Promise.<{metadata: (MetadataTypeMap | MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
      */
     static async retrieveRESTcollection(urlArray, concurrentRequests = 10, logAmountOfUrls = true) {
         if (logAmountOfUrls) {
@@ -1136,11 +1304,11 @@ class MetadataType {
     /**
      * helper for {@link this.retrieveRESTcollection}
      *
-     * @param {Error} ex exception
+     * @param {RestError} ex exception
      * @param {string} id id or key of item
-     * @returns {null} -
+     * @returns {Promise.<any>} -
      */
-    static handleRESTErrors(ex, id) {
+    static async handleRESTErrors(ex, id) {
         // if the ID is too short, the system will throw the 400 error
         Util.logger.debug(` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`);
 
@@ -1170,7 +1338,7 @@ class MetadataType {
     /**
      * Used to execute a query/automation etc.
      *
-     * @param {TYPE.MetadataTypeItem} [metadataEntry] single metadata entry
+     * @param {MetadataTypeItem} [metadataEntry] single metadata entry
      * @returns {Promise.<{key:string, response:object}>} metadata key and API response
      */
     static async executeSOAP(metadataEntry) {
@@ -1202,8 +1370,8 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.retrieveREST} and {@link MetadataType.retrieveSOAP}
      *
-     * @param {string|number} [singleRetrieve] key of single item to filter by
-     * @param {TYPE.MetadataTypeMap} metadataMap saved metadata
+     * @param {string|number} singleRetrieve key of single item to filter by
+     * @param {MetadataTypeMap} metadataMap saved metadata
      * @returns {Promise.<void>} -
      */
     static async runDocumentOnRetrieve(singleRetrieve, metadataMap) {
@@ -1231,8 +1399,8 @@ class MetadataType {
      * Builds map of metadata entries mapped to their keyfields
      *
      * @param {object} body json of response body
-     * @param {string|number} [singleRetrieve] key of single item to filter by
-     * @returns {TYPE.MetadataTypeMap} keyField => metadata map
+     * @param {string} [singleRetrieve] key of single item to filter by
+     * @returns {MetadataTypeMap} keyField => metadata map
      */
     static parseResponseBody(body, singleRetrieve) {
         const bodyIteratorField = this.definition.bodyIteratorField;
@@ -1253,7 +1421,7 @@ class MetadataType {
             } else if (singleRetrieve) {
                 // some types will return a single item intead of an array if the key is supported by their api
                 // ! currently, the id: prefix is only supported by journey (interaction)
-                if (singleRetrieve.startsWith('id:')) {
+                if ('string' === typeof singleRetrieve && singleRetrieve.startsWith('id:')) {
                     singleRetrieve = body[keyField];
                 }
                 metadataStructure[singleRetrieve] = body;
@@ -1280,9 +1448,9 @@ class MetadataType {
      * @example
      * Removes field (or nested fields childs) that are not updateable
      * deleteFieldByDefinition(metadataEntry, 'CustomerKey', 'isUpdateable');
-     * @param {TYPE.MetadataTypeItem} metadataEntry One entry of a metadataType
+     * @param {MetadataTypeItem} metadataEntry One entry of a metadataType
      * @param {string} fieldPath field path to be checked if it conforms to the definition (dot seperated if nested): 'fuu.bar'
-     * @param {'isCreateable'|'isUpdateable'|'retrieving'|'templating'} definitionProperty delete field if definitionProperty equals false for specified field. Options: [isCreateable | isUpdateable]
+     * @param {'isCreateable'|'isUpdateable'|'retrieving'|'template'} definitionProperty delete field if definitionProperty equals false for specified field. Options: [isCreateable | isUpdateable]
      * @param {string} origin string of parent object, required when using arrays as these are parsed slightly differently.
      * @returns {void}
      */
@@ -1353,7 +1521,7 @@ class MetadataType {
     /**
      * Remove fields from metadata entry that are not createable
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
+     * @param {MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static removeNotCreateableFields(metadataEntry) {
@@ -1365,7 +1533,7 @@ class MetadataType {
     /**
      * Remove fields from metadata entry that are not updateable
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
+     * @param {MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static removeNotUpdateableFields(metadataEntry) {
@@ -1377,7 +1545,7 @@ class MetadataType {
     /**
      * Remove fields from metadata entry that are not needed in the template
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
+     * @param {MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static keepTemplateFields(metadataEntry) {
@@ -1389,7 +1557,7 @@ class MetadataType {
     /**
      * Remove fields from metadata entry that are not needed in the stored metadata
      *
-     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
+     * @param {MetadataTypeItem} metadataEntry metadata entry
      * @returns {void}
      */
     static keepRetrieveFields(metadataEntry) {
@@ -1402,7 +1570,7 @@ class MetadataType {
      * checks if the current metadata entry should be saved on retrieve or not
      *
      * @static
-     * @param {TYPE.MetadataTypeItem} metadataEntry metadata entry
+     * @param {MetadataTypeItem} metadataEntry metadata entry
      * @param {boolean} [include] true: use definition.include / options.include; false=exclude: use definition.filter / options.exclude
      * @returns {boolean} true: skip saving == filtered; false: continue with saving
      * @memberof MetadataType
@@ -1551,7 +1719,7 @@ class MetadataType {
      * @returns {?boolean} true: filter value found; false: filter value not found; null: no filter defined
      */
     static _filterOther(myFilter, metadataEntry) {
-        // not possible to check r__folder_Path before parseMetadata was run; handled in `isFilteredFolder()`
+        // not possible to check r__folder_Path before postRetrieveTasks was run; handled in `isFilteredFolder()`
         if (
             !myFilter ||
             !Object.keys(myFilter).filter((item) => item !== 'r__folder_Path').length
@@ -1577,11 +1745,11 @@ class MetadataType {
     /**
      * Helper for writing Metadata to disk, used for Retrieve and deploy
      *
-     * @param {TYPE.MetadataTypeMap} results metadata results from deploy
+     * @param {MetadataTypeMap} results metadata results from deploy
      * @param {string} retrieveDir directory where metadata should be stored after deploy/retrieve
      * @param {string} [overrideType] for use when there is a subtype (such as folder-queries)
-     * @param {TYPE.TemplateMap} [templateVariables] variables to be replaced in the metadata
-     * @returns {Promise.<TYPE.MetadataTypeMap>} Promise of saved metadata
+     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
+     * @returns {Promise.<MetadataTypeMap>} Promise of saved metadata
      */
     static async saveResults(results, retrieveDir, overrideType, templateVariables) {
         const savedResults = {};
@@ -1633,95 +1801,13 @@ class MetadataType {
                     continue;
                 }
 
-                // for complex types like asset, script, query we need to save the scripts that were extracted from the JSON
-                if (results[originalKey].json && results[originalKey].codeArr) {
-                    // replace market values with template variable placeholders (do not do it on .codeArr)
-                    if (templateVariables) {
-                        results[originalKey].json = Util.replaceByObject(
-                            results[originalKey].json,
-                            templateVariables
-                        );
-                        results[originalKey].subFolder = Util.replaceByObject(
-                            results[originalKey].subFolder,
-                            templateVariables
-                        );
-                    }
-
-                    const postRetrieveData = results[originalKey];
-                    // normalize results[metadataEntry]
-                    results[originalKey] = postRetrieveData.json;
-
-                    if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) {
-                        Util.logger.debug(`Filtered ${originalKey} because of --like option`);
-                        continue;
-                    }
-
-                    if (postRetrieveData.subFolder) {
-                        // very complex types have their own subfolder
-                        baseDir.push(...postRetrieveData.subFolder);
-                    }
-                    // save extracted scripts
-                    await Promise.all(
-                        postRetrieveData.codeArr.map(async (script) => {
-                            const dir = [...baseDir];
-                            if (script.subFolder) {
-                                // some files shall be saved in yet a deeper subfolder
-                                dir.push(...script.subFolder);
-                            }
-                            return File.writePrettyToFile(
-                                dir,
-                                script.fileName + subtypeExtension,
-                                script.fileExt,
-                                script.content,
-                                templateVariables
-                            );
-                        })
-                    );
-                } else {
-                    // not a complex type, run the the entire JSON through templating
-                    // replace market values with template variable placeholders
-                    if (templateVariables) {
-                        results[originalKey] = Util.replaceByObject(
-                            results[originalKey],
-                            templateVariables
-                        );
-                    }
-                    if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) {
-                        Util.logger.debug(`Filtered ${originalKey} because of --like option`);
-                        continue;
-                    }
-                }
-
-                // we dont store Id on local disk, but we need it for caching logic,
-                // so its in retrieve but not in save. Here we put into the clone so that the original
-                // object used for caching doesnt have the Id removed.
-                const saveClone = JSON.parse(JSON.stringify(results[originalKey]));
-                if (
-                    !this.definition.keepId &&
-                    this.definition.idField !== this.definition.keyField
-                ) {
-                    delete saveClone[this.definition.idField];
-                }
-
-                if (templateVariables) {
-                    this.keepTemplateFields(saveClone);
-                } else {
-                    this.keepRetrieveFields(saveClone);
-                }
-                savedResults[originalKey] = saveClone;
-                await File.writeJSONToFile(
-                    // manage subtypes
+                savedResults[originalKey] = await this.saveToDisk(
+                    results,
+                    originalKey,
                     baseDir,
-                    originalKey + subtypeExtension,
-                    saveClone
+                    subtypeExtension,
+                    templateVariables
                 );
-                if (templateVariables) {
-                    Util.logger.info(
-                        `- templated ${this.definition.type}: ${
-                            saveClone[this.definition.nameField]
-                        }`
-                    );
-                }
             } catch (ex) {
                 Util.logger.errorStack(
                     ex,
@@ -1737,12 +1823,112 @@ class MetadataType {
         }
         return savedResults;
     }
+
+    /**
+     *
+     * @param {MetadataTypeMap} results metadata results from deploy
+     * @param {string} originalKey key of metadata
+     * @param {string[]} baseDir [retrieveDir, ...overrideType.split('-')]
+     * @param {string} [subtypeExtension] e.g. ".asset-meta" or ".query-meta"
+     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
+     * @returns {MetadataTypeItem} saved metadata
+     */
+    static async saveToDisk(results, originalKey, baseDir, subtypeExtension, templateVariables) {
+        subtypeExtension ||= '.' + this.definition.type + '-meta';
+        // for complex types like asset, script, query we need to save the scripts that were extracted from the JSON
+        if (results[originalKey].json && results[originalKey].codeArr) {
+            // replace market values with template variable placeholders (do not do it on .codeArr)
+            if (templateVariables) {
+                results[originalKey].json = Util.replaceByObject(
+                    results[originalKey].json,
+                    templateVariables
+                );
+                results[originalKey].subFolder = Util.replaceByObject(
+                    results[originalKey].subFolder,
+                    templateVariables
+                );
+            }
+
+            const postRetrieveData = results[originalKey];
+            // normalize results[metadataEntry]
+            results[originalKey] = postRetrieveData.json;
+
+            if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) {
+                Util.logger.debug(`Filtered ${originalKey} because of --like option`);
+                return;
+            }
+
+            if (postRetrieveData.subFolder) {
+                // very complex types have their own subfolder
+                baseDir.push(...postRetrieveData.subFolder);
+            }
+            // save extracted scripts
+            await Promise.all(
+                postRetrieveData.codeArr.map(async (script) => {
+                    const dir = [...baseDir];
+                    if (script.subFolder) {
+                        // some files shall be saved in yet a deeper subfolder
+                        dir.push(...script.subFolder);
+                    }
+                    return File.writePrettyToFile(
+                        dir,
+                        script.fileName + subtypeExtension,
+                        script.fileExt,
+                        script.content,
+                        templateVariables
+                    );
+                })
+            );
+        } else {
+            // not a complex type, run the the entire JSON through templating
+            // replace market values with template variable placeholders
+            if (templateVariables) {
+                results[originalKey] = Util.replaceByObject(
+                    results[originalKey],
+                    templateVariables
+                );
+            }
+            if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) {
+                Util.logger.debug(`Filtered ${originalKey} because of --like option`);
+                return;
+            }
+        }
+
+        // we dont store Id on local disk, but we need it for caching logic,
+        // so its in retrieve but not in save. Here we put into the clone so that the original
+        // object used for caching doesnt have the Id removed.
+
+        const saveClone = structuredClone(results[originalKey]);
+        if (!this.definition.keepId && this.definition.idField !== this.definition.keyField) {
+            delete saveClone[this.definition.idField];
+        }
+
+        if (templateVariables) {
+            this.keepTemplateFields(saveClone);
+        } else {
+            this.keepRetrieveFields(saveClone);
+        }
+
+        await File.writeJSONToFile(
+            // manage subtypes
+            baseDir,
+            originalKey + subtypeExtension,
+            saveClone
+        );
+        if (templateVariables) {
+            Util.logger.info(
+                `- templated ${this.definition.type}: ${saveClone[this.definition.nameField]}`
+            );
+        }
+
+        return saveClone;
+    }
     /**
      * helper for {@link MetadataType.buildDefinitionForNested}
      * searches extracted file for template variable names and applies the market values
      *
      * @param {string} code code from extracted code
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
      * @returns {string} code with markets applied
      */
     static applyTemplateValues(code, templateVariables) {
@@ -1754,7 +1940,7 @@ class MetadataType {
      * searches extracted file for template variable values and applies the market variable names
      *
      * @param {string} code code from extracted code
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
      * @returns {string} code with markets applied
      */
     static applyTemplateNames(code, templateVariables) {
@@ -1766,9 +1952,9 @@ class MetadataType {
      * handles extracted code if any are found for complex types (e.g script, asset, query)
      *
      * @param {string} templateDir Directory where metadata templates are stored
-     * @param {string} targetDir Directory where built definitions will be saved
-     * @param {TYPE.MetadataTypeItem} metadata main JSON file that was read from file system
-     * @param {TYPE.TemplateMap} variables variables to be replaced in the metadata
+     * @param {string | string[]} targetDir Directory where built definitions will be saved
+     * @param {MetadataTypeItem} metadata main JSON file that was read from file system
+     * @param {TemplateMap} variables variables to be replaced in the metadata
      * @param {string} templateName name of the template to be built
      * @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
      */
@@ -1788,8 +1974,8 @@ class MetadataType {
      *
      * @param {string} templateDir Directory where metadata templates are stored
      * @param {string|string[]} targetDir (List of) Directory where built definitions will be saved
-     * @param {TYPE.MetadataTypeItem} metadata main JSON file that was read from file system
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @param {MetadataTypeItem} metadata main JSON file that was read from file system
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
      * @param {string} templateName name of the template to be built
      * @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
      */
@@ -1821,11 +2007,11 @@ class MetadataType {
      * @param {string} templateName name of the metadata template
      * @param {string} fileName name of the metadata template file w/o extension
      * @param {Error} ex error from first attempt
-     * @returns {object} metadata
+     * @returns {Promise.<string>} metadata in string form
      */
     static async readSecondaryFolder(templateDir, typeDirArr, templateName, fileName, ex) {
         // we just want to push the method into the catch here
-        throw new Error(ex);
+        return;
     }
     /**
      * Builds definition based on template
@@ -1835,8 +2021,8 @@ class MetadataType {
      * @param {string} templateDir Directory where metadata templates are stored
      * @param {string | string[]} targetDir (List of) Directory where built definitions will be saved
      * @param {string} templateName name of the metadata file
-     * @param {TYPE.TemplateMap} variables variables to be replaced in the metadata
-     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of item map
+     * @param {TemplateMap} variables variables to be replaced in the metadata
+     * @returns {Promise.<MetadataTypeMapObj>} Promise of item map
      */
     static async buildDefinition(templateDir, targetDir, templateName, variables) {
         // retrieve metadata template
@@ -1856,6 +2042,9 @@ class MetadataType {
                 fileName,
                 'json'
             );
+            if (!metadataStr) {
+                throw new Error('File not found');
+            }
         } catch (ex) {
             try {
                 metadataStr = await this.readSecondaryFolder(
@@ -1866,22 +2055,26 @@ class MetadataType {
                     ex
                 );
             } catch {
-                throw new Error(
-                    `${this.definition.type}:: Could not find ./${File.normalizePath([
-                        templateDir,
-                        ...typeDirArr,
-                        fileName + '.json',
-                    ])}.`
+                // only happening for types that use readSecondaryFolder (e.g. asset)
+                // if we still have no metadataStr then we have to skip this metadata for all types and hence handle it outside of this catch
+            }
+            if (!metadataStr) {
+                Util.logger.warn(
+                    Util.getGrayMsg(
+                        `- skipped ${this.definition.type} ${templateName}: template not found`
+                    )
                 );
+                return;
             }
-            // return;
         }
 
         let metadata;
         try {
             // update all initial variables & create metadata object
-            metadata = JSON.parse(Mustache.render(metadataStr, variables, {}, ['{{{', '}}}']));
-            typeDirArr = typeDirArr.map((el) => Mustache.render(el, variables, {}, ['{{{', '}}}']));
+            metadata = JSON.parse(this.applyTemplateValues(metadataStr, variables));
+            typeDirArr = typeDirArr
+                .map((el) => (el === templateName ? metadata[this.definition.keyField] : el))
+                .map((el) => this.applyTemplateValues(el, variables));
         } catch {
             throw new Error(
                 `${this.definition.type}:: Error applying template variables on ${
@@ -1904,7 +2097,6 @@ class MetadataType {
         try {
             // write to file
             const targetDirArr = Array.isArray(targetDir) ? targetDir : [targetDir];
-
             for (const targetDir of targetDirArr) {
                 await File.writeJSONToFile(
                     [targetDir, ...typeDirArr],
@@ -1927,7 +2119,7 @@ class MetadataType {
      *  Standardizes a check for multiple messages
      *
      * @param {object} ex response payload from REST API
-     * @returns {string[] | void} formatted Error Message
+     * @returns {string[]} formatted Error Message
      */
     static getErrorsREST(ex) {
         const errors = [];
@@ -1966,7 +2158,7 @@ class MetadataType {
     /**
      * Gets metadata cache with limited fields and does not store value to disk
      *
-     * @param {TYPE.MetadataTypeMap} [metadata] a list of type definitions
+     * @param {MetadataTypeMap} [metadata] a list of type definitions
      * @param {boolean} [isDeploy] used to skip non-supported message during deploy
      * @returns {void}
      */
@@ -1976,13 +2168,24 @@ class MetadataType {
         }
     }
 
+    /**
+     * get name & key for provided id
+     *
+     * @param {string} id Identifier of metadata
+     * @returns {Promise.<{key:string, name:string}>} key, name and path of metadata; null if not found
+     */
+    static async resolveId(id) {
+        Util.logger.error(`resolveId type ${this.definition.type} is not supported.`);
+        return;
+    }
+
     /**
      * Delete a metadata item from the specified business unit
      *
      * @param {string} customerKey Identifier of data extension
-     * @returns {boolean} deletion success status
+     * @returns {Promise.<boolean>} deletion success status
      */
-    static deleteByKey(customerKey) {
+    static async deleteByKey(customerKey) {
         Util.logger.error(`Deletion is not yet supported for ${this.definition.typeName}!`);
         return false;
     }
@@ -2014,13 +2217,14 @@ class MetadataType {
      * @param {string} customerKey Identifier of metadata
      * @param {string} [overrideKeyField] optionally change the name of the key field if the api uses a different name
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
-     * @returns {boolean} deletion success flag
+     * @returns {Promise.<boolean>} deletion success flag
      */
     static async deleteByKeySOAP(customerKey, overrideKeyField, handleOutside) {
         const metadata = {};
         metadata[overrideKeyField || this.definition.keyField] = customerKey;
         const soapType = this.definition.soapType || this.definition.type;
         try {
+            // @ts-expect-error - wrong jsdoc in SFMC-SDK
             await this.client.soap.delete(Util.capitalizeFirstLetter(soapType), metadata, null);
             if (!handleOutside) {
                 Util.logger.info(` - deleted ${this.definition.type}: ${customerKey}`);
@@ -2049,7 +2253,7 @@ class MetadataType {
      * @param {string} url endpoint
      * @param {string} key Identifier of metadata
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
-     * @returns {boolean} deletion success flag
+     * @returns {Promise.<boolean>} deletion success flag
      */
     static async deleteByKeyREST(url, key, handleOutside) {
         try {
@@ -2076,15 +2280,15 @@ class MetadataType {
      * @param {string} readDir root directory of metadata.
      * @param {boolean} [listBadKeys] do not print errors, used for badKeys()
      * @param {object} [buMetadata] Metadata of BU in local directory
-     * @returns {object} Metadata of BU in local directory
+     * @returns {Promise.<object>} Metadata of BU in local directory
      */
-    static readBUMetadataForType(readDir, listBadKeys, buMetadata) {
+    static async readBUMetadataForType(readDir, listBadKeys, buMetadata) {
         buMetadata ||= {};
         readDir = File.normalizePath([readDir, this.definition.type]);
         try {
-            if (File.pathExistsSync(readDir)) {
+            if (await File.pathExists(readDir)) {
                 // check if folder name is a valid metadataType, then check if the user limited to a certain type in the command params
-                buMetadata[this.definition.type] = this.getJsonFromFS(readDir, listBadKeys);
+                buMetadata[this.definition.type] = await this.getJsonFromFS(readDir, listBadKeys);
                 return buMetadata;
             } else {
                 throw new Error(`Directory '${readDir}' does not exist.`);
@@ -2100,7 +2304,7 @@ class MetadataType {
      * @param {string[]} keyArr customerkey of the metadata
      * @returns {Promise.<string[]>} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
      */
-    static getFilesToCommit(keyArr) {
+    static async getFilesToCommit(keyArr) {
         const typeExtension = '.' + this.definition.type + '-meta.json';
         const path = File.normalizePath([
             this.properties.directories.retrieve,
@@ -2115,39 +2319,49 @@ class MetadataType {
 
     /**
      *
-     * @param {TYPE.MetadataTypeMap} metadataMap metadata mapped by their keyField
+     * @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
      * @returns {string[]} list of keys
      */
     static getKeysForFixing(metadataMap) {
         const keysForDeploy = [];
+        if (!metadataMap) {
+            // if a type was skipped e.g. because it shall only be looked at on the parent then we would expect metadataMap to be undefined
+            return keysForDeploy;
+        }
         if (Object.keys(metadataMap).length) {
             Util.logger.info(
                 `Searching for ${this.definition.type} keys among downloaded items that need fixing:`
             );
+            Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : '';
+            const maxKeyLength = this.definition.maxKeyLength || 36;
+
             for (const item of Object.values(metadataMap)) {
-                if (item[this.definition.nameField].length > this.definition.maxKeyLength) {
-                    Util.logger.warn(
-                        `Name of the item ${
-                            item[this.definition.keyField]
-                        } is too long for a key. Consider renaming your item. Key will be equal first ${
-                            this.definition.maxKeyLength
-                        } characters of the name`
-                    );
-                    item[this.definition.nameField] = item[this.definition.nameField].slice(
-                        0,
-                        this.definition.maxKeyLength
-                    );
-                }
+                if (this.isFiltered(item, true) || this.isFiltered(item, false)) {
+                    // we would not have saved these items to disk but they exist in the cache and hence need to be skipped here
 
+                    continue;
+                }
                 if (
-                    item[this.definition.nameField] != item[this.definition.keyField] &&
-                    !this.definition.keyIsFixed
+                    (item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+                        item[this.definition.nameField].length > maxKeyLength) ||
+                    (!item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+                        item[this.definition.nameField].length + Util.OPTIONS.keySuffix.length >
+                            maxKeyLength)
                 ) {
-                    keysForDeploy.push(item[this.definition.keyField]);
+                    Util.logger.warn(
+                        `Name of the item ${item[this.definition.keyField]} (${
+                            item[this.definition.nameField]
+                        }) is too long for a key${Util.OPTIONS.keySuffix.length ? ' (including the suffix ' + Util.OPTIONS.keySuffix + ')' : ''}. Consider renaming your item. Key will be equal first ${maxKeyLength} characters of the name`
+                    );
+                }
+                const newKey = this.getNewKey(this.definition.nameField, item, maxKeyLength);
+                if (newKey != item[this.definition.keyField] && !this.definition.keyIsFixed) {
+                    // add key but make sure to turn it into string or else numeric keys will be filtered later
+                    keysForDeploy.push(item[this.definition.keyField] + '');
                     Util.logger.info(
                         ` - added ${this.definition.type} to fixKey queue: ${
                             item[this.definition.keyField]
-                        }`
+                        } >> ${newKey}`
                     );
                 } else {
                     Util.logger.info(
@@ -2163,6 +2377,28 @@ class MetadataType {
         }
         return keysForDeploy;
     }
+    /**
+     * helper for getKeysForFixing and createOrUpdate
+     *
+     * @param {string} baseField name of the field to start the new key with
+     * @param {MetadataTypeItem} metadataItem -
+     * @param {number} maxKeyLength -
+     * @returns {string} newKey
+     */
+    static getNewKey(baseField, metadataItem, maxKeyLength) {
+        const keySuffix = Util.OPTIONS.keySuffix;
+        let newKey;
+        newKey = (metadataItem[baseField] + '').trim().slice(0, maxKeyLength).trim();
+        if (keySuffix.length && !newKey.endsWith(keySuffix)) {
+            newKey =
+                (metadataItem[baseField] + '')
+                    .trim()
+                    .slice(0, maxKeyLength - keySuffix.length)
+                    .trim() + keySuffix;
+        }
+
+        return newKey;
+    }
 }
 
 MetadataType.definition = {
@@ -2176,11 +2412,11 @@ MetadataType.definition = {
     type: '',
 };
 /**
- * @type {TYPE.SDK}
+ * @type {SDK}
  */
 MetadataType.client = undefined;
 /**
- * @type {TYPE.Mcdevrc}
+ * @type {Mcdevrc}
  */
 MetadataType.properties = null;
 /**
@@ -2188,7 +2424,7 @@ MetadataType.properties = null;
  */
 MetadataType.subType = null;
 /**
- * @type {TYPE.BuObject}
+ * @type {BuObject}
  */
 MetadataType.buObject = null;