mcdev 3.1.1 → 4.0.0

package/lib/metadataTypes/DataExtension.js

@@ -1,62 +1,48 @@
 'use strict';
 
 const jsonToTable = require('json-to-table');
+const TYPE = require('../../types/mcdev.d');
 const MetadataType = require('./MetadataType');
 const DataExtensionField = require('./DataExtensionField');
 const Folder = require('./Folder');
 const Util = require('../util/util');
 const File = require('../util/file');
-
-/**
- * @typedef {Object} DataExtensionItem
- * @property {string} CustomerKey key
- * @property {string} Name name
- * @property {string} Description -
- * @property {'true'|'false'} IsSendable -
- * @property {'true'|'false'} IsTestable -
- * @property {Object} SendableDataExtensionField -
- * @property {string} SendableDataExtensionField.Name -
- * @property {Object} SendableSubscriberField -
- * @property {string} SendableSubscriberField.Name -
- * @property {DataExtensionField.DataExtensionFieldItem[]} Fields list of DE fields
- * @property {'dataextension'|'salesforcedataextension'|'synchronizeddataextension'|'shared_dataextension'|'shared_salesforcedataextension'} r__folder_ContentType retrieved from associated folder
- * @property {string} r__folder_Path folder path in which this DE is saved
- * @property {string} [CategoryID] holds folder ID, replaced with r__folder_Path during retrieve
- * @property {string} [r__dataExtensionTemplate_Name] name of optionally associated DE template
- * @property {Object} [Template] -
- * @property {string} [Template.CustomerKey] key of optionally associated DE teplate
- *
- * @typedef {Object.<string, DataExtensionItem>} DataExtensionMap
- */
+const auth = require('../util/auth');
+const cache = require('../util/cache');
 
 /**
  * DataExtension MetadataType
+ *
  * @augments MetadataType
  */
 class DataExtension extends MetadataType {
     /**
      * Upserts dataExtensions after retrieving them from source and target to compare
      * if create or update operation is needed.
-     * @param {DataExtensionMap} desToDeploy dataExtensions mapped by their customerKey
-     * @param {Object} _ -
-     * @param {Util.BuObject} buObject properties for auth
+     *
+     * @param {TYPE.DataExtensionMap} desToDeploy dataExtensions mapped by their customerKey
+     * @param {void} _ unused parameter
+     * @param {TYPE.BuObject} buObject properties for auth
      * @returns {Promise} Promise
      */
     static async upsert(desToDeploy, _, buObject) {
-        Util.logger.info('- Retrieve target metadata for comparison with deploy metadata');
-        const results = await this.retrieveForCache(buObject, null, true);
-        const targetMetadata = results.metadata;
-        Util.logger.info('- Retrieved target metadata');
+        // get dataExtensions from target BU for add/update decision
+        /** @type {TYPE.DataExtensionMap} */
+        const targetMetadata = cache.getCache().dataExtension || {};
+        // get existing de-fields to properly handle add/update/delete of fields
+        await this._attachFields(targetMetadata);
+
         /** @type {Promise[]} */
         const deCreatePromises = [];
         /** @type {Promise[]} */
         const deUpdatePromises = [];
+        let filteredByPreDeploy = 0;
         for (const dataExtension in desToDeploy) {
             if (desToDeploy[dataExtension].Name.startsWith('_')) {
                 Util.logger.warn(
-                    '- Cannot Upsert Strongly Typed Data Extensions - skipping ' +
-                        desToDeploy[dataExtension].Name
+                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}: Cannot Upsert Strongly Typed Data Extensions`
                 );
+                filteredByPreDeploy++;
                 continue;
             }
             if (
@@ -65,8 +51,9 @@ class DataExtension extends MetadataType {
             ) {
                 // this needs to be run before executing preDeployTasks()
                 Util.logger.warn(
-                    `- Cannot Create/Update a Shared Data Extension from the Child BU - skipping ${desToDeploy[dataExtension].Name}`
+                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}: Cannot Create/Update a Shared Data Extension from the Child BU`
                 );
+                filteredByPreDeploy++;
                 continue;
             }
             if (
@@ -74,8 +61,9 @@ class DataExtension extends MetadataType {
             ) {
                 // this needs to be run before executing preDeployTasks()
                 Util.logger.warn(
-                    `- Cannot Create/Update a Synchronized Data Extension. Please use Contact Builder to maintain these - skipping ${desToDeploy[dataExtension].Name}`
+                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}:Cannot Create/Update a Synchronized Data Extension. Please use Contact Builder to maintain these`
                 );
+                filteredByPreDeploy++;
                 continue;
             }
             try {
@@ -100,7 +88,7 @@ class DataExtension extends MetadataType {
         }
         if (deUpdatePromises.length) {
             Util.logger.info(
-                '- Please note that Data Retention Policies can only be set during creation, not during update.'
+                ' - Please note that Data Retention Policies can only be set during creation, not during update.'
             );
         }
 
@@ -112,27 +100,23 @@ class DataExtension extends MetadataType {
         );
 
         const successfulResults = [...createResults, ...updateResults];
-
-        Util.metadataLogger(
-            'info',
-            this.definition.type,
-            'upsert',
-            `${createResults.length} of ${deCreatePromises.length} created / ${updateResults.length} of ${deUpdatePromises.length} updated`
+        Util.logger.info(
+            `${this.definition.type} upsert: ${createResults.length} of ${deCreatePromises.length} created / ${updateResults.length} of ${deUpdatePromises.length} updated` +
+                (filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
         );
         if (successfulResults.length > 0) {
             const metadataResults = successfulResults
-                .map((r) => r.value.body.Results[0].Object)
+                .map((r) => r.value.Results[0].Object)
                 .map((r) => {
                     // if only one fields added will return object otherwise array
-                    if (r.Fields && r.Fields.Field && Array.isArray(r.Fields.Field)) {
+                    if (Array.isArray(r?.Fields?.Field)) {
                         r.Fields = r.Fields.Field;
-                    } else if (r.Fields && r.Fields.Field) {
+                    } else if (r?.Fields?.Field) {
                         r.Fields = [r.Fields.Field];
                     }
                     return r;
                 });
-            const formattedResults = super.parseResponseBody({ Results: metadataResults });
-            return formattedResults;
+            return super.parseResponseBody({ Results: metadataResults });
         } else {
             return {};
         }
@@ -140,8 +124,9 @@ class DataExtension extends MetadataType {
 
     /**
      * helper for upsert()
-     * @param {Object} res -
-     * @returns {Boolean} true: keep, false: discard
+     *
+     * @param {object} res -
+     * @returns {boolean} true: keep, false: discard
      */
     static _filterUpsertResults(res) {
         if (res.status === 'rejected') {
@@ -155,12 +140,12 @@ class DataExtension extends MetadataType {
         } else if (res.value.results) {
             Util.logger.error(
                 '- error upserting dataExtension: ' +
-                    (res.value.results[0].Object ? res.value.results[0].Object.Name : '') +
+                    (res.value.Results[0].Object ? res.value.Results[0].Object.Name : '') +
                     '. ' +
-                    res.value.results[0].StatusMessage
+                    res.value.Results[0].StatusMessage
             );
             return false;
-        } else if (res.status === 'fulfilled' && res.value && res.value.faultstring) {
+        } else if (res.status === 'fulfilled' && res?.value?.faultstring) {
             // can happen that the promise does not reject, but that it resolves an error
             Util.logger.error('- error upserting dataExtension: ' + res.value.faultstring);
             return false;
@@ -171,7 +156,8 @@ class DataExtension extends MetadataType {
 
     /**
      * Create a single dataExtension. Also creates their columns in 'dataExtension.columns'
-     * @param {DataExtensionItem} metadata single metadata entry
+     *
+     * @param {TYPE.DataExtensionItem} metadata single metadata entry
      * @returns {Promise} Promise
      */
     static async create(metadata) {
@@ -193,8 +179,9 @@ class DataExtension extends MetadataType {
      * SFMC saves a date in "RetainUntil" under certain circumstances even
      * if that field duplicates whats in the period fields
      * during deployment, that extra value is not accepted by the APIs which is why it needs to be removed
+     *
      * @private
-     * @param {DataExtensionItem} metadata single metadata entry
+     * @param {TYPE.DataExtensionItem} metadata single metadata entry
      * @returns {void}
      */
     static _cleanupRetentionPolicyFields(metadata) {
@@ -205,23 +192,23 @@ class DataExtension extends MetadataType {
         ) {
             metadata.RetainUntil = '';
             Util.logger.warn(
-                `RetainUntil date was reset automatically because RetentionPeriod info was found in: ${metadata.CustomerKey}`
+                ` - RetainUntil date was reset automatically because RetentionPeriod info was found in: ${metadata.CustomerKey}`
             );
         }
     }
     /**
      * Updates a single dataExtension. Also updates their columns in 'dataExtension.columns'
-     * @param {DataExtensionItem} metadata single metadata entry
+     *
+     * @param {TYPE.DataExtensionItem} metadata single metadata entry
      * @returns {Promise} Promise
      */
     static async update(metadata) {
         // Update dataExtension + Columns if they already exist; Create them if not
         // Modify columns for update call
-        DataExtensionField.cache = this.metadata;
         DataExtensionField.client = this.client;
         DataExtensionField.properties = this.properties;
         DataExtension.oldFields = DataExtension.oldFields || {};
-        DataExtension.oldFields[metadata.CustomerKey] =
+        DataExtension.oldFields[metadata[this.definition.keyField]] =
             await DataExtensionField.prepareDeployColumnsOnUpdate(
                 metadata.Fields,
                 metadata.CustomerKey
@@ -240,97 +227,129 @@ class DataExtension extends MetadataType {
     }
     /**
      * Gets executed after deployment of metadata type
-     * @param {DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     *
+     * @param {TYPE.DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     * @param {TYPE.DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
      * @returns {void}
      */
-    static postDeployTasks(upsertedMetadata) {
-        if (!DataExtension.oldFields) {
-            // only run postDeploy if we are in update mode
-            return;
-        }
-        // somewhat of a workardoun but it ensures we get the field list from the server rather than whatever we might have in cache got returned during update/add. This ensures a complete and correctly ordered field list
+    static postDeployTasks(upsertedMetadata, originalMetadata) {
         for (const key in upsertedMetadata) {
             const item = upsertedMetadata[key];
-            const isUpdate =
-                this.cache &&
-                this.cache.dataExtension &&
-                this.cache.dataExtension[item.CustomerKey];
-            if (isUpdate) {
-                const cachedVersion = this.cache.dataExtension[item.CustomerKey];
+            const cachedVersion = cache.getByKey('dataExtension', item.CustomerKey);
+            if (cachedVersion) {
+                // UPDATE
                 // restore retention values that are typically not returned by the update call
                 item.RowBasedRetention = cachedVersion.RowBasedRetention;
                 item.ResetRetentionPeriodOnImport = cachedVersion.ResetRetentionPeriodOnImport;
                 item.DeleteAtEndOfRetentionPeriod = cachedVersion.DeleteAtEndOfRetentionPeriod;
                 item.RetainUntil = cachedVersion.RetainUntil;
 
-                // ensure we have th
-                const existingFields = DataExtension.oldFields[item[this.definition.nameField]];
-                if (item.Fields !== '' && existingFields) {
-                    // TODO should be replaced by a manual sort using existingFields
-                    // ! this is inefficient because it triggers a new download of the fields during the saveResults() step
-                    item.Fields.length = 0;
+                const existingFields = DataExtension.oldFields[item[this.definition.keyField]];
+                if (item.Fields === '') {
+                    // if no fields were updated, we need to set Fields to "empty string" for the API to work
+                    // reset here to get the correct field list
+                    item.Fields = Object.keys(existingFields)
+                        .map((key) => existingFields[key])
+                        .sort((a, b) => a.Ordinal - b.Ordinal);
+                }
+                if (existingFields) {
+                    // get list of updated fields
+                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    const updatedFieldsArr = originalMetadata[key].Fields.Field.filter(
+                        (field) => field.ObjectID && field.ObjectID !== ''
+                    );
+                    // convert existing fields obj into array and sort
+                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    const finalFieldsArr = Object.keys(existingFields)
+                        .map((key) => {
+                            /** @type {TYPE.DataExtensionFieldItem} */
+                            const existingField = existingFields[key];
+                            // check if the current field was updated and then override with it. otherwise use existing value
+                            const field =
+                                updatedFieldsArr.find(
+                                    (field) => field.ObjectID === existingField.ObjectID
+                                ) || existingField;
+                            // field does not have a ordinal value because we rely on array order
+                            field.Ordinal = existingField.Ordinal;
+                            // updating FieldType is not supported by API and hence removed
+                            field.FieldType = existingField.FieldType;
+                            return field;
+                        })
+                        .sort((a, b) => a.Ordinal - b.Ordinal);
+
+                    // get list of new fields
+                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    const newFieldsArr = originalMetadata[key].Fields.Field.filter(
+                        (field) => !field.ObjectID
+                    );
+                    // push new fields to end of list
+                    if (newFieldsArr.length) {
+                        finalFieldsArr.push(...newFieldsArr);
+                    }
+
+                    // sort Fields entry to the end of the object for saving in .json
+                    delete item.Fields;
+                    item.Fields = finalFieldsArr;
                 }
-                // sort Fields entry to the end of the object for saving in .json
-                const fieldsBackup = item.Fields;
-                delete item.Fields;
-                item.Fields = fieldsBackup;
+            }
+            // UPDATE + CREATE
+            for (const field of item.Fields) {
+                DataExtensionField.postRetrieveTasks(field, true);
             }
         }
     }
 
     /**
      * Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionColumn metadata retrieval
+     *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Util.BuObject} buObject properties for auth
-     * @param {void} [_] -
-     * @param {boolean} [isDeploy] used to signal that fields shall be retrieve in caching mode
-     * @returns {Promise<{metadata:DataExtensionMap,type:string}>} Promise of item map
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @param {void} [_] unused parameter
+     * @param {string} [key] customer key of single item to retrieve
+     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of item map
      */
-    static async retrieve(retrieveDir, additionalFields, buObject, _, isDeploy) {
-        let metadata = await this._retrieveAll(additionalFields);
+    static async retrieve(retrieveDir, additionalFields, buObject, _, key) {
+        /** @type {TYPE.SoapRequestParams} */
+        let requestParams = null;
+        /** @type {TYPE.SoapRequestParams} */
+        let fieldOptions = null;
+        if (key) {
+            requestParams = {
+                filter: {
+                    leftOperand: 'CustomerKey',
+                    operator: 'equals',
+                    rightOperand: key,
+                },
+            };
+            fieldOptions = {
+                filter: {
+                    leftOperand: 'DataExtension.CustomerKey',
+                    operator: 'equals',
+                    rightOperand: key,
+                },
+            };
+        }
+        let metadata = await this._retrieveAll(additionalFields, requestParams);
         // in case of cache dont get fields
-        if (isDeploy || (metadata && retrieveDir)) {
+        if (metadata && retrieveDir) {
             // get fields from API
-            const fieldsObj = await this._retrieveFields(null, additionalFields);
-            const fieldKeys = Object.keys(fieldsObj);
-            // add fields to corresponding DE
-            fieldKeys.forEach((key) => {
-                const field = fieldsObj[key];
-                if (metadata[field?.DataExtension?.CustomerKey]) {
-                    metadata[field.DataExtension.CustomerKey].Fields.push(field);
-                } else {
-                    Util.logger.warn(`Issue retrieving data extension fields. key='${key}'`);
-                }
-            });
-
-            // sort fields by Ordinal value (API returns field unsorted)
-            for (const metadataEntry in metadata) {
-                metadata[metadataEntry].Fields.sort(DataExtensionField.sortDeFields);
-            }
-
-            // remove attributes that we do not want to retrieve
-            // * do this after sorting on the DE's field list
-            fieldKeys.forEach((key) => {
-                DataExtensionField.postRetrieveTasks(fieldsObj[key], true);
-            });
+            await this._attachFields(metadata, fieldOptions, additionalFields);
         }
         if (!retrieveDir && buObject.eid !== buObject.mid) {
             // for caching, we want to retrieve shared DEs as well from the instance parent BU
-            Util.logger.info('- Caching dependent Metadata: dataExtension (shared via _ParentBU_)');
-            /** @type {Util.BuObject} */
+            Util.logger.info(
+                ' - Caching dependent Metadata: dataExtension (shared via _ParentBU_)'
+            );
+            /** @type {TYPE.BuObject} */
             const buObjectParentBu = {
-                clientId: this.properties.credentials[buObject.credential].clientId,
-                clientSecret: this.properties.credentials[buObject.credential].clientSecret,
-                tenant: this.properties.credentials[buObject.credential].tenant,
                 eid: this.properties.credentials[buObject.credential].eid,
                 mid: this.properties.credentials[buObject.credential].eid,
                 businessUnit: Util.parentBuName,
                 credential: buObject.credential,
             };
-            const clientBackup = this.client;
             try {
-                this.client = await Util.getETClient(buObjectParentBu);
+                this.client = auth.getSDK(buObjectParentBu);
             } catch (ex) {
                 Util.logger.error(ex.message);
                 return;
@@ -338,14 +357,11 @@ class DataExtension extends MetadataType {
             const metadataParentBu = await this._retrieveAll(additionalFields);
 
             // get shared folders to match our shared / synched Data Extensions
-            Util.logger.info('- Caching dependent Metadata: folder (shared via _ParentBU_)');
-            Folder.cache = {};
+            Util.logger.info(' - Caching dependent Metadata: folder (shared via _ParentBU_)');
             Folder.client = this.client;
             Folder.properties = this.properties;
             const result = await Folder.retrieveForCache(buObjectParentBu);
-            const parentCache = {
-                folder: result.metadata,
-            };
+            cache.mergeMetadata('folder', result.metadata, buObject.eid);
 
             // get the types and clean out non-shared ones
             const folderTypesFromParent = require('../MetadataTypeDefinitions').folder
@@ -353,12 +369,12 @@ class DataExtension extends MetadataType {
             for (const metadataEntry in metadataParentBu) {
                 try {
                     // get the data extension type from the folder
-                    const folderContentType = Util.getFromCache(
-                        parentCache,
+                    const folderContentType = cache.searchForField(
                         'folder',
                         metadataParentBu[metadataEntry].CategoryID,
                         'ID',
-                        'ContentType'
+                        'ContentType',
+                        buObject.eid
                     );
                     if (!folderTypesFromParent.includes(folderContentType)) {
                         Util.logger.verbose(
@@ -375,16 +391,14 @@ class DataExtension extends MetadataType {
             }
 
             // revert client to current default
-            this.client = clientBackup;
-            Folder.client = clientBackup;
-            Folder.cache = this.cache;
+            this.client = auth.getSDK(this.buObject);
+            Folder.client = auth.getSDK(this.buObject);
 
             // make sure to overwrite parent bu DEs with local ones
             metadata = { ...metadataParentBu, ...metadata };
         }
-
         if (retrieveDir) {
-            const savedMetadata = await this.saveResults(metadata, retrieveDir, null);
+            const savedMetadata = await super.saveResults(metadata, retrieveDir, null);
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
             );
@@ -394,41 +408,62 @@ class DataExtension extends MetadataType {
         }
         return { metadata: metadata, type: 'dataExtension' };
     }
+
+    /**
+     * helper to retrieve all dataExtension fields and attach them to the dataExtension metadata
+     *
+     * @private
+     * @param {TYPE.DataExtensionMap} metadata already cached dataExtension metadata
+     * @param {TYPE.SoapRequestParams} [fieldOptions] optionally filter results
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @returns {Promise.<void>} -
+     */
+    static async _attachFields(metadata, fieldOptions, additionalFields) {
+        const fieldsObj = await this._retrieveFields(fieldOptions, additionalFields);
+        const fieldKeys = Object.keys(fieldsObj);
+        // add fields to corresponding DE
+        for (const key of fieldKeys) {
+            const field = fieldsObj[key];
+            if (metadata[field?.DataExtension?.CustomerKey]) {
+                metadata[field.DataExtension.CustomerKey].Fields.push(field);
+            } else {
+                Util.logger.warn(` - Issue retrieving data extension fields. key='${key}'`);
+            }
+        }
+
+        // sort fields by Ordinal value (API returns field unsorted)
+        for (const metadataEntry in metadata) {
+            metadata[metadataEntry].Fields.sort(DataExtensionField.sortDeFields);
+        }
+
+        // remove attributes that we do not want to retrieve
+        // * do this after sorting on the DE's field list
+        for (const key of fieldKeys) {
+            DataExtensionField.postRetrieveTasks(fieldsObj[key], true);
+        }
+    }
+
     /**
      * Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionColumn metadata retrieval
+     *
+     * @param {TYPE.BuObject} [buObject] properties for auth
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise<{metadata:DataExtensionMap,type:string}>} Promise of item map
+     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of item map
      */
-    static async retrieveChangelog(additionalFields) {
+    static async retrieveChangelog(buObject, additionalFields) {
         const metadata = await this._retrieveAll(additionalFields);
         return { metadata: metadata, type: 'dataExtension' };
     }
     /**
      * manages post retrieve steps
-     * @param {DataExtensionItem} metadata a single dataExtension
-     * @param {string} [_] unused
-     * @param {boolean} [isTemplating] signals that we are retrieving templates
-     * @returns {DataExtensionItem} metadata
+     *
+     * @param {TYPE.DataExtensionItem} metadata a single dataExtension
+     * @returns {TYPE.DataExtensionItem} metadata
      */
-    static async postRetrieveTasks(metadata, _, isTemplating) {
-        if (!metadata.Fields || !metadata.Fields.length) {
-            // assume we were in deploy mode. retrieve fields.
-            const tempList = {};
-            tempList[metadata[this.definition.keyField]] = metadata;
-            await this._retrieveFieldsForSingleDe(tempList, metadata[this.definition.keyField]);
-        }
-        // if retrieving template, replace the name with customer key if that wasn't already the case
-        if (isTemplating) {
-            const warningMsg =
-                'Ensure that Queries that write into this DE are updated with the new key before deployment.';
-            this.overrideKeyWithName(metadata, warningMsg);
-        }
+    static async postRetrieveTasks(metadata) {
         // Error during deploy if SendableSubscriberField.Name = '_SubscriberKey' even though it is retrieved like that
         // Therefore map it to 'Subscriber Key'. Retrieving afterward still results in '_SubscriberKey'
-        if (
-            metadata.SendableSubscriberField &&
-            metadata.SendableSubscriberField.Name === '_SubscriberKey'
-        ) {
+        if (metadata.SendableSubscriberField?.Name === '_SubscriberKey') {
             metadata.SendableSubscriberField.Name = 'Subscriber Key';
         }
         return this._parseMetadata(JSON.parse(JSON.stringify(metadata)));
@@ -436,17 +471,17 @@ class DataExtension extends MetadataType {
 
     /**
      * Helper to retrieve Data Extension Fields
+     *
      * @private
-     * @param {Object} [options] options (e.g. continueRequest)
+     * @param {TYPE.SoapRequestParams} [options] options (e.g. continueRequest)
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise<DataExtensionField.DataExtensionFieldMap>} Promise of items
+     * @returns {Promise.<TYPE.DataExtensionFieldMap>} Promise of items
      */
     static async _retrieveFields(options, additionalFields) {
         if (!options) {
             // dont print this during updates or templating which retrieves fields DE-by-DE
-            Util.logger.info('- Caching dependent Metadata: dataExtensionField');
+            Util.logger.info(' - Caching dependent Metadata: dataExtensionField');
         }
-        DataExtensionField.cache = this.metadata;
         DataExtensionField.client = this.client;
         DataExtensionField.properties = this.properties;
 
@@ -455,10 +490,11 @@ class DataExtension extends MetadataType {
     }
     /**
      * helps retrieving fields during templating and deploy where we dont want the full list
+     *
      * @private
-     * @param {DataExtensionMap} metadata list of DEs
+     * @param {TYPE.DataExtensionMap} metadata list of DEs
      * @param {string} customerKey external key of single DE
-     * @returns {Promise<void>} updates are made directly to `metadata`
+     * @returns {Promise.<void>} -
      */
     static async _retrieveFieldsForSingleDe(metadata, customerKey) {
         const fieldOptions = {
@@ -470,34 +506,28 @@ class DataExtension extends MetadataType {
         };
         const fieldsObj = await this._retrieveFields(fieldOptions);
 
-        DataExtensionField.cache = this.metadata;
         DataExtensionField.client = this.client;
         DataExtensionField.properties = this.properties;
         const fieldArr = DataExtensionField.convertToSortedArray(fieldsObj);
 
         // remove attributes that we do not want to retrieve
         // * do this after sorting on the DE's field list
-        fieldArr.forEach((field) => {
+        for (const field of fieldArr) {
             DataExtensionField.postRetrieveTasks(field, true);
-        });
+        }
 
         metadata[customerKey].Fields = fieldArr;
     }
 
     /**
      * prepares a DataExtension for deployment
-     * @param {DataExtensionItem} metadata a single data Extension
-     * @returns {Promise<DataExtensionItem>} Promise of updated single DE
+     *
+     * @param {TYPE.DataExtensionItem} metadata a single data Extension
+     * @returns {Promise.<TYPE.DataExtensionItem>} Promise of updated single DE
      */
     static async preDeployTasks(metadata) {
         // folder
-        metadata.CategoryID = Util.getFromCache(
-            this.cache,
-            'folder',
-            metadata.r__folder_Path,
-            'Path',
-            'ID'
-        );
+        metadata.CategoryID = cache.searchForField('folder', metadata.r__folder_Path, 'Path', 'ID');
         delete metadata.r__folder_Path;
 
         // DataExtensionTemplate
@@ -519,8 +549,7 @@ class DataExtension extends MetadataType {
             // get template's CustomerKey
             try {
                 metadata.Template = {
-                    CustomerKey: Util.getFromCache(
-                        this.cache,
+                    CustomerKey: cache.searchForField(
                         'dataExtensionTemplate',
                         metadata.r__dataExtensionTemplate_Name,
                         'Name',
@@ -561,9 +590,10 @@ class DataExtension extends MetadataType {
     /**
      * Saves json content to a html table in the local file system. Will create the parent directory if it does not exist.
      * The json's first level of keys must represent the rows and the secend level the columns
+     *
      * @private
-     * @param {DataExtensionItem} json single dataextension
-     * @param {Array} tabled prepped array for output in tabular format
+     * @param {TYPE.DataExtensionItem} json single dataextension
+     * @param {object[][]} tabled prepped array for output in tabular format
      * @returns {string} file content
      */
     static _generateDocHtml(json, tabled) {
@@ -577,37 +607,34 @@ class DataExtension extends MetadataType {
 
         output += `<p><b>Description:</b> ${json.Description || 'n/a'}</p>`;
         output += `<p><b>Folder:</b> ${
-            json.r__folder_Path
-                ? json.r__folder_Path
-                : '<i>Hidden! Could not find folder with ID ' + json.CategoryID + '</i>'
+            json.r__folder_Path ||
+            '<i>Hidden! Could not find folder with ID ' + json.CategoryID + '</i>'
         }</p>`;
         output += `<p><b>Fields in table:</b> ${tabled.length - 1}</p>`;
         output += '<p><b>Sendable:</b> ';
-        if (json.IsSendable === 'true') {
-            output +=
-                'Yes (<i>' +
-                json.SendableDataExtensionField.Name +
-                '</i> to <i>' +
-                json.SendableSubscriberField.Name +
-                '</i>)</p>\n\n';
-        } else {
-            output += `No</p>\n\n`;
-        }
-        output += `<p><b>Testable:</b> ${json.IsTestable === 'true' ? 'Yes' : 'No'}</p>\n\n`;
+        output +=
+            json.IsSendable === true
+                ? 'Yes (<i>' +
+                  json.SendableDataExtensionField.Name +
+                  '</i> to <i>' +
+                  json.SendableSubscriberField.Name +
+                  '</i>)</p>\n\n'
+                : `No</p>\n\n`;
+        output += `<p><b>Testable:</b> ${json.IsTestable === true ? 'Yes' : 'No'}</p>\n\n`;
         if (json.r__dataExtensionTemplate_Name) {
             output += `<p><b>Template:</b> ${json.r__dataExtensionTemplate_Name}</p>`;
         }
 
         output += '<table><thead><tr>';
-        tabled[0].forEach((element) => {
+        for (const element of tabled[0]) {
             output += '<th>' + element + '</th>';
-        });
+        }
         output += '</tr><thead><tbody>';
         for (let i = 1; i < tabled.length; i++) {
             output += '<tr>';
-            tabled[i].forEach((field) => {
+            for (const field of tabled[i]) {
                 output += `<td>${field}</td>`;
-            });
+            }
             output += '</tr>';
         }
         output += '</tbody></table>';
@@ -618,9 +645,10 @@ class DataExtension extends MetadataType {
      * Experimental: Only working for DataExtensions:
      * Saves json content to a html table in the local file system. Will create the parent directory if it does not exist.
      * The json's first level of keys must represent the rows and the secend level the columns
+     *
      * @private
-     * @param {DataExtensionItem} json dataextension
-     * @param {Array} tabled prepped array for output in tabular format
+     * @param {TYPE.DataExtensionItem} json dataextension
+     * @param {object[][]} tabled prepped array for output in tabular format
      * @returns {string} file content
      */
     static _generateDocMd(json, tabled) {
@@ -632,37 +660,34 @@ class DataExtension extends MetadataType {
         output +=
             `**Description:** ${json.Description || 'n/a'}\n\n` +
             `**Folder:** ${
-                json.r__folder_Path
-                    ? json.r__folder_Path
-                    : '_Hidden! Could not find folder with ID ' + json.CategoryID + '_'
+                json.r__folder_Path ||
+                '_Hidden! Could not find folder with ID ' + json.CategoryID + '_'
             }/\n\n` +
             `**Fields in table:** ${tabled.length - 1}\n\n`;
         output += '**Sendable:** ';
-        if (json.IsSendable === 'true') {
-            output +=
-                'Yes (`' +
-                json.SendableDataExtensionField.Name +
-                '` to `' +
-                json.SendableSubscriberField.Name +
-                '`)\n\n';
-        } else {
-            output += `No\n\n`;
-        }
-        output += `**Testable:** ${json.IsTestable === 'true' ? 'Yes' : 'No'}\n\n`;
+        output +=
+            json.IsSendable === true
+                ? 'Yes (`' +
+                  json.SendableDataExtensionField.Name +
+                  '` to `' +
+                  json.SendableSubscriberField.Name +
+                  '`)\n\n'
+                : `No\n\n`;
+        output += `**Testable:** ${json.IsTestable === true ? 'Yes' : 'No'}\n\n`;
         if (json.r__dataExtensionTemplate_Name) {
             output += `**Template:** ${json.r__dataExtensionTemplate_Name}\n\n`;
         }
 
         let tableSeparator = '';
-        tabled[0].forEach((column) => {
+        for (const column of tabled[0]) {
             output += `| ${column} `;
             tableSeparator += '| --- ';
-        });
+        }
         output += `|\n${tableSeparator}|\n`;
         for (let i = 1; i < tabled.length; i++) {
-            tabled[i].forEach((field) => {
+            for (const field of tabled[i]) {
                 output += `| ${field} `;
-            });
+            }
             output += '|\n';
         }
         return output;
@@ -671,28 +696,26 @@ class DataExtension extends MetadataType {
     /**
      * Saves json content to a html table in the local file system. Will create the parent directory if it does not exist.
      * The json's first level of keys must represent the rows and the secend level the columns
+     *
      * @private
      * @param {string} directory directory the file will be written to
      * @param {string} filename name of the file without '.json' ending
-     * @param {DataExtensionItem} json dataextension.columns
+     * @param {TYPE.DataExtensionItem} json dataextension.columns
      * @param {'html'|'md'} mode html or md
      * @param {string[]} [fieldsToKeep] list of keys(columns) to show. This will also specify
-     * @returns {Promise<boolean>} Promise of success of saving the file
+     * @returns {Promise.<boolean>} Promise of success of saving the file
      */
     static async _writeDoc(directory, filename, json, mode, fieldsToKeep) {
-        if (!File.existsSync(directory)) {
-            File.mkdirpSync(directory);
-        }
         let fieldsJson = Object.values(json.Fields);
         if (fieldsToKeep) {
             const newJson = [];
-            fieldsJson.forEach((element) => {
+            for (const element of fieldsJson) {
                 const newJsonElement = {};
-                fieldsToKeep.forEach((field) => {
+                for (const field of fieldsToKeep) {
                     newJsonElement[field] = element[field];
-                });
+                }
                 newJson.push(newJsonElement);
-            });
+            }
             fieldsJson = newJson;
         }
         const tabled = jsonToTable(fieldsJson);
@@ -704,41 +727,44 @@ class DataExtension extends MetadataType {
         }
         try {
             // write to disk
-            await File.writeToFile(directory, filename + '.dataExtension', mode, output);
+            await File.writeToFile(directory, filename + '.dataExtension-doc', mode, output);
         } catch (ex) {
             Util.logger.error(`DataExtension.writeDeToX(${mode}):: error | ` + ex.message);
         }
     }
     /**
      * Parses metadata into a readable Markdown/HTML format then saves it
-     * @param {Util.BuObject} buObject properties for auth
-     * @param {DataExtensionMap} [metadata] a list of dataExtension definitions
-     * @param {boolean} [isDeploy] used to skip non-supported message during deploy
-     * @returns {Promise<void>} -
+     *
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @param {TYPE.DataExtensionMap} [metadata] a list of dataExtension definitions
+     * @returns {Promise.<void>} -
      */
-    static async document(buObject, metadata, isDeploy) {
-        if (!metadata) {
-            metadata = this.readBUMetadataForType(
-                File.normalizePath([
-                    this.properties.directories.retrieve,
-                    buObject.credential,
-                    buObject.businessUnit,
-                ]),
-                true
-            ).dataExtension;
+    static async document(buObject, metadata) {
+        try {
+            if (!metadata) {
+                metadata = this.readBUMetadataForType(
+                    File.normalizePath([
+                        this.properties.directories.retrieve,
+                        buObject.credential,
+                        buObject.businessUnit,
+                    ]),
+                    true
+                ).dataExtension;
+            }
+        } catch (ex) {
+            Util.logger.error(ex.message);
+            return;
         }
         const docPath = File.normalizePath([
-            this.properties.directories.dataExtension,
+            this.properties.directories.retrieve,
             buObject.credential,
             buObject.businessUnit,
+            this.definition.type,
         ]);
         if (!metadata || !Object.keys(metadata).length) {
             // as part of retrieve & manual execution we could face an empty folder
             return;
         }
-        if (!isDeploy) {
-            File.removeSync(docPath);
-        }
         const columnsToIterateThrough = ['IsNullable', 'IsPrimaryKey'];
         const columnsToPrint = [
             'Name',
@@ -748,130 +774,101 @@ class DataExtension extends MetadataType {
             'IsNullable',
             'DefaultValue',
         ];
-        for (const customerKey in metadata) {
-            if (metadata[customerKey].Fields && metadata[customerKey].Fields.length) {
-                metadata[customerKey].Fields.forEach((field) => {
-                    field.IsNullable = (!(field.IsRequired === 'true')).toString();
-                    columnsToIterateThrough.forEach((key) => {
-                        if (field[key] === 'true') {
-                            field[key] = '+';
-                        } else if (field[key] === 'false') {
-                            field[key] = '-';
+        return Promise.all(
+            Object.keys(metadata).map((customerKey) => {
+                // for (const customerKey in metadata) {
+                if (metadata[customerKey]?.Fields?.length) {
+                    for (const field of metadata[customerKey].Fields) {
+                        field.IsNullable = !Util.isTrue(field.IsRequired);
+                        for (const key of columnsToIterateThrough) {
+                            if (Util.isTrue(field[key])) {
+                                field[key] = '+';
+                            } else if (Util.isFalse(field[key])) {
+                                field[key] = '-';
+                            }
                         }
-                    });
-                });
+                    }
 
-                if (['html', 'both'].includes(this.properties.options.documentType)) {
-                    this._writeDoc(
-                        docPath + '/',
-                        customerKey,
-                        metadata[customerKey],
-                        'html',
-                        columnsToPrint
-                    );
-                }
-                if (['md', 'both'].includes(this.properties.options.documentType)) {
-                    this._writeDoc(
-                        docPath + '/',
-                        customerKey,
-                        metadata[customerKey],
-                        'md',
-                        columnsToPrint
-                    );
+                    if (['html', 'both'].includes(this.properties.options.documentType)) {
+                        return this._writeDoc(
+                            docPath + '/',
+                            customerKey,
+                            metadata[customerKey],
+                            'html',
+                            columnsToPrint
+                        );
+                    }
+                    if (['md', 'both'].includes(this.properties.options.documentType)) {
+                        return this._writeDoc(
+                            docPath + '/',
+                            customerKey,
+                            metadata[customerKey],
+                            'md',
+                            columnsToPrint
+                        );
+                    }
                 }
-            }
-        }
-        if (['html', 'both'].includes(this.properties.options.documentType)) {
-            Util.logger.info(`Created ${docPath}/*.dataExtension.html`);
-        }
-        if (['md', 'both'].includes(this.properties.options.documentType)) {
-            Util.logger.info(`Created ${docPath}/*.dataExtension.md`);
-        }
+            })
+        );
     }
 
     /**
-     * Delete a data extension from the specified business unit
-     * @param {Object} buObject references credentials
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {TYPE.BuObject} buObject references credentials
      * @param {string} customerKey Identifier of data extension
-     * @returns {Promise<void>} -
+     * @returns {Promise.<boolean>} deletion success status
      */
-    static async deleteByKey(buObject, customerKey) {
-        let client;
-        try {
-            client = await Util.getETClient(buObject);
-        } catch (ex) {
-            Util.logger.error(ex.message);
-            return;
-        }
-        const config = {
-            props: { CustomerKey: customerKey },
-        };
+    static deleteByKey(buObject, customerKey) {
+        return super.deleteByKeySOAP(buObject, customerKey, false);
+    }
 
-        client.dataExtension(config).delete((error, response) => {
-            if (error && error.results && error.results[0]) {
-                if (error.results[0].ErrorCode === '310007') {
-                    Util.logger.error(
-                        'mcdev.deleteDE:: It seems the DataExtension you were trying to delete does not exist on the given BU or its External Key was changed.'
-                    );
-                } else {
-                    Util.logger.error('mcdev.deleteDE:: ' + error.results[0].StatusMessage);
-                }
-            } else if (error) {
-                Util.logger.error('mcdev.deleteDE:: ' + JSON.stringify(error));
-            } else if (
-                response &&
-                response.body &&
-                response.body.Results &&
-                response.body.Results[0]
-            ) {
-                Util.logger.info(
-                    `mcdev.deleteDE:: Success: ${response.body.Results[0].StatusMessage} (${customerKey})`
-                );
-                // delete local copy: retrieve/cred/bu/dataExtension/...json
-                const jsonFile = File.normalizePath([
-                    this.properties.directories.retrieve,
-                    buObject.credential,
-                    buObject.businessUnit,
-                    this.definition.type,
-                    `${customerKey}.${this.definition.type}-meta.json`,
-                ]);
-                if (File.existsSync(jsonFile)) {
-                    File.unlinkSync(jsonFile);
-                }
-                // delete local copy: doc/dataExtension/cred/bu/...md
-                const mdFile = File.normalizePath([
-                    this.properties.directories.dataExtension,
-                    buObject.credential,
-                    buObject.businessUnit,
-                    `${customerKey}.${this.definition.type}.md`,
-                ]);
-                if (File.existsSync(mdFile)) {
-                    File.unlinkSync(mdFile);
-                }
-            } else {
-                Util.logger.info('mcdev.deleteDE:: Success: ' + JSON.stringify(response));
-            }
-        });
+    /**
+     * clean up after deleting a metadata item
+     *
+     * @param {TYPE.BuObject} buObject references credentials
+     * @param {string} customerKey Identifier of metadata item
+     * @returns {void}
+     */
+    static async postDeleteTasks(buObject, customerKey) {
+        // delete local copy: retrieve/cred/bu/dataExtension/...json
+        const jsonFile = File.normalizePath([
+            this.properties.directories.retrieve,
+            buObject.credential,
+            buObject.businessUnit,
+            this.definition.type,
+            `${customerKey}.${this.definition.type}-meta.json`,
+        ]);
+        await File.remove(jsonFile);
+        // delete local copy: doc/dataExtension/cred/bu/...md
+        const mdFile = File.normalizePath([
+            this.properties.directories.docs,
+            'dataExtension',
+            buObject.credential,
+            buObject.businessUnit,
+            `${customerKey}.${this.definition.type}.md`,
+        ]);
+        await File.remove(mdFile);
     }
 
     /**
      * Retrieves folder metadata into local filesystem. Also creates a uniquePath attribute for each folder.
-     * @param {Object} buObject properties for auth
-     * @param {void} [_] -
-     * @param {boolean} [isDeploy] used to signal that fields shall be retrieve in caching mode
-     * @returns {Promise} Promise
+     *
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise
      */
-    static async retrieveForCache(buObject, _, isDeploy) {
-        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], buObject, null, isDeploy);
+    static async retrieveForCache(buObject) {
+        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], buObject, null, null);
     }
     /**
      * Retrieves dataExtension metadata in template format.
+     *
      * @param {string} templateDir Directory where retrieved metadata directory will be saved
      * @param {string} name name of the metadata item
-     * @param {Util.TemplateMap} variables variables to be replaced in the metadata
-     * @returns {Promise<{metadata:DataExtensionMap,type:string}>} Promise of items
+     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of items
      */
-    static async retrieveAsTemplate(templateDir, name, variables) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables) {
         const options = {
             filter: {
                 leftOperand: 'Name',
@@ -879,8 +876,7 @@ class DataExtension extends MetadataType {
                 rightOperand: name,
             },
         };
-        /** @type DataExtensionMap */
-        // const metadata = super.parseResponseBody(response.body);
+
         const metadata = await this._retrieveAll(null, options);
 
         if (!Object.keys(metadata).length) {
@@ -896,46 +892,42 @@ class DataExtension extends MetadataType {
                 // API returns field unsorted
                 metadata[key].Fields.sort((a, b) => a.Ordinal - b.Ordinal);
 
+                const originalKey = key;
                 const metadataCleaned = JSON.parse(
-                    JSON.stringify(await this.postRetrieveTasks(metadata[key], null, !!variables))
+                    JSON.stringify(await this.postRetrieveTasks(metadata[key]))
                 );
 
                 this.keepTemplateFields(metadataCleaned);
                 const metadataTemplated = JSON.parse(
-                    Util.replaceByObject(JSON.stringify(metadataCleaned), variables)
+                    Util.replaceByObject(JSON.stringify(metadataCleaned), templateVariables)
                 );
-                File.writeJSONToFile(
+                await File.writeJSONToFile(
                     [templateDir, this.definition.type].join('/'),
-                    metadataTemplated[this.definition.keyField] +
-                        '.' +
-                        this.definition.type +
-                        '-meta',
+                    originalKey + '.' + this.definition.type + '-meta',
                     metadataTemplated
                 );
             } catch (ex) {
                 Util.metadataLogger('error', this.definition.type, 'retrieve', ex, key);
             }
         }
+        Util.logger.info(`- templated ${this.definition.type}: ${customerKey}`);
 
-        Util.logger.info(
-            `DataExtension.retrieveAsTemplate:: All records written to filesystem (${customerKey})`
-        );
-        return { metadata: metadata, type: 'dataExtension' };
+        return { metadata: metadata[customerKey], type: 'dataExtension' };
     }
 
     /**
      * parses retrieved Metadata before saving
+     *
      * @private
-     * @param {DataExtensionItem} metadata a single dataExtension definition
-     * @returns {DataExtensionItem} a single dataExtension definition
+     * @param {TYPE.DataExtensionItem} metadata a single dataExtension definition
+     * @returns {TYPE.DataExtensionItem} a single dataExtension definition
      */
     static _parseMetadata(metadata) {
         let error = false;
         let verbose = false;
         // data extension type (from folder)
         try {
-            metadata.r__folder_ContentType = Util.getFromCache(
-                this.cache,
+            metadata.r__folder_ContentType = cache.searchForField(
                 'folder',
                 metadata.CategoryID,
                 'ID',
@@ -947,13 +939,12 @@ class DataExtension extends MetadataType {
                 metadata.r__folder_ContentType = 'synchronizeddataextension';
             } else {
                 error = true;
-                Util.logger.warn(`Data Extension '${metadata.Name}': ${ex.message}`);
+                Util.logger.warn(` - dataExtension '${metadata.Name}': ${ex.message}`);
             }
         }
         // folder
         try {
-            metadata.r__folder_Path = Util.getFromCache(
-                this.cache,
+            metadata.r__folder_Path = cache.searchForField(
                 'folder',
                 metadata.CategoryID,
                 'ID',
@@ -975,10 +966,9 @@ class DataExtension extends MetadataType {
             }
         }
         // DataExtensionTemplate
-        if (metadata.Template && metadata.Template.CustomerKey) {
+        if (metadata.Template?.CustomerKey) {
             try {
-                metadata.r__dataExtensionTemplate_Name = Util.getFromCache(
-                    this.cache,
+                metadata.r__dataExtensionTemplate_Name = cache.searchForField(
                     'dataExtensionTemplate',
                     metadata.Template.CustomerKey,
                     'CustomerKey',
@@ -992,26 +982,29 @@ class DataExtension extends MetadataType {
                 // A workaround exists but it's likely not beneficial to explain to users:
                 // Create a DE based on the not-supported template on the target BU, retrieve it, copy the Template.CustomerKey into the to-be-deployed DE (or use mcdev-templating), done
                 Util.logger.warn(
-                    `Issue with DataExtension '${
+                    ` - Issue with dataExtension '${
                         metadata[this.definition.nameField]
                     }': Could not find specified DataExtension Template. Please note that DataExtensions based on SMSMessageTracking and SMSSubscriptionLog cannot be deployed automatically across BUs at this point.`
                 );
             }
         }
+        // remove the date fields manually here because we need them in the changelog but not in the saved json
+        delete metadata.CreatedDate;
+        delete metadata.ModifiedDate;
 
         return metadata;
     }
 
     /**
      * Retrieves dataExtension metadata and cleans it
+     *
      * @private
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Object} [options] e.g. filter
-     * @returns {Promise<DataExtensionMap>} keyField => metadata map
+     * @param {TYPE.SoapRequestParams} [options] e.g. filter
+     * @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
      */
     static async _retrieveAll(additionalFields, options) {
-        const metadata = (await super.retrieveSOAPgeneric(null, null, options, additionalFields))
-            .metadata;
+        const { metadata } = await super.retrieveSOAP(null, null, options, additionalFields);
         for (const key in metadata) {
             // some system data extensions do not have CategoryID which throws errors in other places. These do not need to be parsed
             if (!metadata[key].CategoryID) {
@@ -1022,14 +1015,36 @@ class DataExtension extends MetadataType {
         }
         return metadata;
     }
+    /**
+     * should return only the json for all but asset, query and script that are saved as multiple files
+     * additionally, the documentation for dataExtension and automation should be returned
+     *
+     * @param {string[]} keyArr customerkey of the metadata
+     * @returns {string[]} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
+     */
+    static getFilesToCommit(keyArr) {
+        if (!this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
+            // document dataExtension is not active upon retrieve, run default method instead
+            return super.getFilesToCommit(keyArr);
+        } else {
+            // document dataExtension is active. assume we want to commit the MD file as well
+            const path = File.normalizePath([
+                this.properties.directories.retrieve,
+                this.buObject.credential,
+                this.buObject.businessUnit,
+                this.definition.type,
+            ]);
+
+            const fileList = keyArr.flatMap((key) => [
+                File.normalizePath([path, `${key}.${this.definition.type}-meta.json`]),
+                File.normalizePath([path, `${key}.${this.definition.type}-doc.md`]),
+            ]);
+            return fileList;
+        }
+    }
 }
 
 // Assign definition to static attributes
 DataExtension.definition = require('../MetadataTypeDefinitions').dataExtension;
-/**
- * @type {Util.ET_Client}
- */
-DataExtension.client = undefined;
-DataExtension.cache = {};
 
 module.exports = DataExtension;