mcdev 6.0.2 → 7.0.0

package/lib/metadataTypes/DataExtension.js

@@ -1,7 +1,6 @@
 'use strict';
 
 import jsonToTable from 'json-to-table';
-import TYPE from '../../types/mcdev.d.js';
 import MetadataType from './MetadataType.js';
 import AttributeSet from './AttributeSet.js';
 import DataExtensionField from './DataExtensionField.js';
@@ -13,22 +12,40 @@ import cache from '../util/cache.js';
 import pLimit from 'p-limit';
 import inquirer from 'inquirer';
 
+/**
+ * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
+ * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ */
+/**
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldItem} DataExtensionFieldItem
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionFieldMap} DataExtensionFieldMap
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionItem} DataExtensionItem
+ * @typedef {import('../../types/mcdev.d.js').DataExtensionMap} DataExtensionMap
+ */
+
 /**
  * DataExtension MetadataType
  *
  * @augments MetadataType
  */
 class DataExtension extends MetadataType {
+    /** @type {DataExtensionFieldMap} */
+    static oldFields;
     /**
      * Upserts dataExtensions after retrieving them from source and target to compare
      * if create or update operation is needed.
      *
-     * @param {TYPE.DataExtensionMap} metadataMap dataExtensions mapped by their customerKey
+     * @param {DataExtensionMap} metadataMap dataExtensions mapped by their customerKey
      * @returns {Promise} Promise
      */
     static async upsert(metadataMap) {
         // get existing DE-fields for DE-keys in deployment package to properly handle add/update/delete of fields
         // we need to use IN here because it would fail otherwise if we try to deploy too many DEs at the same time
+        /** @type {SoapRequestParams} */
         const fieldOptions = {
             filter: {
                 leftOperand: 'DataExtension.CustomerKey',
@@ -88,9 +105,7 @@ class DataExtension extends MetadataType {
             await Promise.allSettled(
                 metadataToUpdate
                     .filter((r) => r !== undefined && r !== null)
-                    .map((metadataEntry) =>
-                        updateLimit(() => this.update(metadataEntry.after, metadataEntry.before))
-                    )
+                    .map((metadataEntry) => updateLimit(() => this.update(metadataEntry.after)))
             )
         )
             .filter((r) => r !== undefined && r !== null)
@@ -104,6 +119,7 @@ class DataExtension extends MetadataType {
         let upsertResults;
         if (successfulResults.length > 0) {
             const metadataResults = successfulResults
+                // @ts-expect-error not sure why this produces a type error. looks 100% correct
                 .map((r) => r.value.Results[0].Object)
                 .map((r) => {
                     // if only one fields added will return object otherwise array
@@ -128,12 +144,12 @@ class DataExtension extends MetadataType {
     /**
      * helper for {@link MetadataType.upsert}
      *
-     * @param {TYPE.MetadataTypeMap} metadataMap list of metadata
+     * @param {MetadataTypeMap} metadataMap list of metadata
      * @param {string} metadataKey key of item we are looking at
      * @param {boolean} hasError error flag from previous code
-     * @param {TYPE.MetadataTypeItemDiff[]} metadataToUpdate list of items to update
-     * @param {TYPE.MetadataTypeItem[]} metadataToCreate list of items to create
-     * @returns {'create' | 'update' | 'skip'} action to take
+     * @param {MetadataTypeItemDiff[]} metadataToUpdate list of items to update
+     * @param {MetadataTypeItem[]} metadataToCreate list of items to create
+     * @returns {Promise.<'create'|'update'|'skip'>} action to take
      */
     static async createOrUpdate(
         metadataMap,
@@ -156,6 +172,7 @@ class DataExtension extends MetadataType {
             DataExtensionField.client = this.client;
             DataExtensionField.properties = this.properties;
             DataExtension.oldFields ||= {};
+            // @ts-expect-error not sure why this produces a type error. looks 100% correct
             DataExtension.oldFields[metadataMap[metadataKey][this.definition.keyField]] =
                 await DataExtensionField.prepareDeployColumnsOnUpdate(
                     metadataMap[metadataKey].Fields,
@@ -193,11 +210,11 @@ class DataExtension extends MetadataType {
             // </Fields>
             metadataMap[metadataKey].Fields = { Field: metadataMap[metadataKey].Fields };
         }
+        return action;
     }
     /**
      * helper for {@link DataExtension.upsert}
      *
-     * @private
      * @param {object} res -
      * @returns {boolean} true: keep, false: discard
      */
@@ -230,7 +247,7 @@ class DataExtension extends MetadataType {
     /**
      * Create a single dataExtension. Also creates their columns in 'dataExtension.columns'
      *
-     * @param {TYPE.DataExtensionItem} metadata single metadata entry
+     * @param {DataExtensionItem} metadata single metadata entry
      * @returns {Promise} Promise
      */
     static async create(metadata) {
@@ -242,8 +259,7 @@ class DataExtension extends MetadataType {
      * if that field duplicates whats in the period fields
      * during deployment, that extra value is not accepted by the APIs which is why it needs to be removed
      *
-     * @private
-     * @param {TYPE.DataExtensionItem} metadata single metadata entry
+     * @param {DataExtensionItem} metadata single metadata entry
      * @returns {void}
      */
     static #cleanupRetentionPolicyFields(metadata) {
@@ -261,7 +277,7 @@ class DataExtension extends MetadataType {
     /**
      * Updates a single dataExtension. Also updates their columns in 'dataExtension.columns'
      *
-     * @param {TYPE.DataExtensionItem} metadata single metadata entry
+     * @param {DataExtensionItem} metadata single metadata entry
      * @returns {Promise} Promise
      */
     static async update(metadata) {
@@ -270,10 +286,10 @@ class DataExtension extends MetadataType {
     /**
      * Gets executed after deployment of metadata type
      *
-     * @param {TYPE.DataExtensionMap} upsertedMetadata metadata mapped by their keyField
-     * @param {TYPE.DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     * @param {DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
      * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
-     * @returns {void}
+     * @returns {Promise.<void>} -
      */
     static async postDeployTasks(upsertedMetadata, originalMetadata, createdUpdated) {
         for (const key in upsertedMetadata) {
@@ -295,6 +311,7 @@ class DataExtension extends MetadataType {
 
                 const existingFields = DataExtension.oldFields[item[this.definition.keyField]];
 
+                // @ts-expect-error Fields is a special case that cannot be properly typed; emtpy string is required for SOAP API
                 if (item.Fields === '') {
                     // if no fields were updated, we need to set Fields to "empty string" for the API to work
                     // reset here to get the correct field list
@@ -303,15 +320,15 @@ class DataExtension extends MetadataType {
                         .sort((a, b) => a.Ordinal - b.Ordinal);
                 } else if (existingFields) {
                     // get list of updated fields
-                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    /** @type {DataExtensionFieldItem[]} */ // @ts-ignore Fields.Field is a special case that cannot be properly typed; only required for SOAP API
                     const updatedFieldsArr = originalMetadata[oldKey].Fields.Field.filter(
                         (field) => field.ObjectID && field.ObjectID !== ''
                     );
                     // convert existing fields obj into array and sort
-                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    /** @type {DataExtensionFieldItem[]} */
                     const finalFieldsArr = Object.keys(existingFields)
                         .map((el) => {
-                            /** @type {TYPE.DataExtensionFieldItem} */
+                            /** @type {DataExtensionFieldItem} */
                             const existingField = existingFields[el];
                             // check if the current field was updated and then override with it. otherwise use existing value
                             const field =
@@ -327,7 +344,7 @@ class DataExtension extends MetadataType {
                         .sort((a, b) => a.Ordinal - b.Ordinal);
 
                     // get list of new fields
-                    /** @type {TYPE.DataExtensionFieldItem[]} */
+                    /** @type {DataExtensionFieldItem[]} */ // @ts-ignore Fields.Field is a special case that cannot be properly typed; only required for SOAP API
                     const newFieldsArr = originalMetadata[oldKey].Fields.Field.filter(
                         (field) => !field.ObjectID
                     );
@@ -343,7 +360,7 @@ class DataExtension extends MetadataType {
             }
             // UPDATE + CREATE
             for (const field of item.Fields) {
-                DataExtensionField.postRetrieveTasks(field, true);
+                DataExtensionField.postRetrieveTasksDE(field);
             }
         }
         await this.#fixShared(upsertedMetadata, originalMetadata, createdUpdated);
@@ -354,10 +371,10 @@ class DataExtension extends MetadataType {
      * helper for {@link DataExtension.postDeployTasks}
      * fixes an issue where shared data extensions are not visible in data designer on child BU; SF known issue: https://issues.salesforce.com/#q=W-11031095
      *
-     * @param {TYPE.DataExtensionMap} upsertedMetadata metadata mapped by their keyField
-     * @param {TYPE.DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     * @param {DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
      * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
-     * @returns {void}
+     * @returns {Promise.<void>} -
      */
     static async #fixShared(upsertedMetadata, originalMetadata, createdUpdated) {
         if (this.buObject.eid !== this.buObject.mid) {
@@ -391,6 +408,7 @@ class DataExtension extends MetadataType {
             const clientBak = this.client;
 
             // get dataExtension ID-Key relationship
+            /** @type {Object.<string, string>} */
             const sharedDataExtensionMap = {};
             for (const key of sharedDataExtensionsKeys) {
                 try {
@@ -434,7 +452,7 @@ class DataExtension extends MetadataType {
     /**
      * helper for {@link DataExtension.#fixShared}
      *
-     * @returns {string[]} list of selected BU names
+     * @returns {Promise.<string[]>} list of selected BU names
      */
     static async #fixShared_getBUs() {
         const buListObj = this.properties.credentials[this.buObject.credential].businessUnits;
@@ -467,13 +485,16 @@ class DataExtension extends MetadataType {
             value: name,
             checked: fixBuPreselected.includes(name),
         }));
-        const questions = {
-            type: 'checkbox',
-            name: 'businessUnits',
-            message: 'Please select BUs that have access to the updated Shared Data Extensions:',
-            pageSize: 10,
-            choices: buList,
-        };
+        const questions = [
+            {
+                type: 'checkbox',
+                name: 'businessUnits',
+                message:
+                    'Please select BUs that have access to the updated Shared Data Extensions:',
+                pageSize: 10,
+                choices: buList,
+            },
+        ];
         let responses = null;
 
         try {
@@ -488,7 +509,7 @@ class DataExtension extends MetadataType {
      * helper for {@link DataExtension.#fixShared}
      *
      * @param {string} childBuName name of child BU to fix
-     * @param {TYPE.BuObject} buObjectParent bu object for parent BU
+     * @param {BuObject} buObjectParent bu object for parent BU
      * @param {object} clientParent SDK for parent BU
      * @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
      * @returns {Promise.<string[]>} updated shared DE keys on BU
@@ -499,7 +520,7 @@ class DataExtension extends MetadataType {
         clientParent,
         sharedDataExtensionMap
     ) {
-        /** @type {TYPE.BuObject} */
+        /** @type {BuObject} */
         const buObjectChildBu = {
             eid: this.properties.credentials[buObjectParent.credential].eid,
             mid: this.properties.credentials[buObjectParent.credential].businessUnits[childBuName],
@@ -571,9 +592,9 @@ class DataExtension extends MetadataType {
      *
      * @param {string} deId data extension ObjectID
      * @param {string} deKey dataExtension key
-     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {BuObject} buObjectChildBu BU object for Child BU
      * @param {object} clientChildBu SDK for child BU
-     * @param {TYPE.BuObject} buObjectParent BU object for Parent BU
+     * @param {BuObject} buObjectParent BU object for Parent BU
      * @param {object} clientParent SDK for parent BU
      * @returns {Promise.<boolean>} flag that signals if the fix was successful
      */
@@ -628,7 +649,7 @@ class DataExtension extends MetadataType {
      * add a new field to the shared DE to trigger an update to the data model
      * helper for {@link DataExtension.#fixShared_item}
      *
-     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {BuObject} buObjectChildBu BU object for Child BU
      * @param {object} clientChildBu SDK for child BU
      * @param {string} deKey dataExtension key
      * @param {string} deId dataExtension ObjectID
@@ -639,28 +660,26 @@ class DataExtension extends MetadataType {
         this.client = clientChildBu;
         const randomSuffix = Util.OPTIONS._runningTest
             ? '_randomNumber_'
-            : Math.floor(Math.random() * 9999999999);
+            : Math.floor(Math.random() * 9999999999).toString();
         // add a new field to the shared DE to trigger an update to the data model
         const soapType = this.definition.soapType || this.definition.type;
-        await this.client.soap.update(
-            Util.capitalizeFirstLetter(soapType),
-            {
-                CustomerKey: deKey,
-                ObjectID: deId,
-                Fields: {
-                    Field: [
-                        {
-                            Name: 'TriggerUpdate' + randomSuffix,
-                            IsRequired: false,
-                            IsPrimaryKey: false,
-                            FieldType: 'Boolean',
-                            ObjectID: null,
-                        },
-                    ],
-                },
+        const payload = {
+            CustomerKey: deKey,
+            ObjectID: deId,
+            Fields: {
+                Field: [
+                    {
+                        Name: 'TriggerUpdate' + randomSuffix,
+                        IsRequired: false,
+                        IsPrimaryKey: false,
+                        FieldType: 'Boolean',
+                        ObjectID: null,
+                    },
+                ],
             },
-            null
-        );
+        };
+        // @ts-expect-error SFMC-SDK typing error
+        await this.client.soap.update(Util.capitalizeFirstLetter(soapType), payload, null);
         return randomSuffix;
     }
 
@@ -669,7 +688,7 @@ class DataExtension extends MetadataType {
      * helper for {@link DataExtension.#fixShared_item}
      *
      * @param {string} randomSuffix -
-     * @param {TYPE.BuObject} buObjectParent BU object for Parent BU
+     * @param {BuObject} buObjectParent BU object for Parent BU
      * @param {object} clientParent SDK for parent BU
      * @param {string} deKey dataExtension key
      * @returns {Promise.<string>} fieldObjectID
@@ -678,7 +697,7 @@ class DataExtension extends MetadataType {
         DataExtensionField.buObject = buObjectParent;
         DataExtensionField.client = clientParent;
         const fieldKey = `[${deKey}].[TriggerUpdate${randomSuffix}]`;
-        const fieldResponse = await DataExtensionField.retrieveForCache(
+        const fieldResponse = await DataExtensionField.retrieveForCacheDE(
             {
                 filter: {
                     leftOperand: 'CustomerKey',
@@ -697,7 +716,7 @@ class DataExtension extends MetadataType {
      * helper for {@link DataExtension.#fixShared_item}
      *
      * @param {string} randomSuffix -
-     * @param {TYPE.BuObject} buObjectChildBu BU object for Child BU
+     * @param {BuObject} buObjectChildBu BU object for Child BU
      * @param {object} clientChildBu SDK for child BU
      * @param {string} deKey dataExtension key
      * @param {string} fieldObjectID field ObjectID
@@ -723,14 +742,14 @@ class DataExtension extends MetadataType {
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {void} [_] unused parameter
+     * @param {void | string[]} [_] unused parameter
      * @param {string} [key] customer key of single item to retrieve
-     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of item map
+     * @returns {Promise.<{metadata: DataExtensionMap, type: string}>} Promise of item map
      */
     static async retrieve(retrieveDir, additionalFields, _, key) {
-        /** @type {TYPE.SoapRequestParams} */
+        /** @type {SoapRequestParams} */
         let requestParams = null;
-        /** @type {TYPE.SoapRequestParams} */
+        /** @type {SoapRequestParams} */
         let fieldOptions = null;
         if (key) {
             requestParams = {
@@ -776,14 +795,14 @@ class DataExtension extends MetadataType {
      * helper for {@link DataExtension.retrieve} and for AttributeSet.fixShared_retrieve
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
+     * @returns {Promise.<DataExtensionMap>} keyField => metadata map
      */
     static async retrieveSharedForCache(additionalFields = []) {
         // for caching, we want to retrieve shared DEs as well from the instance parent BU
         Util.logger.info(' - Caching dependent Metadata: dataExtension (shared via _ParentBU_)');
         const buObjectBak = this.buObject;
         const clientBak = this.client;
-        /** @type {TYPE.BuObject} */
+        /** @type {BuObject} */
         const buObjectParentBu = {
             eid: this.properties.credentials[this.buObject.credential].eid,
             mid: this.properties.credentials[this.buObject.credential].eid,
@@ -849,9 +868,8 @@ class DataExtension extends MetadataType {
     /**
      * helper to retrieve all dataExtension fields and attach them to the dataExtension metadata
      *
-     * @private
-     * @param {TYPE.DataExtensionMap} metadata already cached dataExtension metadata
-     * @param {TYPE.SoapRequestParams} [fieldOptions] optionally filter results
+     * @param {DataExtensionMap} metadata already cached dataExtension metadata
+     * @param {SoapRequestParams} [fieldOptions] optionally filter results
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @returns {Promise.<void>} -
      */
@@ -877,7 +895,7 @@ class DataExtension extends MetadataType {
         // remove attributes that we do not want to retrieve
         // * do this after sorting on the DE's field list
         for (const key of fieldKeys) {
-            DataExtensionField.postRetrieveTasks(fieldsObj[key], true);
+            DataExtensionField.postRetrieveTasksDE(fieldsObj[key]);
         }
     }
 
@@ -885,7 +903,7 @@ class DataExtension extends MetadataType {
      * Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionColumn metadata retrieval
      *
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of item map
+     * @returns {Promise.<{metadata: DataExtensionMap, type: string}>} Promise of item map
      */
     static async retrieveChangelog(additionalFields) {
         const metadata = await this._retrieveAll(additionalFields);
@@ -894,8 +912,8 @@ class DataExtension extends MetadataType {
     /**
      * manages post retrieve steps
      *
-     * @param {TYPE.DataExtensionItem} metadata a single dataExtension
-     * @returns {TYPE.DataExtensionItem} metadata
+     * @param {DataExtensionItem} metadata a single dataExtension
+     * @returns {Promise.<DataExtensionItem>} metadata
      */
     static async postRetrieveTasks(metadata) {
         // Error during deploy if SendableSubscriberField.Name = '_SubscriberKey' even though it is retrieved like that
@@ -908,7 +926,7 @@ class DataExtension extends MetadataType {
         // DataExtensionTemplate
         if (metadata.Template?.CustomerKey) {
             try {
-                metadata.r__dataExtensionTemplate_Name = cache.searchForField(
+                metadata.r__dataExtensionTemplate_name = cache.searchForField(
                     'dataExtensionTemplate',
                     metadata.Template.CustomerKey,
                     'CustomerKey',
@@ -932,6 +950,52 @@ class DataExtension extends MetadataType {
         delete metadata.CreatedDate;
         delete metadata.ModifiedDate;
 
+        // Retention policy
+        if (
+            metadata.RowBasedRetention === false &&
+            metadata.DeleteAtEndOfRetentionPeriod === false &&
+            metadata.RetainUntil === ''
+        ) {
+            // Note: RetainUntil expected to NOT have a value
+            metadata.c__retentionPolicy = 'none';
+            delete metadata.RetainUntil;
+            delete metadata.ResetRetentionPeriodOnImport;
+        } else if (
+            metadata.RowBasedRetention === false &&
+            metadata.DeleteAtEndOfRetentionPeriod === false
+        ) {
+            // Note: RetainUntil expected to have a value
+            metadata.c__retentionPolicy = 'allRecordsAndDataextension';
+        } else if (
+            metadata.RowBasedRetention === false &&
+            metadata.DeleteAtEndOfRetentionPeriod === true
+        ) {
+            // Note: RetainUntil expected to have a value
+            metadata.c__retentionPolicy = 'allRecords';
+        } else if (
+            metadata.RowBasedRetention === true &&
+            metadata.DeleteAtEndOfRetentionPeriod === false
+        ) {
+            // Note: RetainUntil expected to NOT have a value
+            metadata.c__retentionPolicy = 'individialRecords';
+            delete metadata.RetainUntil;
+        }
+        delete metadata.RowBasedRetention;
+        delete metadata.DeleteAtEndOfRetentionPeriod;
+
+        if (metadata.RetainUntil) {
+            const retainUntil = new Date(metadata.RetainUntil);
+            metadata.c__retainUntil = `${retainUntil.getFullYear()}-${retainUntil.getMonth() + 1}-${retainUntil.getDate()}`;
+        }
+        delete metadata.RetainUntil;
+        if (metadata.DataRetentionPeriodUnitOfMeasure) {
+            metadata.c__dataRetentionPeriodUnitOfMeasure = Util.inverseGet(
+                this.definition.dataRetentionPeriodUnitOfMeasureMapping,
+                metadata.DataRetentionPeriodUnitOfMeasure
+            );
+            delete metadata.DataRetentionPeriodUnitOfMeasure;
+        }
+
         return metadata;
     }
 
@@ -939,9 +1003,9 @@ class DataExtension extends MetadataType {
      * Helper to retrieve Data Extension Fields
      *
      * @private
-     * @param {TYPE.SoapRequestParams} [options] options (e.g. continueRequest)
+     * @param {SoapRequestParams} [options] options (e.g. continueRequest)
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @returns {Promise.<TYPE.DataExtensionFieldMap>} Promise of items
+     * @returns {Promise.<DataExtensionFieldMap>} Promise of items
      */
     static async _retrieveFields(options, additionalFields) {
         if (!options) {
@@ -951,18 +1015,19 @@ class DataExtension extends MetadataType {
         DataExtensionField.client = this.client;
         DataExtensionField.properties = this.properties;
 
-        const response = await DataExtensionField.retrieveForCache(options, additionalFields);
+        const response = await DataExtensionField.retrieveForCacheDE(options, additionalFields);
         return response.metadata;
     }
     /**
      * helps retrieving fields during templating and deploy where we dont want the full list
      *
      * @private
-     * @param {TYPE.DataExtensionMap} metadata list of DEs
+     * @param {DataExtensionMap} metadata list of DEs
      * @param {string} customerKey external key of single DE
      * @returns {Promise.<void>} -
      */
     static async _retrieveFieldsForSingleDe(metadata, customerKey) {
+        /** @type {SoapRequestParams} */
         const fieldOptions = {
             filter: {
                 leftOperand: 'DataExtension.CustomerKey',
@@ -979,7 +1044,7 @@ class DataExtension extends MetadataType {
         // remove attributes that we do not want to retrieve
         // * do this after sorting on the DE's field list
         for (const field of fieldArr) {
-            DataExtensionField.postRetrieveTasks(field, true);
+            DataExtensionField.postRetrieveTasksDE(field);
         }
 
         metadata[customerKey].Fields = fieldArr;
@@ -987,19 +1052,18 @@ class DataExtension extends MetadataType {
     /**
      * helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP} that removes old files after the key was changed
      *
-     * @private
-     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
-     * @returns {void}
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     * @returns {Promise.<void>} -
      */
     static async _postChangeKeyTasks(metadataEntry) {
-        super._postChangeKeyTasks(metadataEntry, true);
+        return super._postChangeKeyTasks(metadataEntry, true);
     }
 
     /**
      * prepares a DataExtension for deployment
      *
-     * @param {TYPE.DataExtensionItem} metadata a single data Extension
-     * @returns {Promise.<TYPE.DataExtensionItem>} Promise of updated single DE
+     * @param {DataExtensionItem} metadata a single data Extension
+     * @returns {Promise.<DataExtensionItem>} Promise of updated single DE
      */
     static async preDeployTasks(metadata) {
         if (metadata.Name?.startsWith('_')) {
@@ -1026,10 +1090,10 @@ class DataExtension extends MetadataType {
         super.setFolderId(metadata);
 
         // DataExtensionTemplate
-        if (metadata.r__dataExtensionTemplate_Name) {
+        if (metadata.r__dataExtensionTemplate_name) {
             // remove templated fields
             for (const templateField of this.definition.templateFields[
-                metadata.r__dataExtensionTemplate_Name
+                metadata.r__dataExtensionTemplate_name
             ]) {
                 for (let index = 0; index < metadata.Fields.length; index++) {
                     const element = metadata.Fields[index];
@@ -1046,15 +1110,15 @@ class DataExtension extends MetadataType {
                 metadata.Template = {
                     CustomerKey: cache.searchForField(
                         'dataExtensionTemplate',
-                        metadata.r__dataExtensionTemplate_Name,
+                        metadata.r__dataExtensionTemplate_name,
                         'Name',
                         'CustomerKey'
                     ),
                 };
-                delete metadata.r__dataExtensionTemplate_Name;
+                delete metadata.r__dataExtensionTemplate_name;
             } catch (ex) {
                 Util.logger.debug(ex.message);
-                // It is assumed that non-supported types would not have been converted to r__dataExtensionTemplate_Name upon retrieve.
+                // It is assumed that non-supported types would not have been converted to r__dataExtensionTemplate_name upon retrieve.
                 // Deploying to same BU therefore still works!
                 // A workaround for cross-BU deploy exists but it's likely not beneficial to explain to users:
                 // Create a DE based on the not-supported template on the target BU, retrieve it, copy the Template.CustomerKey into the to-be-deployed DE (or use mcdev-templating), done
@@ -1077,6 +1141,55 @@ class DataExtension extends MetadataType {
             metadata.SendableSubscriberField.Name = 'Subscriber Key';
         }
 
+        // Retention policy
+        switch (metadata.c__retentionPolicy) {
+            case 'none': {
+                metadata.RowBasedRetention = false;
+                metadata.DeleteAtEndOfRetentionPeriod = false;
+                metadata.ResetRetentionPeriodOnImport = false;
+                break;
+            }
+            case 'allRecordsAndDataextension': {
+                metadata.RowBasedRetention = false;
+                metadata.DeleteAtEndOfRetentionPeriod = false;
+                break;
+            }
+            case 'allRecords': {
+                metadata.RowBasedRetention = false;
+                metadata.DeleteAtEndOfRetentionPeriod = true;
+                break;
+            }
+            case 'individialRecords': {
+                metadata.RowBasedRetention = true;
+                metadata.DeleteAtEndOfRetentionPeriod = false;
+                break;
+            }
+            default: {
+                Util.logger.debug(
+                    ` - dataExtension '${
+                        metadata[this.definition.nameField]
+                    }': Unknown retention policy: ${metadata.c__retentionPolicy}`
+                );
+            }
+        }
+        delete metadata.c__retentionPolicy;
+
+        if (metadata.c__retainUntil) {
+            const retainUntil = new Date(metadata.c__retainUntil);
+            metadata.RetainUntil = `${retainUntil.getMonth() + 1}/${retainUntil.getDate()}/${retainUntil.getFullYear()} 12:00:00 AM`;
+            delete metadata.c__retainUntil;
+        } else {
+            metadata.RetainUntil = '';
+        }
+
+        if (metadata.c__dataRetentionPeriodUnitOfMeasure) {
+            metadata.DataRetentionPeriodUnitOfMeasure =
+                this.definition.dataRetentionPeriodUnitOfMeasureMapping[
+                    metadata.c__dataRetentionPeriodUnitOfMeasure
+                ];
+            delete metadata.c__dataRetentionPeriodUnitOfMeasure;
+        }
+
         return metadata;
     }
 
@@ -1085,7 +1198,7 @@ class DataExtension extends MetadataType {
      * The json's first level of keys must represent the rows and the secend level the columns
      *
      * @private
-     * @param {TYPE.DataExtensionItem} json single dataextension
+     * @param {DataExtensionItem} json single dataextension
      * @param {object[][]} tabled prepped array for output in tabular format
      * @returns {string} file content
      */
@@ -1114,8 +1227,8 @@ class DataExtension extends MetadataType {
                   '</i>)</p>\n\n'
                 : `No</p>\n\n`;
         output += `<p><b>Testable:</b> ${json.IsTestable === true ? 'Yes' : 'No'}</p>\n\n`;
-        if (json.r__dataExtensionTemplate_Name) {
-            output += `<p><b>Template:</b> ${json.r__dataExtensionTemplate_Name}</p>`;
+        if (json.r__dataExtensionTemplate_name) {
+            output += `<p><b>Template:</b> ${json.r__dataExtensionTemplate_name}</p>`;
         }
 
         output += '<table><thead><tr>';
@@ -1140,7 +1253,7 @@ class DataExtension extends MetadataType {
      * The json's first level of keys must represent the rows and the secend level the columns
      *
      * @private
-     * @param {TYPE.DataExtensionItem} json dataextension
+     * @param {DataExtensionItem} json dataextension
      * @param {object[][]} tabled prepped array for output in tabular format
      * @returns {string} file content
      */
@@ -1167,8 +1280,8 @@ class DataExtension extends MetadataType {
                   '`)\n\n'
                 : `No\n\n`;
         output += `**Testable:** ${json.IsTestable === true ? 'Yes' : 'No'}\n\n`;
-        if (json.r__dataExtensionTemplate_Name) {
-            output += `**Template:** ${json.r__dataExtensionTemplate_Name}\n\n`;
+        if (json.r__dataExtensionTemplate_name) {
+            output += `**Template:** ${json.r__dataExtensionTemplate_name}\n\n`;
         }
 
         let tableSeparator = '';
@@ -1193,14 +1306,15 @@ class DataExtension extends MetadataType {
      * @private
      * @param {string} directory directory the file will be written to
      * @param {string} filename name of the file without '.json' ending
-     * @param {TYPE.DataExtensionItem} json dataextension.columns
+     * @param {DataExtensionItem} json dataextension.columns
      * @param {'html'|'md'} mode html or md
      * @param {string[]} [fieldsToKeep] list of keys(columns) to show. This will also specify
-     * @returns {Promise.<boolean>} Promise of success of saving the file
+     * @returns {Promise.<void>} Promise of success of saving the file
      */
     static async _writeDoc(directory, filename, json, mode, fieldsToKeep) {
         let fieldsJson = Object.values(json.Fields);
         if (fieldsToKeep) {
+            /** @type {DataExtensionFieldItem[]} */
             const newJson = [];
             for (const element of fieldsJson) {
                 const newJsonElement = {};
@@ -1211,6 +1325,7 @@ class DataExtension extends MetadataType {
                         newJsonElement[field] = element[field];
                     }
                 }
+                // @ts-ignore hard to tell ts that this is in fact a DataExtensionFieldItem
                 newJson.push(newJsonElement);
             }
             fieldsJson = newJson;
@@ -1232,13 +1347,13 @@ class DataExtension extends MetadataType {
     /**
      * Parses metadata into a readable Markdown/HTML format then saves it
      *
-     * @param {TYPE.DataExtensionMap} [metadataMap] a list of dataExtension definitions
-     * @returns {Promise.<void>} -
+     * @param {DataExtensionMap} [metadataMap] a list of dataExtension definitions
+     * @returns {Promise.<any>} -
      */
     static async document(metadataMap) {
         try {
             if (!metadataMap) {
-                metadataMap = this.readBUMetadataForType(
+                metadataMap = await this.readBUMetadataForType(
                     File.normalizePath([
                         this.properties.directories.retrieve,
                         this.buObject.credential,
@@ -1332,20 +1447,21 @@ class DataExtension extends MetadataType {
     /**
      * Retrieves folder metadata into local filesystem. Also creates a uniquePath attribute for each folder.
      *
-     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise
+     * @returns {Promise.<{metadata: DataExtensionMap, type: string}>} Promise
      */
     static async retrieveForCache() {
-        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], this.buObject, null, null);
+        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name']);
     }
     /**
      * Retrieves dataExtension metadata in template format.
      *
      * @param {string} templateDir Directory where retrieved metadata directory will be saved
      * @param {string} name name of the metadata item
-     * @param {TYPE.TemplateMap} templateVariables variables to be replaced in the metadata
-     * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise of items
+     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
+     * @returns {Promise.<{metadata: DataExtensionItem, type: string}>} Promise of items
      */
     static async retrieveAsTemplate(templateDir, name, templateVariables) {
+        /** @type {SoapRequestParams} */
         const options = {
             filter: {
                 leftOperand: 'Name',
@@ -1359,7 +1475,7 @@ class DataExtension extends MetadataType {
         if (!Object.keys(metadata).length) {
             Util.logger.error(`${this.definition.type} '${name}' not found on server.`);
             Util.logger.info('Downloaded: dataExtension (0)');
-            return { metadata: {}, type: 'dataExtension' };
+            return { metadata: null, type: 'dataExtension' };
         }
         const customerKey = Object.keys(metadata)[0];
         await this._retrieveFieldsForSingleDe(metadata, customerKey);
@@ -1370,8 +1486,8 @@ class DataExtension extends MetadataType {
                 metadata[key].Fields.sort((a, b) => a.Ordinal - b.Ordinal);
 
                 const originalKey = key;
-                const metadataCleaned = JSON.parse(
-                    JSON.stringify(await this.postRetrieveTasks(metadata[key]))
+                const metadataCleaned = structuredClone(
+                    await this.postRetrieveTasks(metadata[key])
                 );
 
                 this.keepTemplateFields(metadataCleaned);
@@ -1395,7 +1511,7 @@ class DataExtension extends MetadataType {
      * dataExtension logic that retrieves the folder path from cache and updates the given metadata with it after retrieve
      * it also sets the content type which is basically the subtype
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single script activity definition
+     * @param {MetadataTypeItem} metadata a single script activity definition
      */
     static setFolderPath(metadata) {
         let error = false;
@@ -1455,8 +1571,8 @@ class DataExtension extends MetadataType {
      *
      * @private
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {TYPE.SoapRequestParams} [options] e.g. filter
-     * @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
+     * @param {SoapRequestParams} [options] e.g. filter
+     * @returns {Promise.<DataExtensionMap>} keyField => metadata map
      */
     static async _retrieveAll(additionalFields, options) {
         const { metadata } = await super.retrieveSOAP(null, options, null, additionalFields);
@@ -1475,9 +1591,9 @@ class DataExtension extends MetadataType {
      * additionally, the documentation for dataExtension and automation should be returned
      *
      * @param {string[]} keyArr customerkey of the metadata
-     * @returns {string[]} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
+     * @returns {Promise.<string[]>} list of all files that need to be committed in a flat array ['path/file1.ext', 'path/file2.ext']
      */
-    static getFilesToCommit(keyArr) {
+    static async getFilesToCommit(keyArr) {
         if (this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)) {
             // document dataExtension is active. assume we want to commit the MD file as well
             const path = File.normalizePath([