mcdev 4.3.4 → 5.0.1

package/lib/metadataTypes/DataExtension.js

@@ -9,6 +9,7 @@ const Util = require('../util/util');
 const File = require('../util/file');
 const auth = require('../util/auth');
 const cache = require('../util/cache');
+const pLimit = require('p-limit');
 
 /**
  * DataExtension MetadataType
@@ -20,16 +21,13 @@ class DataExtension extends MetadataType {
      * Upserts dataExtensions after retrieving them from source and target to compare
      * if create or update operation is needed.
      *
-     * @param {TYPE.DataExtensionMap} desToDeploy dataExtensions mapped by their customerKey
+     * @param {TYPE.DataExtensionMap} metadataMap dataExtensions mapped by their customerKey
      * @returns {Promise} Promise
      */
-    static async upsert(desToDeploy) {
-        // get dataExtensions from target BU for add/update decision
-        /** @type {TYPE.DataExtensionMap} */
-        const targetMetadata = cache.getCache().dataExtension || {};
+    static async upsert(metadataMap) {
         // get existing DE-fields for DE-keys in deployment package to properly handle add/update/delete of fields
         const fieldOptions = {};
-        for (const key of Object.keys(desToDeploy)) {
+        for (const key of Object.keys(metadataMap)) {
             fieldOptions.filter = fieldOptions.filter
                 ? {
                       leftOperand: {
@@ -47,80 +45,71 @@ class DataExtension extends MetadataType {
                   };
         }
         Util.logger.info(` - Caching dependent Metadata: dataExtensionField`);
-        await this._attachFields(desToDeploy, fieldOptions);
+        await this._attachFields(metadataMap, fieldOptions);
 
-        /** @type {Promise[]} */
-        const deCreatePromises = [];
-        /** @type {Promise[]} */
-        const deUpdatePromises = [];
+        /** @type {object[]} */
+        const metadataToCreate = [];
+        /** @type {object[]} */
+        const metadataToUpdate = [];
         let filteredByPreDeploy = 0;
-        for (const dataExtension in desToDeploy) {
-            if (desToDeploy[dataExtension].Name.startsWith('_')) {
-                Util.logger.warn(
-                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}: Cannot Upsert Strongly Typed Data Extensions`
-                );
-                filteredByPreDeploy++;
-                continue;
-            }
-            if (
-                this.buObject.eid !== this.buObject.mid &&
-                desToDeploy[dataExtension].r__folder_Path.startsWith('Shared Items')
-            ) {
-                // this needs to be run before executing preDeployTasks()
-                Util.logger.warn(
-                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}: Cannot Create/Update a Shared Data Extension from the Child BU`
-                );
-                filteredByPreDeploy++;
-                continue;
-            }
-            if (
-                desToDeploy[dataExtension].r__folder_Path.startsWith('Synchronized Data Extensions')
-            ) {
-                // this needs to be run before executing preDeployTasks()
-                Util.logger.warn(
-                    ` ☇ skipping dataExtension ${desToDeploy[dataExtension].Name}:Cannot Create/Update a Synchronized Data Extension. Please use Contact Builder to maintain these`
-                );
-                filteredByPreDeploy++;
-                continue;
-            }
+        for (const metadataKey in metadataMap) {
             try {
-                desToDeploy[dataExtension] = await this.preDeployTasks(desToDeploy[dataExtension]);
+                metadataMap[metadataKey] = await this.preDeployTasks(metadataMap[metadataKey]);
             } catch (ex) {
-                // problem with retrieving folder for this DE found
                 // output error & remove from deploy list
                 Util.logger.error(
-                    `- dataExtension ${desToDeploy[dataExtension].Name}: ${ex.message}`
+                    ` ☇ skipping ${this.definition.type} ${
+                        metadataMap[this.definition.keyField]
+                    } / ${metadataMap[this.definition.nameField]}: ${ex.message}`
                 );
-                delete desToDeploy[dataExtension];
+                delete metadataMap[metadataKey];
                 // skip rest of handling for this DE
+                filteredByPreDeploy++;
                 continue;
             }
-            if (targetMetadata[dataExtension]) {
-                // data extension already exists in target and needs to be updated
-                deUpdatePromises.push(DataExtension.update(desToDeploy[dataExtension]));
-            } else {
-                // data extension does not exist in target and has to be created
-                deCreatePromises.push(DataExtension.create(desToDeploy[dataExtension]));
-            }
+            await this.createOrUpdate(
+                metadataMap,
+                metadataKey,
+                false,
+                metadataToUpdate,
+                metadataToCreate
+            );
         }
-        if (deUpdatePromises.length) {
+        if (metadataToUpdate.length) {
             Util.logger.info(
                 ' - Please note that Data Retention Policies can only be set during creation, not during update.'
             );
         }
+        const createLimit = pLimit(10);
+        const createResults = (
+            await Promise.allSettled(
+                metadataToCreate
+                    .filter((r) => r !== undefined && r !== null)
+                    .map((metadataEntry) => createLimit(() => this.create(metadataEntry)))
+            )
+        )
+            .filter((r) => r !== undefined && r !== null)
+            .filter(this.#filterUpsertResults);
 
-        const createResults = (await Promise.allSettled(deCreatePromises)).filter(
-            this._filterUpsertResults
-        );
-        const updateResults = (await Promise.allSettled(deUpdatePromises)).filter(
-            this._filterUpsertResults
-        );
+        const updateLimit = pLimit(10);
+        const updateResults = (
+            await Promise.allSettled(
+                metadataToUpdate
+                    .filter((r) => r !== undefined && r !== null)
+                    .map((metadataEntry) =>
+                        updateLimit(() => this.update(metadataEntry.after, metadataEntry.before))
+                    )
+            )
+        )
+            .filter((r) => r !== undefined && r !== null)
+            .filter(this.#filterUpsertResults);
 
         const successfulResults = [...createResults, ...updateResults];
         Util.logger.info(
-            `${this.definition.type} upsert: ${createResults.length} of ${deCreatePromises.length} created / ${updateResults.length} of ${deUpdatePromises.length} updated` +
+            `${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
                 (filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
         );
+        let upsertResults;
         if (successfulResults.length > 0) {
             const metadataResults = successfulResults
                 .map((r) => r.value.Results[0].Object)
@@ -133,19 +122,94 @@ class DataExtension extends MetadataType {
                     }
                     return r;
                 });
-            return super.parseResponseBody({ Results: metadataResults });
+            upsertResults = super.parseResponseBody({ Results: metadataResults });
         } else {
-            return {};
+            upsertResults = {};
         }
+        await this.postDeployTasks(upsertResults, metadataMap, {
+            created: createResults.length,
+            updated: updateResults.length,
+        });
+        return upsertResults;
     }
 
+    /**
+     * helper for {@link MetadataType.upsert}
+     *
+     * @param {TYPE.MetadataTypeMap} metadataMap list of metadata
+     * @param {string} metadataKey key of item we are looking at
+     * @param {boolean} hasError error flag from previous code
+     * @param {TYPE.MetadataTypeItemDiff[]} metadataToUpdate list of items to update
+     * @param {TYPE.MetadataTypeItem[]} metadataToCreate list of items to create
+     * @returns {'create' | 'update' | 'skip'} action to take
+     */
+    static async createOrUpdate(
+        metadataMap,
+        metadataKey,
+        hasError,
+        metadataToUpdate,
+        metadataToCreate
+    ) {
+        const action = await super.createOrUpdate(
+            metadataMap,
+            metadataKey,
+            hasError,
+            metadataToUpdate,
+            metadataToCreate
+        );
+
+        if (action === 'update') {
+            // Update dataExtension + Columns if they already exist; Create them if not
+            // Modify columns for update call
+            DataExtensionField.client = this.client;
+            DataExtensionField.properties = this.properties;
+            DataExtension.oldFields ||= {};
+            DataExtension.oldFields[metadataMap[metadataKey][this.definition.keyField]] =
+                await DataExtensionField.prepareDeployColumnsOnUpdate(
+                    metadataMap[metadataKey].Fields,
+                    metadataKey
+                );
+
+            if (
+                metadataMap[metadataKey][this.definition.keyField] !== metadataKey &&
+                metadataMap[metadataKey].Fields.length
+            ) {
+                // changeKeyValue / changeKeyField used
+                Util.logger.warn(
+                    ` - ${this.definition.type} ${metadataKey}: Cannot change fields while updating the key. Skipping field update in favor of key update.`
+                );
+                metadataMap[metadataKey].Fields.length = 0;
+            }
+
+            // convert simple array into object.Array.object format to cope with how the XML body in the SOAP call needs to look like:
+            // <Fields>
+            //   <Field>
+            //     <CustomerKey>SubscriberKey</CustomerKey>
+            //      ..
+            //   </Field>
+            // </Fields>
+            metadataMap[metadataKey].Fields = { Field: metadataMap[metadataKey].Fields };
+        } else if (action === 'create') {
+            this.#cleanupRetentionPolicyFields(metadataMap[metadataKey]);
+
+            // convert simple array into object.Array.object format to cope with how the XML body in the SOAP call needs to look like:
+            // <Fields>
+            //   <Field>
+            //     <CustomerKey>SubscriberKey</CustomerKey>
+            //      ..
+            //   </Field>
+            // </Fields>
+            metadataMap[metadataKey].Fields = { Field: metadataMap[metadataKey].Fields };
+        }
+    }
     /**
      * helper for {@link upsert}
      *
+     * @private
      * @param {object} res -
      * @returns {boolean} true: keep, false: discard
      */
-    static _filterUpsertResults(res) {
+    static #filterUpsertResults(res) {
         if (res.status === 'rejected') {
             // promise rejects, whole request failed
             Util.logger.error('- error upserting dataExtension: ' + res.reason);
@@ -178,17 +242,6 @@ class DataExtension extends MetadataType {
      * @returns {Promise} Promise
      */
     static async create(metadata) {
-        this._cleanupRetentionPolicyFields(metadata);
-
-        // convert simple array into object.Array.object format to cope with how the XML body in the SOAP call needs to look like:
-        // <Fields>
-        //   <Field>
-        //     <CustomerKey>SubscriberKey</CustomerKey>
-        //      ..
-        //   </Field>
-        // </Fields>
-        metadata.Fields = { Field: metadata.Fields };
-
         return super.createSOAP(metadata);
     }
 
@@ -201,7 +254,7 @@ class DataExtension extends MetadataType {
      * @param {TYPE.DataExtensionItem} metadata single metadata entry
      * @returns {void}
      */
-    static _cleanupRetentionPolicyFields(metadata) {
+    static #cleanupRetentionPolicyFields(metadata) {
         if (
             metadata.DataRetentionPeriodLength &&
             metadata.DataRetentionPeriodUnitOfMeasure &&
@@ -220,26 +273,6 @@ class DataExtension extends MetadataType {
      * @returns {Promise} Promise
      */
     static async update(metadata) {
-        // Update dataExtension + Columns if they already exist; Create them if not
-        // Modify columns for update call
-        DataExtensionField.client = this.client;
-        DataExtensionField.properties = this.properties;
-        DataExtension.oldFields = DataExtension.oldFields || {};
-        DataExtension.oldFields[metadata[this.definition.keyField]] =
-            await DataExtensionField.prepareDeployColumnsOnUpdate(
-                metadata.Fields,
-                metadata.CustomerKey
-            );
-
-        // convert simple array into object.Array.object format to cope with how the XML body in the SOAP call needs to look like:
-        // <Fields>
-        //   <Field>
-        //     <CustomerKey>SubscriberKey</CustomerKey>
-        //      ..
-        //   </Field>
-        // </Fields>
-
-        metadata.Fields = { Field: metadata.Fields };
         return super.updateSOAP(metadata);
     }
     /**
@@ -247,12 +280,19 @@ class DataExtension extends MetadataType {
      *
      * @param {TYPE.DataExtensionMap} upsertedMetadata metadata mapped by their keyField
      * @param {TYPE.DataExtensionMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
      * @returns {void}
      */
-    static postDeployTasks(upsertedMetadata, originalMetadata) {
+    static postDeployTasks(upsertedMetadata, originalMetadata, createdUpdated) {
         for (const key in upsertedMetadata) {
             const item = upsertedMetadata[key];
-            const cachedVersion = cache.getByKey('dataExtension', item.CustomerKey);
+
+            const oldKey = Util.changedKeysMap?.[this.definition.type]?.[key] || key;
+            delete Util.changedKeysMap?.[this.definition.type]?.[key];
+
+            const cachedVersion = createdUpdated.updated
+                ? cache.getByKey(this.definition.type, oldKey)
+                : null;
             if (cachedVersion) {
                 // UPDATE
                 // restore retention values that are typically not returned by the update call
@@ -262,25 +302,25 @@ class DataExtension extends MetadataType {
                 item.RetainUntil = cachedVersion.RetainUntil;
 
                 const existingFields = DataExtension.oldFields[item[this.definition.keyField]];
+
                 if (item.Fields === '') {
                     // if no fields were updated, we need to set Fields to "empty string" for the API to work
                     // reset here to get the correct field list
                     item.Fields = Object.keys(existingFields)
-                        .map((key) => existingFields[key])
+                        .map((el) => existingFields[el])
                         .sort((a, b) => a.Ordinal - b.Ordinal);
-                }
-                if (existingFields) {
+                } else if (existingFields) {
                     // get list of updated fields
                     /** @type {TYPE.DataExtensionFieldItem[]} */
-                    const updatedFieldsArr = originalMetadata[key].Fields.Field.filter(
+                    const updatedFieldsArr = originalMetadata[oldKey].Fields.Field.filter(
                         (field) => field.ObjectID && field.ObjectID !== ''
                     );
                     // convert existing fields obj into array and sort
                     /** @type {TYPE.DataExtensionFieldItem[]} */
                     const finalFieldsArr = Object.keys(existingFields)
-                        .map((key) => {
+                        .map((el) => {
                             /** @type {TYPE.DataExtensionFieldItem} */
-                            const existingField = existingFields[key];
+                            const existingField = existingFields[el];
                             // check if the current field was updated and then override with it. otherwise use existing value
                             const field =
                                 updatedFieldsArr.find(
@@ -296,7 +336,7 @@ class DataExtension extends MetadataType {
 
                     // get list of new fields
                     /** @type {TYPE.DataExtensionFieldItem[]} */
-                    const newFieldsArr = originalMetadata[key].Fields.Field.filter(
+                    const newFieldsArr = originalMetadata[oldKey].Fields.Field.filter(
                         (field) => !field.ObjectID
                     );
                     // push new fields to end of list
@@ -487,7 +527,36 @@ class DataExtension extends MetadataType {
         if (metadata.SendableSubscriberField?.Name === '_SubscriberKey') {
             metadata.SendableSubscriberField.Name = 'Subscriber Key';
         }
-        return this._parseMetadata(JSON.parse(JSON.stringify(metadata)));
+        this.setFolderPath(metadata);
+
+        // DataExtensionTemplate
+        if (metadata.Template?.CustomerKey) {
+            try {
+                metadata.r__dataExtensionTemplate_Name = cache.searchForField(
+                    'dataExtensionTemplate',
+                    metadata.Template.CustomerKey,
+                    'CustomerKey',
+                    'Name'
+                );
+                delete metadata.Template;
+            } catch (ex) {
+                Util.logger.debug(ex.message);
+                // Let's allow retrieving such DEs but warn that they cannot be deployed to another BU.
+                // Deploying to same BU  still works!
+                // A workaround exists but it's likely not beneficial to explain to users:
+                // Create a DE based on the not-supported template on the target BU, retrieve it, copy the Template.CustomerKey into the to-be-deployed DE (or use mcdev-templating), done
+                Util.logger.warn(
+                    ` - Issue with dataExtension '${
+                        metadata[this.definition.nameField]
+                    }': Could not find specified DataExtension Template. Please note that DataExtensions based on SMSMessageTracking and SMSSubscriptionLog cannot be deployed automatically across BUs at this point.`
+                );
+            }
+        }
+        // remove the date fields manually here because we need them in the changelog but not in the saved json
+        delete metadata.CreatedDate;
+        delete metadata.ModifiedDate;
+
+        return metadata;
     }
 
     /**
@@ -539,6 +608,16 @@ class DataExtension extends MetadataType {
 
         metadata[customerKey].Fields = fieldArr;
     }
+    /**
+     * helper for {@link super.updateREST} and {@link super.updateSOAP} that removes old files after the key was changed
+     *
+     * @private
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @returns {void}
+     */
+    static async _postChangeKeyTasks(metadataEntry) {
+        super._postChangeKeyTasks(metadataEntry, true);
+    }
 
     /**
      * prepares a DataExtension for deployment
@@ -547,6 +626,21 @@ class DataExtension extends MetadataType {
      * @returns {Promise.<TYPE.DataExtensionItem>} Promise of updated single DE
      */
     static async preDeployTasks(metadata) {
+        if (metadata.Name.startsWith('_')) {
+            throw new Error(`Cannot Upsert Strongly Typed Data Extensions`);
+        }
+        if (
+            this.buObject.eid !== this.buObject.mid &&
+            metadata.r__folder_Path.startsWith('Shared Items')
+        ) {
+            throw new Error(`Cannot Create/Update a Shared Data Extension from the Child BU`);
+        }
+        if (metadata.r__folder_Path.startsWith('Synchronized Data Extensions')) {
+            throw new Error(
+                `Cannot Create/Update a Synchronized Data Extension. Please use Contact Builder to maintain these`
+            );
+        }
+
         // folder
         super.setFolderId(metadata);
 
@@ -584,9 +678,7 @@ class DataExtension extends MetadataType {
                 // A workaround for cross-BU deploy exists but it's likely not beneficial to explain to users:
                 // Create a DE based on the not-supported template on the target BU, retrieve it, copy the Template.CustomerKey into the to-be-deployed DE (or use mcdev-templating), done
                 throw new Error(
-                    `Skipping DataExtension '${
-                        metadata[this.definition.nameField]
-                    }': Could not find specified DataExtension Template. Please note that DataExtensions based on SMSMessageTracking and SMSSubscriptionLog cannot be deployed automatically across BUs at this point.`
+                    `Could not find specified DataExtension Template. Please note that DataExtensions based on SMSMessageTracking and SMSSubscriptionLog cannot be deployed automatically across BUs at this point.`
                 );
             }
         }
@@ -755,13 +847,13 @@ class DataExtension extends MetadataType {
     /**
      * Parses metadata into a readable Markdown/HTML format then saves it
      *
-     * @param {TYPE.DataExtensionMap} [metadata] a list of dataExtension definitions
+     * @param {TYPE.DataExtensionMap} [metadataMap] a list of dataExtension definitions
      * @returns {Promise.<void>} -
      */
-    static async document(metadata) {
+    static async document(metadataMap) {
         try {
-            if (!metadata) {
-                metadata = this.readBUMetadataForType(
+            if (!metadataMap) {
+                metadataMap = this.readBUMetadataForType(
                     File.normalizePath([
                         this.properties.directories.retrieve,
                         this.buObject.credential,
@@ -780,7 +872,7 @@ class DataExtension extends MetadataType {
             this.buObject.businessUnit,
             this.definition.type,
         ]);
-        if (!metadata || !Object.keys(metadata).length) {
+        if (!metadataMap || !Object.keys(metadataMap).length) {
             // as part of retrieve & manual execution we could face an empty folder
             return;
         }
@@ -794,16 +886,15 @@ class DataExtension extends MetadataType {
             'DefaultValue',
         ];
         return Promise.all(
-            Object.keys(metadata).map((customerKey) => {
-                // for (const customerKey in metadata) {
-                if (metadata[customerKey]?.Fields?.length) {
-                    for (const field of metadata[customerKey].Fields) {
+            Object.keys(metadataMap).map((key) => {
+                if (metadataMap[key]?.Fields?.length) {
+                    for (const field of metadataMap[key].Fields) {
                         field.IsNullable = !Util.isTrue(field.IsRequired);
-                        for (const key of columnsToIterateThrough) {
-                            if (Util.isTrue(field[key])) {
-                                field[key] = '+';
-                            } else if (Util.isFalse(field[key])) {
-                                field[key] = '-';
+                        for (const prop of columnsToIterateThrough) {
+                            if (Util.isTrue(field[prop])) {
+                                field[prop] = '+';
+                            } else if (Util.isFalse(field[prop])) {
+                                field[prop] = '-';
                             }
                         }
                     }
@@ -811,8 +902,8 @@ class DataExtension extends MetadataType {
                     if (['html', 'both'].includes(this.properties.options.documentType)) {
                         return this._writeDoc(
                             docPath + '/',
-                            customerKey,
-                            metadata[customerKey],
+                            key,
+                            metadataMap[key],
                             'html',
                             columnsToPrint
                         );
@@ -820,8 +911,8 @@ class DataExtension extends MetadataType {
                     if (['md', 'both'].includes(this.properties.options.documentType)) {
                         return this._writeDoc(
                             docPath + '/',
-                            customerKey,
-                            metadata[customerKey],
+                            key,
+                            metadataMap[key],
                             'md',
                             columnsToPrint
                         );
@@ -838,34 +929,19 @@ class DataExtension extends MetadataType {
      * @returns {Promise.<boolean>} deletion success status
      */
     static deleteByKey(customerKey) {
-        return super.deleteByKeySOAP(customerKey, false);
+        return super.deleteByKeySOAP(customerKey);
     }
 
     /**
      * clean up after deleting a metadata item
      *
      * @param {string} customerKey Identifier of metadata item
-     * @returns {void}
+     * @returns {Promise.<void>} - promise
      */
     static async postDeleteTasks(customerKey) {
-        // delete local copy: retrieve/cred/bu/dataExtension/...json
-        const jsonFile = File.normalizePath([
-            this.properties.directories.retrieve,
-            this.buObject.credential,
-            this.buObject.businessUnit,
-            this.definition.type,
-            `${customerKey}.${this.definition.type}-meta.json`,
-        ]);
-        await File.remove(jsonFile);
+        // delete local copy: retrieve/cred/bu/dataExtension/...-meta.json
         // delete local copy: doc/dataExtension/cred/bu/...md
-        const mdFile = File.normalizePath([
-            this.properties.directories.docs,
-            'dataExtension',
-            this.buObject.credential,
-            this.buObject.businessUnit,
-            `${customerKey}.${this.definition.type}.md`,
-        ]);
-        await File.remove(mdFile);
+        await super.postDeleteTasks(customerKey, [`${this.definition.type}-doc.md`]);
     }
 
     /**
@@ -988,45 +1064,6 @@ class DataExtension extends MetadataType {
             }
         }
     }
-    /**
-     * parses retrieved Metadata before saving
-     *
-     * @private
-     * @param {TYPE.DataExtensionItem} metadata a single dataExtension definition
-     * @returns {TYPE.DataExtensionItem} a single dataExtension definition
-     */
-    static _parseMetadata(metadata) {
-        this.setFolderPath(metadata);
-
-        // DataExtensionTemplate
-        if (metadata.Template?.CustomerKey) {
-            try {
-                metadata.r__dataExtensionTemplate_Name = cache.searchForField(
-                    'dataExtensionTemplate',
-                    metadata.Template.CustomerKey,
-                    'CustomerKey',
-                    'Name'
-                );
-                delete metadata.Template;
-            } catch (ex) {
-                Util.logger.debug(ex.message);
-                // Let's allow retrieving such DEs but warn that they cannot be deployed to another BU.
-                // Deploying to same BU  still works!
-                // A workaround exists but it's likely not beneficial to explain to users:
-                // Create a DE based on the not-supported template on the target BU, retrieve it, copy the Template.CustomerKey into the to-be-deployed DE (or use mcdev-templating), done
-                Util.logger.warn(
-                    ` - Issue with dataExtension '${
-                        metadata[this.definition.nameField]
-                    }': Could not find specified DataExtension Template. Please note that DataExtensions based on SMSMessageTracking and SMSSubscriptionLog cannot be deployed automatically across BUs at this point.`
-                );
-            }
-        }
-        // remove the date fields manually here because we need them in the changelog but not in the saved json
-        delete metadata.CreatedDate;
-        delete metadata.ModifiedDate;
-
-        return metadata;
-    }
 
     /**
      * Retrieves dataExtension metadata and cleans it
@@ -1037,7 +1074,7 @@ class DataExtension extends MetadataType {
      * @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
      */
     static async _retrieveAll(additionalFields, options) {
-        const { metadata } = await super.retrieveSOAP(null, options, additionalFields);
+        const { metadata } = await super.retrieveSOAP(null, options, null, additionalFields);
         for (const key in metadata) {
             // some system data extensions do not have CategoryID which throws errors in other places. These do not need to be parsed
             if (metadata[key].CategoryID) {