mcdev 3.0.3 → 3.1.0

package/lib/metadataTypes/DataExtension.js

@@ -3,6 +3,7 @@
 const jsonToTable = require('json-to-table');
 const MetadataType = require('./MetadataType');
 const DataExtensionField = require('./DataExtensionField');
+const Folder = require('./Folder');
 const Util = require('../util/util');
 const File = require('../util/file');
 
@@ -43,7 +44,7 @@ class DataExtension extends MetadataType {
      */
     static async upsert(desToDeploy, _, buObject) {
         Util.logger.info('- Retrieve target metadata for comparison with deploy metadata');
-        const results = await this.retrieveForCache(buObject);
+        const results = await this.retrieveForCache(buObject, null, true);
         const targetMetadata = results.metadata;
         Util.logger.info('- Retrieved target metadata');
         /** @type {Promise[]} */
@@ -89,21 +90,7 @@ class DataExtension extends MetadataType {
                 // skip rest of handling for this DE
                 continue;
             }
-            // Error if SendableSubscriberField.Name = '_SubscriberKey' even though it is retrieved like that
-            // Therefore map it to 'Subscriber Key'. Retrieving afterward still results in '_SubscriberKey'
-            if (
-                desToDeploy[dataExtension].SendableSubscriberField &&
-                desToDeploy[dataExtension].SendableSubscriberField.Name === '_SubscriberKey'
-            ) {
-                desToDeploy[dataExtension].SendableSubscriberField.Name = 'Subscriber Key';
-            }
             if (targetMetadata[dataExtension]) {
-                // Update dataExtension + Columns if they already exist; Create them if not
-                // Modify columns for update call
-                DataExtension.prepareDeployColumnsOnUpdate(
-                    desToDeploy[dataExtension].Fields,
-                    targetMetadata[dataExtension].Fields
-                );
                 // data extension already exists in target and needs to be updated
                 deUpdatePromises.push(DataExtension.update(desToDeploy[dataExtension]));
             } else {
@@ -113,7 +100,7 @@ class DataExtension extends MetadataType {
         }
         if (deUpdatePromises.length) {
             Util.logger.info(
-                'Please note that Data Retention Policies can only be set during creation, not during update.'
+                '- Please note that Data Retention Policies can only be set during creation, not during update.'
             );
         }
 
@@ -123,6 +110,7 @@ class DataExtension extends MetadataType {
         const updateResults = (await Promise.allSettled(deUpdatePromises)).filter(
             this._filterUpsertResults
         );
+
         const successfulResults = [...createResults, ...updateResults];
 
         Util.metadataLogger(
@@ -160,6 +148,10 @@ class DataExtension extends MetadataType {
             // promise rejects, whole request failed
             Util.logger.error('- error upserting dataExtension: ' + res.reason);
             return false;
+        } else if (res.value == undefined || Object.keys(res.value).length === 0) {
+            // in case of returning empty result handle gracefully
+            // TODO: consider if SOAP handler for this should really return empty object
+            return false;
         } else if (res.value.results) {
             Util.logger.error(
                 '- error upserting dataExtension: ' +
@@ -177,39 +169,6 @@ class DataExtension extends MetadataType {
         }
     }
 
-    /**
-     * Mofifies passed deployColumns for update by mapping ObjectID to their target column's values.
-     * Removes FieldType field if its the same in deploy and target column, because it results in an error even if its of the same type
-     *
-     * @param {DataExtensionField.DataExtensionFieldItem[]} deployColumns Columns of data extension that will be deployed
-     * @param {DataExtensionField.DataExtensionFieldItem[]} targetColumns Columns of data extension that currently exists in target
-     * @returns {void}
-     */
-    static prepareDeployColumnsOnUpdate(deployColumns, targetColumns) {
-        // Map data extension column ObjectIDs to their target, because they are environment specific
-        for (const column in deployColumns) {
-            const deployColumn = deployColumns[column];
-            // Check if column exists in target
-            if (targetColumns[column]) {
-                // Map ObjectID to value of target, because DataExtensionField updates are based on ObjectID
-                deployColumn.ObjectID = targetColumns[column].ObjectID;
-                // Remove FieldType if it is the same in target, because an error occurs if a FieldTypes gets passed on a field update (even if its the same type)
-                if (targetColumns[column].FieldType === deployColumn.FieldType) {
-                    delete deployColumn.FieldType;
-                } else {
-                    // Updating to a new FieldType will result in an error
-                    Util.logger.warn(
-                        'DataExtension.prepareDeployColumnsOnUpdate:: Cannot update FieldType of field: ' +
-                            deployColumn.CustomerKey
-                    );
-                }
-            } else {
-                // Field doesn't exist in target, therefore Remove ObjectID if present
-                delete deployColumn.ObjectID;
-            }
-        }
-    }
-
     /**
      * Create a single dataExtension. Also creates their columns in 'dataExtension.columns'
      * @param {DataExtensionItem} metadata single metadata entry
@@ -256,33 +215,17 @@ class DataExtension extends MetadataType {
      * @returns {Promise} Promise
      */
     static async update(metadata) {
-        // retrieve existing fields to enable updating them
-        const fieldsObj = await this._retrieveFields(['Name', 'ObjectID'], {
-            filter: {
-                leftOperand: 'DataExtension.CustomerKey',
-                operator: 'equals',
-                rightOperand: metadata.CustomerKey,
-            },
-        });
-
-        // ensure fields can be updated properly by their adding ObjectId based on Name-matching
-        const existingFieldObj = {};
-        Object.keys(fieldsObj).forEach((key) => {
-            existingFieldObj[fieldsObj[key].Name] = fieldsObj[key].ObjectID;
-        });
-        metadata.Fields.map((item) => {
-            if (existingFieldObj[item.Name]) {
-                // field is getting updated ---
-                // remove FieldType as it cannot be updated mostly and will lead to an API error
-                delete item.FieldType;
-
-                // set the ObjectId for clear identification
-                item.ObjectID = existingFieldObj[item.Name];
-            } else {
-                // field is getting added ---
-                delete item.ObjectID; // make sure this is gone or it will cause issues on new fields!
-            }
-        });
+        // Update dataExtension + Columns if they already exist; Create them if not
+        // Modify columns for update call
+        DataExtensionField.cache = this.metadata;
+        DataExtensionField.client = this.client;
+        DataExtensionField.properties = this.properties;
+        DataExtension.oldFields = DataExtension.oldFields || {};
+        DataExtension.oldFields[metadata.CustomerKey] =
+            await DataExtensionField.prepareDeployColumnsOnUpdate(
+                metadata.Fields,
+                metadata.CustomerKey
+            );
 
         // convert simple array into object.Array.object format to cope with how the XML body in the SOAP call needs to look like:
         // <Fields>
@@ -295,25 +238,70 @@ class DataExtension extends MetadataType {
         metadata.Fields = { Field: metadata.Fields };
         return super.updateSOAP(metadata);
     }
+    /**
+     * Gets executed after deployment of metadata type
+     * @param {DataExtensionMap} upsertedMetadata metadata mapped by their keyField
+     * @returns {void}
+     */
+    static postDeployTasks(upsertedMetadata) {
+        if (!DataExtension.oldFields) {
+            // only run postDeploy if we are in update mode
+            return;
+        }
+        // somewhat of a workardoun but it ensures we get the field list from the server rather than whatever we might have in cache got returned during update/add. This ensures a complete and correctly ordered field list
+        for (const key in upsertedMetadata) {
+            const item = upsertedMetadata[key];
+            const isUpdate =
+                this.cache &&
+                this.cache.dataExtension &&
+                this.cache.dataExtension[item.CustomerKey];
+            if (isUpdate) {
+                const cachedVersion = this.cache.dataExtension[item.CustomerKey];
+                // restore retention values that are typically not returned by the update call
+                item.RowBasedRetention = cachedVersion.RowBasedRetention;
+                item.ResetRetentionPeriodOnImport = cachedVersion.ResetRetentionPeriodOnImport;
+                item.DeleteAtEndOfRetentionPeriod = cachedVersion.DeleteAtEndOfRetentionPeriod;
+                item.RetainUntil = cachedVersion.RetainUntil;
+
+                // ensure we have th
+                const existingFields = DataExtension.oldFields[item[this.definition.nameField]];
+                if (item.Fields !== '' && existingFields) {
+                    // TODO should be replaced by a manual sort using existingFields
+                    // ! this is inefficient because it triggers a new download of the fields during the saveResults() step
+                    item.Fields.length = 0;
+                }
+                // sort Fields entry to the end of the object for saving in .json
+                const fieldsBackup = item.Fields;
+                delete item.Fields;
+                item.Fields = fieldsBackup;
+            }
+        }
+    }
 
     /**
      * Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionColumn metadata retrieval
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {Util.BuObject} buObject properties for auth
+     * @param {void} [_] -
+     * @param {boolean} [isDeploy] used to signal that fields shall be retrieve in caching mode
      * @returns {Promise<{metadata:DataExtensionMap,type:string}>} Promise of item map
      */
-    static async retrieve(retrieveDir, additionalFields, buObject) {
+    static async retrieve(retrieveDir, additionalFields, buObject, _, isDeploy) {
         let metadata = await this._retrieveAll(additionalFields);
         // in case of cache dont get fields
-        if (metadata && retrieveDir) {
+        if (isDeploy || (metadata && retrieveDir)) {
             // get fields from API
-            const fieldsObj = await this._retrieveFields(additionalFields);
+            const fieldsObj = await this._retrieveFields(null, additionalFields);
             const fieldKeys = Object.keys(fieldsObj);
             // add fields to corresponding DE
             fieldKeys.forEach((key) => {
                 const field = fieldsObj[key];
-                metadata[field.DataExtension.CustomerKey].Fields.push(field);
+                if (metadata[field?.DataExtension?.CustomerKey]) {
+                    metadata[field.DataExtension.CustomerKey].Fields.push(field);
+                } else {
+                    Util.logger.warn(`Issue retrieving data extension fields. key='${key}'`);
+                }
             });
 
             // sort fields by Ordinal value (API returns field unsorted)
@@ -336,9 +324,7 @@ class DataExtension extends MetadataType {
                 clientSecret: this.properties.credentials[buObject.credential].clientSecret,
                 tenant: this.properties.credentials[buObject.credential].tenant,
                 eid: this.properties.credentials[buObject.credential].eid,
-                mid: this.properties.credentials[buObject.credential].businessUnits[
-                    Util.parentBuName
-                ],
+                mid: this.properties.credentials[buObject.credential].eid,
                 businessUnit: Util.parentBuName,
                 credential: buObject.credential,
             };
@@ -351,6 +337,16 @@ class DataExtension extends MetadataType {
             }
             const metadataParentBu = await this._retrieveAll(additionalFields);
 
+            // get shared folders to match our shared / synched Data Extensions
+            Util.logger.info('- Caching dependent Metadata: folder (shared via _ParentBU_)');
+            Folder.cache = {};
+            Folder.client = this.client;
+            Folder.properties = this.properties;
+            const result = await Folder.retrieveForCache(buObjectParentBu);
+            const parentCache = {
+                folder: result.metadata,
+            };
+
             // get the types and clean out non-shared ones
             const folderTypesFromParent = require('../MetadataTypeDefinitions').folder
                 .folderTypesFromParent;
@@ -358,22 +354,30 @@ class DataExtension extends MetadataType {
                 try {
                     // get the data extension type from the folder
                     const folderContentType = Util.getFromCache(
-                        this.cache,
+                        parentCache,
                         'folder',
                         metadataParentBu[metadataEntry].CategoryID,
                         'ID',
                         'ContentType'
                     );
                     if (!folderTypesFromParent.includes(folderContentType)) {
+                        Util.logger.verbose(
+                            `removing ${metadataEntry} because r__folder_ContentType '${folderContentType}' identifies this DE as not being shared`
+                        );
                         delete metadataParentBu[metadataEntry];
                     }
                 } catch (ex) {
+                    Util.logger.debug(
+                        `removing ${metadataEntry} because of error while retrieving r__folder_ContentType: ${ex.message}`
+                    );
                     delete metadataParentBu[metadataEntry];
                 }
             }
 
             // revert client to current default
             this.client = clientBackup;
+            Folder.client = clientBackup;
+            Folder.cache = this.cache;
 
             // make sure to overwrite parent bu DEs with local ones
             metadata = { ...metadataParentBu, ...metadata };
@@ -390,6 +394,15 @@ class DataExtension extends MetadataType {
         }
         return { metadata: metadata, type: 'dataExtension' };
     }
+    /**
+     * Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionColumn metadata retrieval
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @returns {Promise<{metadata:DataExtensionMap,type:string}>} Promise of item map
+     */
+    static async retrieveChangelog(additionalFields) {
+        const metadata = await this._retrieveAll(additionalFields);
+        return { metadata: metadata, type: 'dataExtension' };
+    }
     /**
      * manages post retrieve steps
      * @param {DataExtensionItem} metadata a single dataExtension
@@ -410,17 +423,25 @@ class DataExtension extends MetadataType {
                 'Ensure that Queries that write into this DE are updated with the new key before deployment.';
             this.overrideKeyWithName(metadata, warningMsg);
         }
+        // Error during deploy if SendableSubscriberField.Name = '_SubscriberKey' even though it is retrieved like that
+        // Therefore map it to 'Subscriber Key'. Retrieving afterward still results in '_SubscriberKey'
+        if (
+            metadata.SendableSubscriberField &&
+            metadata.SendableSubscriberField.Name === '_SubscriberKey'
+        ) {
+            metadata.SendableSubscriberField.Name = 'Subscriber Key';
+        }
         return this._parseMetadata(JSON.parse(JSON.stringify(metadata)));
     }
 
     /**
      * Helper to retrieve Data Extension Fields
      * @private
-     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @param {Object} [options] options (e.g. continueRequest)
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
      * @returns {Promise<DataExtensionField.DataExtensionFieldMap>} Promise of items
      */
-    static async _retrieveFields(additionalFields, options) {
+    static async _retrieveFields(options, additionalFields) {
         if (!options) {
             // dont print this during updates or templating which retrieves fields DE-by-DE
             Util.logger.info('- Caching dependent Metadata: dataExtensionField');
@@ -447,7 +468,7 @@ class DataExtension extends MetadataType {
                 rightOperand: customerKey,
             },
         };
-        const fieldsObj = await this._retrieveFields(null, fieldOptions);
+        const fieldsObj = await this._retrieveFields(fieldOptions);
 
         DataExtensionField.cache = this.metadata;
         DataExtensionField.client = this.client;
@@ -524,6 +545,16 @@ class DataExtension extends MetadataType {
         // contenttype
         delete metadata.r__folder_ContentType;
 
+        // Error if SendableSubscriberField.Name = '_SubscriberKey' even though it is retrieved like that
+        // Therefore map it to 'Subscriber Key'. Retrieving afterward still results in '_SubscriberKey'
+        // TODO remove from preDeploy with release of version 4, keep until then to help with migration of old metadata
+        if (
+            metadata.SendableSubscriberField &&
+            metadata.SendableSubscriberField.Name === '_SubscriberKey'
+        ) {
+            metadata.SendableSubscriberField.Name = 'Subscriber Key';
+        }
+
         return metadata;
     }
 
@@ -826,10 +857,12 @@ class DataExtension extends MetadataType {
     /**
      * Retrieves folder metadata into local filesystem. Also creates a uniquePath attribute for each folder.
      * @param {Object} buObject properties for auth
+     * @param {void} [_] -
+     * @param {boolean} [isDeploy] used to signal that fields shall be retrieve in caching mode
      * @returns {Promise} Promise
      */
-    static async retrieveForCache(buObject) {
-        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], buObject);
+    static async retrieveForCache(buObject, _, isDeploy) {
+        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], buObject, null, isDeploy);
     }
     /**
      * Retrieves dataExtension metadata in template format.
@@ -993,6 +1026,9 @@ class DataExtension extends MetadataType {
 
 // Assign definition to static attributes
 DataExtension.definition = require('../MetadataTypeDefinitions').dataExtension;
+/**
+ * @type {Util.ET_Client}
+ */
 DataExtension.client = undefined;
 DataExtension.cache = {};
 

package/lib/metadataTypes/DataExtensionField.js

@@ -1,17 +1,21 @@
 'use strict';
 
+const Util = require('../util/util');
+
 /**
  * @typedef {Object} DataExtensionFieldItem
  * @property {string} [ObjectID] id
- * @property {string} [CustomerKey] key
+ * @property {string} [CustomerKey] key in format [DEkey].[FieldName]
  * @property {Object} [DataExtension] -
  * @property {string} DataExtension.CustomerKey key of DE
- * @property {string} Name name
- * @property {string} DefaultValue -
+ * @property {string} Name name of field
+ * @property {string} [Name_new] custom attribute that is only used when trying to rename a field from Name to Name_new
+ * @property {string} DefaultValue empty string for not set
  * @property {'true'|'false'} IsRequired -
  * @property {'true'|'false'} IsPrimaryKey -
  * @property {string} Ordinal 1, 2, 3, ...
- * @property {'Text'|'Date'|'Number'|'Decimal'|'Email'} FieldType -
+ * @property {'Text'|'Number'|'Date'|'Boolean'|'Decimal'|'EmailAddress'|'Phone'|'Locale'} FieldType can only be set on create
+ * @property {string} Scale the number of places after the decimal that the field can hold; example: "0","1", ...
  *
  * @typedef {Object.<string, DataExtensionFieldItem>} DataExtensionFieldMap
  */
@@ -91,9 +95,135 @@ class DataExtensionField extends MetadataType {
             delete metadata.CustomerKey;
             delete metadata.DataExtension;
             delete metadata.Ordinal;
+            if (metadata.FieldType !== 'Decimal') {
+                // remove scale - it's only used for "Decimal" to define the digits behind the decimal
+                delete metadata.Scale;
+            }
         }
         return metadata;
     }
+    /**
+     * Mofifies passed deployColumns for update by mapping ObjectID to their target column's values.
+     * Removes FieldType field if its the same in deploy and target column, because it results in an error even if its of the same type
+     *
+     * @param {DataExtensionFieldItem[]} deployColumns Columns of data extension that will be deployed
+     * @param {string} deKey external/customer key of Data Extension
+     * @returns {Object<string,DataExtensionFieldItem>} existing fields by their original name to allow re-adding FieldType after update
+     */
+    static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
+        // retrieve existing fields to enable updating them
+        const response = await this.retrieveForCache(
+            {
+                filter: {
+                    leftOperand: 'DataExtension.CustomerKey',
+                    operator: 'equals',
+                    rightOperand: deKey,
+                },
+            },
+            ['Name', 'ObjectID']
+        );
+
+        const fieldsObj = response.metadata;
+
+        // ensure fields can be updated properly by their adding ObjectId based on Name-matching
+        /** @type {Object<string,DataExtensionFieldItem>} */
+        const existingFieldByName = {};
+
+        Object.keys(fieldsObj).forEach((key) => {
+            existingFieldByName[fieldsObj[key].Name] = fieldsObj[key];
+        });
+        for (let i = deployColumns.length - 1; i >= 0; i--) {
+            const item = deployColumns[i];
+            const itemOld = existingFieldByName[item.Name];
+            if (itemOld) {
+                // field is getting updated ---
+
+                // Updating to a new FieldType will result in an error; warn & afterwards remove it
+                if (itemOld.FieldType !== item.FieldType) {
+                    Util.logger.warn(
+                        `- The Field Type of an existing field cannot be changed. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.FieldType}'`
+                    );
+                    item.FieldType = itemOld.FieldType;
+                }
+                if (item.FieldType !== 'Decimal') {
+                    // remove scale - it's only used for "Decimal" to define the digits behind the decimal
+                    delete item.Scale;
+                }
+                delete item.FieldType;
+
+                if (itemOld.MaxLength > item.MaxLength) {
+                    Util.logger.warn(
+                        `- The length of an existing field cannot be decreased. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.MaxLength}'`
+                    );
+                    item.MaxLength = itemOld.MaxLength;
+                }
+                if (Util.isFalse(itemOld.IsRequired) && Util.isTrue(item.IsRequired)) {
+                    Util.logger.warn(
+                        `- A field cannot be changed to be required on update after it was created to allow nulls: [${deKey}].[${item.Name}]`
+                    );
+                    item.IsRequired = itemOld.IsRequired;
+                }
+
+                // enable renaming
+                if (item.Name_new) {
+                    item.Name = item.Name_new;
+                    delete item.Name_new;
+                    Util.logger.warn(
+                        `Found 'Name_new' value '${item.Name_new}' for ${deKey}.${item.Name} - trying to rename.`
+                    );
+                }
+
+                // check if any changes were found
+                let changeFound = false;
+                Object.keys(item).forEach((key) => {
+                    if (item[key] !== itemOld[key]) {
+                        changeFound = true;
+                    }
+                });
+                if (!changeFound) {
+                    deployColumns.splice(i, 1);
+                    Util.logger.verbose(`no change - removed field [${deKey}].[${item.Name}]`);
+                    continue;
+                }
+
+                // set the ObjectId for clear identification during update
+                item.ObjectID = itemOld.ObjectID;
+            } else {
+                // field is getting added ---
+                if (Util.isTrue(item.IsRequired) && item.DefaultValue === '') {
+                    Util.logger.warn(
+                        `- Adding new fields to an existing table requires that these fields are either not-required (nullable) or have a default value set. Changing [${deKey}].[${item.Name}] to be not-required`
+                    );
+                    item.IsRequired = 'false';
+                }
+                if (item.Name_new) {
+                    Util.logger.warn(
+                        `Found 'Name_new' value '${item.Name_new}' for ${deKey}.${item.Name} but could not find a corresponding DE field on the server - adding new field instead of updating.`
+                    );
+                    delete item.Name_new;
+                }
+                // Field doesn't exist in target, therefore Remove ObjectID if present
+                delete item.ObjectID;
+            }
+            if (!Util.isTrue(item.IsRequired) && !Util.isFalse(item.IsRequired)) {
+                Util.logger.error(
+                    `- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'. Removing field from deploy!`
+                );
+                deployColumns.splice(i, 1);
+            }
+            if (!Util.isTrue(item.IsPrimaryKey) && !Util.isFalse(item.IsPrimaryKey)) {
+                Util.logger.error(
+                    `- Invalid value for 'IsPrimaryKey' of [${deKey}].[${item.Name}]. Found '${item.IsPrimaryKey}' instead of 'true'/'false'. Removing field from deploy!`
+                );
+                deployColumns.splice(i, 1);
+            }
+        }
+        Util.logger.debug(
+            `${deployColumns.length} Fields added/updated for [${deKey}]: ` +
+                deployColumns.map((item) => item.Name).join(', ')
+        );
+        return deployColumns.length ? existingFieldByName : null;
+    }
 }
 
 // Assign definition to static attributes

package/lib/metadataTypes/ImportFile.js

@@ -147,12 +147,10 @@ class ImportFile extends MetadataType {
             }
         }
         // When the destinationObjectTypeId is 584 is refers to Mobile Connect which is not supported as an Import Type
-        metadata.destinationObjectTypeId = this.definition.destinationObjectTypeMapping[
-            metadata.c__destinationType
-        ];
-        metadata.subscriberImportTypeId = this.definition.subscriberImportTypeMapping[
-            metadata.c__subscriberImportType
-        ];
+        metadata.destinationObjectTypeId =
+            this.definition.destinationObjectTypeMapping[metadata.c__destinationType];
+        metadata.subscriberImportTypeId =
+            this.definition.subscriberImportTypeMapping[metadata.c__subscriberImportType];
         metadata.updateTypeId = this.definition.updateTypeMapping[metadata.c__dataAction];
         return metadata;
     }

package/lib/metadataTypes/MetadataType.js

@@ -204,6 +204,16 @@ class MetadataType {
         const metadata = {};
         return { metadata: null, type: this.definition.type };
     }
+    /**
+     * Gets metadata from Marketing Cloud
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @param {Util.BuObject} buObject properties for auth
+     * @param {string} [subType] optionally limit to a single subtype
+     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     */
+    static retrieveChangelog(additionalFields, buObject, subType) {
+        return this.retrieve(null, additionalFields, buObject, subType);
+    }
 
     /**
      * Gets metadata cache with limited fields and does not store value to disk
@@ -304,9 +314,10 @@ class MetadataType {
                         });
                     } else if (this.cache[this.definition.type][normalizedKey]) {
                         // normal way of processing update files
-                        metadata[metadataKey][this.definition.idField] = this.cache[
-                            this.definition.type
-                        ][normalizedKey][this.definition.idField];
+                        metadata[metadataKey][this.definition.idField] =
+                            this.cache[this.definition.type][normalizedKey][
+                                this.definition.idField
+                            ];
                         metadataToUpdate.push({
                             before: this.cache[this.definition.type][normalizedKey],
                             after: metadata[metadataKey],
@@ -474,8 +485,12 @@ class MetadataType {
                 async () => (response = await this.client.RestClient.patch(options))
             );
             this.checkForErrors(response);
+            // some times, e.g. automation dont return a key in their update response and hence we need to fall back to name
             Util.logger.info(
-                `- updated ${this.definition.type}: ${metadataEntry[this.definition.keyField]}`
+                `- updated ${this.definition.type}: ${
+                    metadataEntry[this.definition.keyField] ||
+                    metadataEntry[this.definition.nameField]
+                }`
             );
             return response;
         } catch (ex) {
@@ -565,7 +580,6 @@ class MetadataType {
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
             );
-
             if (
                 buObject &&
                 this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)

package/lib/metadataTypes/Query.js

@@ -144,9 +144,8 @@ class Query extends MetadataType {
         } catch (ex) {
             throw new Error(`Query '${metadata.key}': ${ex.message}`);
         }
-        metadata.targetUpdateTypeId = this.definition.targetUpdateTypeMapping[
-            metadata.targetUpdateTypeName
-        ];
+        metadata.targetUpdateTypeId =
+            this.definition.targetUpdateTypeMapping[metadata.targetUpdateTypeName];
         return metadata;
     }
 

package/lib/metadataTypes/Role.js

@@ -107,14 +107,19 @@ class Role extends MetadataType {
             return;
         }
         if (!metadata) {
-            metadata = this.readBUMetadataForType(
-                File.normalizePath([
-                    this.properties.directories.retrieve,
-                    buObject.credential,
-                    Util.parentBuName,
-                ]),
-                true
-            ).role;
+            try {
+                metadata = this.readBUMetadataForType(
+                    File.normalizePath([
+                        this.properties.directories.retrieve,
+                        buObject.credential,
+                        Util.parentBuName,
+                    ]),
+                    true
+                ).role;
+            } catch (ex) {
+                Util.logger.error(ex.message);
+                return;
+            }
         }
         const directory = this.properties.directories.roles;