mcdev 3.0.3 → 3.1.3

package/lib/metadataTypes/DataExtensionField.js

@@ -1,17 +1,21 @@
 'use strict';
 
+const Util = require('../util/util');
+
 /**
  * @typedef {Object} DataExtensionFieldItem
  * @property {string} [ObjectID] id
- * @property {string} [CustomerKey] key
+ * @property {string} [CustomerKey] key in format [DEkey].[FieldName]
  * @property {Object} [DataExtension] -
  * @property {string} DataExtension.CustomerKey key of DE
- * @property {string} Name name
- * @property {string} DefaultValue -
+ * @property {string} Name name of field
+ * @property {string} [Name_new] custom attribute that is only used when trying to rename a field from Name to Name_new
+ * @property {string} DefaultValue empty string for not set
  * @property {'true'|'false'} IsRequired -
  * @property {'true'|'false'} IsPrimaryKey -
  * @property {string} Ordinal 1, 2, 3, ...
- * @property {'Text'|'Date'|'Number'|'Decimal'|'Email'} FieldType -
+ * @property {'Text'|'Number'|'Date'|'Boolean'|'Decimal'|'EmailAddress'|'Phone'|'Locale'} FieldType can only be set on create
+ * @property {string} Scale the number of places after the decimal that the field can hold; example: "0","1", ...
  *
  * @typedef {Object.<string, DataExtensionFieldItem>} DataExtensionFieldMap
  */
@@ -91,9 +95,135 @@ class DataExtensionField extends MetadataType {
             delete metadata.CustomerKey;
             delete metadata.DataExtension;
             delete metadata.Ordinal;
+            if (metadata.FieldType !== 'Decimal') {
+                // remove scale - it's only used for "Decimal" to define the digits behind the decimal
+                delete metadata.Scale;
+            }
         }
         return metadata;
     }
+    /**
+     * Mofifies passed deployColumns for update by mapping ObjectID to their target column's values.
+     * Removes FieldType field if its the same in deploy and target column, because it results in an error even if its of the same type
+     *
+     * @param {DataExtensionFieldItem[]} deployColumns Columns of data extension that will be deployed
+     * @param {string} deKey external/customer key of Data Extension
+     * @returns {Object<string,DataExtensionFieldItem>} existing fields by their original name to allow re-adding FieldType after update
+     */
+    static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
+        // retrieve existing fields to enable updating them
+        const response = await this.retrieveForCache(
+            {
+                filter: {
+                    leftOperand: 'DataExtension.CustomerKey',
+                    operator: 'equals',
+                    rightOperand: deKey,
+                },
+            },
+            ['Name', 'ObjectID']
+        );
+
+        const fieldsObj = response.metadata;
+
+        // ensure fields can be updated properly by their adding ObjectId based on Name-matching
+        /** @type {Object<string,DataExtensionFieldItem>} */
+        const existingFieldByName = {};
+
+        Object.keys(fieldsObj).forEach((key) => {
+            existingFieldByName[fieldsObj[key].Name] = fieldsObj[key];
+        });
+        for (let i = deployColumns.length - 1; i >= 0; i--) {
+            const item = deployColumns[i];
+            const itemOld = existingFieldByName[item.Name];
+            if (itemOld) {
+                // field is getting updated ---
+
+                // Updating to a new FieldType will result in an error; warn & afterwards remove it
+                if (itemOld.FieldType !== item.FieldType) {
+                    Util.logger.warn(
+                        `- The Field Type of an existing field cannot be changed. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.FieldType}'`
+                    );
+                    item.FieldType = itemOld.FieldType;
+                }
+                if (item.FieldType !== 'Decimal') {
+                    // remove scale - it's only used for "Decimal" to define the digits behind the decimal
+                    delete item.Scale;
+                }
+                delete item.FieldType;
+
+                if (itemOld.MaxLength > item.MaxLength) {
+                    Util.logger.warn(
+                        `- The length of an existing field cannot be decreased. Keeping the original value for [${deKey}].[${item.Name}]: '${itemOld.MaxLength}'`
+                    );
+                    item.MaxLength = itemOld.MaxLength;
+                }
+                if (Util.isFalse(itemOld.IsRequired) && Util.isTrue(item.IsRequired)) {
+                    Util.logger.warn(
+                        `- A field cannot be changed to be required on update after it was created to allow nulls: [${deKey}].[${item.Name}]`
+                    );
+                    item.IsRequired = itemOld.IsRequired;
+                }
+
+                // enable renaming
+                if (item.Name_new) {
+                    item.Name = item.Name_new;
+                    delete item.Name_new;
+                    Util.logger.warn(
+                        `Found 'Name_new' value '${item.Name_new}' for ${deKey}.${item.Name} - trying to rename.`
+                    );
+                }
+
+                // check if any changes were found
+                let changeFound = false;
+                Object.keys(item).forEach((key) => {
+                    if (item[key] !== itemOld[key]) {
+                        changeFound = true;
+                    }
+                });
+                if (!changeFound) {
+                    deployColumns.splice(i, 1);
+                    Util.logger.verbose(`no change - removed field [${deKey}].[${item.Name}]`);
+                    continue;
+                }
+
+                // set the ObjectId for clear identification during update
+                item.ObjectID = itemOld.ObjectID;
+            } else {
+                // field is getting added ---
+                if (Util.isTrue(item.IsRequired) && item.DefaultValue === '') {
+                    Util.logger.warn(
+                        `- Adding new fields to an existing table requires that these fields are either not-required (nullable) or have a default value set. Changing [${deKey}].[${item.Name}] to be not-required`
+                    );
+                    item.IsRequired = 'false';
+                }
+                if (item.Name_new) {
+                    Util.logger.warn(
+                        `Found 'Name_new' value '${item.Name_new}' for ${deKey}.${item.Name} but could not find a corresponding DE field on the server - adding new field instead of updating.`
+                    );
+                    delete item.Name_new;
+                }
+                // Field doesn't exist in target, therefore Remove ObjectID if present
+                delete item.ObjectID;
+            }
+            if (!Util.isTrue(item.IsRequired) && !Util.isFalse(item.IsRequired)) {
+                Util.logger.error(
+                    `- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'. Removing field from deploy!`
+                );
+                deployColumns.splice(i, 1);
+            }
+            if (!Util.isTrue(item.IsPrimaryKey) && !Util.isFalse(item.IsPrimaryKey)) {
+                Util.logger.error(
+                    `- Invalid value for 'IsPrimaryKey' of [${deKey}].[${item.Name}]. Found '${item.IsPrimaryKey}' instead of 'true'/'false'. Removing field from deploy!`
+                );
+                deployColumns.splice(i, 1);
+            }
+        }
+        Util.logger.debug(
+            `${deployColumns.length} Fields added/updated for [${deKey}]: ` +
+                deployColumns.map((item) => item.Name).join(', ')
+        );
+        return deployColumns.length ? existingFieldByName : null;
+    }
 }
 
 // Assign definition to static attributes

package/lib/metadataTypes/DataExtract.js

@@ -30,10 +30,10 @@ class DataExtract extends MetadataType {
      * Retrieve a specific dataExtract Definition by Name
      * @param {String} templateDir Directory where retrieved metadata directory will be saved
      * @param {String} name name of the metadata file
-     * @param {Object} variables variables to be replaced in the metadata
+     * @param {Object} templateVariables variables to be replaced in the metadata
      * @returns {Promise<Object>} Promise of metadata
      */
-    static async retrieveAsTemplate(templateDir, name, variables) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables) {
         const options = {
             uri: '/automation/v1/dataextracts/?$filter=name%20eq%20' + name.split(' ').join('%20'),
         };
@@ -51,16 +51,20 @@ class DataExtract extends MetadataType {
             const extended = await this.client.RestClient.get({
                 uri: '/automation/v1/dataextracts/' + metadata.id,
             });
+            const originalKey = extended.body[this.definition.keyField];
             const val = JSON.parse(
-                Util.replaceByObject(JSON.stringify(this.parseMetadata(extended.body)), variables)
+                Util.replaceByObject(
+                    JSON.stringify(this.parseMetadata(extended.body)),
+                    templateVariables
+                )
             );
 
             // remove all fields listed in Definition for templating
             this.keepTemplateFields(val);
             File.writeJSONToFile(
                 [templateDir, this.definition.type].join('/'),
-                val.customerKey + '.' + this.definition.type + '-meta',
-                JSON.parse(Util.replaceByObject(JSON.stringify(val), variables))
+                originalKey + '.' + this.definition.type + '-meta',
+                JSON.parse(Util.replaceByObject(JSON.stringify(val), templateVariables))
             );
             Util.logger.info(
                 `Dataextracts.retrieveAsTemplate:: Written Metadata to filesystem (${name})`

package/lib/metadataTypes/EventDefinition.js

@@ -32,10 +32,10 @@ class EventDefinition extends MetadataType {
      * Retrieve a specific Event Definition by Name
      * @param {String} templateDir Directory where retrieved metadata directory will be saved
      * @param {String} name name of the metadata file
-     * @param {Object} variables variables to be replaced in the metadata
+     * @param {Object} templateVariables variables to be replaced in the metadata
      * @returns {Promise<Object>} Promise of metadata
      */
-    static async retrieveAsTemplate(templateDir, name, variables) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables) {
         // todo template based on name
         const options = {
             uri: '/interaction/v1/EventDefinitions?name=' + encodeURIComponent(name),
@@ -51,8 +51,12 @@ class EventDefinition extends MetadataType {
                         `please rename to be unique to avoid issues`
                 );
             } else if (event && event.length === 1) {
+                const originalKey = event[0][this.definition.keyField];
                 const eventDef = JSON.parse(
-                    Util.replaceByObject(JSON.stringify(this.parseMetadata(event[0])), variables)
+                    Util.replaceByObject(
+                        JSON.stringify(this.parseMetadata(event[0])),
+                        templateVariables
+                    )
                 );
                 if (!eventDef.dataExtensionId) {
                     throw new Error(
@@ -67,8 +71,8 @@ class EventDefinition extends MetadataType {
                 this.keepTemplateFields(eventDef);
                 File.writeJSONToFile(
                     [templateDir, this.definition.type].join('/'),
-                    eventDef.customerKey + '.' + this.definition.type + '-meta',
-                    JSON.parse(Util.replaceByObject(JSON.stringify(eventDef), variables))
+                    originalKey + '.' + this.definition.type + '-meta',
+                    JSON.parse(Util.replaceByObject(JSON.stringify(eventDef), templateVariables))
                 );
                 Util.logger.info(
                     `EventDefinition.retrieveAsTemplate:: Written Metadata to filesystem (${name})`

package/lib/metadataTypes/FileTransfer.js

@@ -30,10 +30,10 @@ class FileTransfer extends MetadataType {
      * Retrieve a specific File Transfer Definition by Name
      * @param {String} templateDir Directory where retrieved metadata directory will be saved
      * @param {String} name name of the metadata file
-     * @param {Object} variables variables to be replaced in the metadata
+     * @param {Object} templateVariables variables to be replaced in the metadata
      * @returns {Promise} Promise
      */
-    static async retrieveAsTemplate(templateDir, name, variables) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables) {
         const options = {
             uri: '/automation/v1/filetransfers/?$filter=name%20eq%20' + name.split(' ').join('%20'),
         };
@@ -51,16 +51,20 @@ class FileTransfer extends MetadataType {
             const extended = await this.client.RestClient.get({
                 uri: '/automation/v1/filetransfers/' + metadata.id,
             });
+            const originalKey = extended.body[this.definition.keyField];
             const val = JSON.parse(
-                Util.replaceByObject(JSON.stringify(this.parseMetadata(extended.body)), variables)
+                Util.replaceByObject(
+                    JSON.stringify(this.parseMetadata(extended.body)),
+                    templateVariables
+                )
             );
 
             // remove all fields listed in Definition for templating
             this.keepTemplateFields(val);
             File.writeJSONToFile(
                 [templateDir, this.definition.type].join('/'),
-                val.customerKey + '.' + this.definition.type + '-meta',
-                JSON.parse(Util.replaceByObject(JSON.stringify(val), variables))
+                originalKey + '.' + this.definition.type + '-meta',
+                JSON.parse(Util.replaceByObject(JSON.stringify(val), templateVariables))
             );
             Util.logger.info(
                 `FileTransfer.retrieveAsTemplate:: Written Metadata to filesystem (${name})`

package/lib/metadataTypes/ImportFile.js

@@ -32,10 +32,10 @@ class ImportFile extends MetadataType {
      * Retrieve a specific Import Definition by Name
      * @param {String} templateDir Directory where retrieved metadata directory will be saved
      * @param {String} name name of the metadata file
-     * @param {Object} variables variables to be replaced in the metadata
+     * @param {Object} templateVariables variables to be replaced in the metadata
      * @returns {Promise} Promise
      */
-    static async retrieveAsTemplate(templateDir, name, variables) {
+    static async retrieveAsTemplate(templateDir, name, templateVariables) {
         const options = {
             uri: '/automation/v1/imports/?$filter=name%20eq%20' + name.split(' ').join('%20'),
         };
@@ -48,17 +48,20 @@ class ImportFile extends MetadataType {
                 Util.logger.error(`No ${this.definition.typeName} found with name "${name}"`);
                 return;
             }
-
+            const originalKey = metadata[this.definition.keyField];
             const val = JSON.parse(
-                Util.replaceByObject(JSON.stringify(this.parseMetadata(metadata)), variables)
+                Util.replaceByObject(
+                    JSON.stringify(this.parseMetadata(metadata)),
+                    templateVariables
+                )
             );
 
             // remove all fields listed in Definition for templating
             this.keepTemplateFields(val);
             File.writeJSONToFile(
                 [templateDir, this.definition.type].join('/'),
-                val.customerKey + '.' + this.definition.type + '-meta',
-                JSON.parse(Util.replaceByObject(JSON.stringify(val), variables))
+                originalKey + '.' + this.definition.type + '-meta',
+                JSON.parse(Util.replaceByObject(JSON.stringify(val), templateVariables))
             );
             Util.logger.info(
                 `ImportFile.retrieveAsTemplate:: Written Metadata to filesystem (${name})`
@@ -147,12 +150,10 @@ class ImportFile extends MetadataType {
             }
         }
         // When the destinationObjectTypeId is 584 is refers to Mobile Connect which is not supported as an Import Type
-        metadata.destinationObjectTypeId = this.definition.destinationObjectTypeMapping[
-            metadata.c__destinationType
-        ];
-        metadata.subscriberImportTypeId = this.definition.subscriberImportTypeMapping[
-            metadata.c__subscriberImportType
-        ];
+        metadata.destinationObjectTypeId =
+            this.definition.destinationObjectTypeMapping[metadata.c__destinationType];
+        metadata.subscriberImportTypeId =
+            this.definition.subscriberImportTypeMapping[metadata.c__subscriberImportType];
         metadata.updateTypeId = this.definition.updateTypeMapping[metadata.c__dataAction];
         return metadata;
     }

package/lib/metadataTypes/MetadataType.js

@@ -204,6 +204,16 @@ class MetadataType {
         const metadata = {};
         return { metadata: null, type: this.definition.type };
     }
+    /**
+     * Gets metadata from Marketing Cloud
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @param {Util.BuObject} buObject properties for auth
+     * @param {string} [subType] optionally limit to a single subtype
+     * @returns {Promise<{metadata:MetadataTypeMap,type:string}>} metadata
+     */
+    static retrieveChangelog(additionalFields, buObject, subType) {
+        return this.retrieve(null, additionalFields, buObject, subType);
+    }
 
     /**
      * Gets metadata cache with limited fields and does not store value to disk
@@ -304,9 +314,10 @@ class MetadataType {
                         });
                     } else if (this.cache[this.definition.type][normalizedKey]) {
                         // normal way of processing update files
-                        metadata[metadataKey][this.definition.idField] = this.cache[
-                            this.definition.type
-                        ][normalizedKey][this.definition.idField];
+                        metadata[metadataKey][this.definition.idField] =
+                            this.cache[this.definition.type][normalizedKey][
+                                this.definition.idField
+                            ];
                         metadataToUpdate.push({
                             before: this.cache[this.definition.type][normalizedKey],
                             after: metadata[metadataKey],
@@ -474,8 +485,12 @@ class MetadataType {
                 async () => (response = await this.client.RestClient.patch(options))
             );
             this.checkForErrors(response);
+            // some times, e.g. automation dont return a key in their update response and hence we need to fall back to name
             Util.logger.info(
-                `- updated ${this.definition.type}: ${metadataEntry[this.definition.keyField]}`
+                `- updated ${this.definition.type}: ${
+                    metadataEntry[this.definition.keyField] ||
+                    metadataEntry[this.definition.nameField]
+                }`
             );
             return response;
         } catch (ex) {
@@ -565,7 +580,6 @@ class MetadataType {
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
             );
-
             if (
                 buObject &&
                 this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type)
@@ -1058,11 +1072,11 @@ class MetadataType {
         const savedResults = {};
         const subtypeExtension = '.' + (overrideType || this.definition.type) + '-meta';
         let filterCounter = 0;
-        for (const metadataEntry in results) {
+        for (const originalKey in results) {
             try {
                 if (
-                    this.isFiltered(results[metadataEntry], true) ||
-                    this.isFiltered(results[metadataEntry], false)
+                    this.isFiltered(results[originalKey], true) ||
+                    this.isFiltered(results[originalKey], false)
                 ) {
                     // if current metadata entry is filtered don't save it
                     filterCounter++;
@@ -1071,21 +1085,21 @@ class MetadataType {
                 // define directory into which the current metdata shall be saved
                 const baseDir = [retrieveDir, ...(overrideType || this.definition.type).split('-')];
 
-                results[metadataEntry] = await this.postRetrieveTasks(
-                    results[metadataEntry],
+                results[originalKey] = await this.postRetrieveTasks(
+                    results[originalKey],
                     retrieveDir,
                     templateVariables ? true : false
                 );
-                if (!results[metadataEntry] || results[metadataEntry] === null) {
+                if (!results[originalKey] || results[originalKey] === null) {
                     // we encountered a situation in our postRetrieveTasks that made us want to filter this record
-                    delete results[metadataEntry];
+                    delete results[originalKey];
                     filterCounter++;
                     continue;
                 }
 
                 if (
-                    this.isFilteredFolder(results[metadataEntry], true) ||
-                    this.isFilteredFolder(results[metadataEntry], false)
+                    this.isFilteredFolder(results[originalKey], true) ||
+                    this.isFilteredFolder(results[originalKey], false)
                 ) {
                     // if current metadata entry is filtered don't save it
                     filterCounter++;
@@ -1093,20 +1107,20 @@ class MetadataType {
                 }
 
                 // for complex types like asset, script, query we need to save the scripts that were extracted from the JSON
-                if (results[metadataEntry].json && results[metadataEntry].codeArr) {
+                if (results[originalKey].json && results[originalKey].codeArr) {
                     // replace market values with template variable placeholders (do not do it on .codeArr)
                     if (templateVariables) {
-                        results[metadataEntry].json = Util.replaceByObject(
-                            results[metadataEntry].json,
+                        results[originalKey].json = Util.replaceByObject(
+                            results[originalKey].json,
                             templateVariables
                         );
-                        results[metadataEntry].subFolder = Util.replaceByObject(
-                            results[metadataEntry].subFolder,
+                        results[originalKey].subFolder = Util.replaceByObject(
+                            results[originalKey].subFolder,
                             templateVariables
                         );
                     }
 
-                    const postRetrieveData = results[metadataEntry];
+                    const postRetrieveData = results[originalKey];
                     if (postRetrieveData.subFolder) {
                         // very complex types have their own subfolder
                         baseDir.push(...postRetrieveData.subFolder);
@@ -1127,13 +1141,13 @@ class MetadataType {
                         );
                     }
                     // normalize results[metadataEntry]
-                    results[metadataEntry] = postRetrieveData.json;
+                    results[originalKey] = postRetrieveData.json;
                 } else {
                     // not a complex type, run the the entire JSON through templating
                     // replace market values with template variable placeholders
                     if (templateVariables) {
-                        results[metadataEntry] = Util.replaceByObject(
-                            results[metadataEntry],
+                        results[originalKey] = Util.replaceByObject(
+                            results[originalKey],
                             templateVariables
                         );
                     }
@@ -1142,7 +1156,7 @@ class MetadataType {
                 // we dont store Id on local disk, but we need it for caching logic,
                 // so its in retrieve but not in save. Here we put into the clone so that the original
                 // object used for caching doesnt have the Id removed.
-                const saveClone = JSON.parse(JSON.stringify(results[metadataEntry]));
+                const saveClone = JSON.parse(JSON.stringify(results[originalKey]));
                 if (!this.definition.keepId) {
                     delete saveClone[this.definition.idField];
                 }
@@ -1152,22 +1166,16 @@ class MetadataType {
                 } else {
                     this.keepRetrieveFields(saveClone);
                 }
-                savedResults[metadataEntry] = saveClone;
+                savedResults[originalKey] = saveClone;
                 File.writeJSONToFile(
                     // manage subtypes
                     baseDir,
-                    saveClone[this.definition.keyField] + subtypeExtension,
+                    originalKey + subtypeExtension,
                     saveClone
                 );
             } catch (ex) {
                 console.log(ex.stack);
-                Util.metadataLogger(
-                    'error',
-                    this.definition.type,
-                    'saveResults',
-                    ex,
-                    metadataEntry
-                );
+                Util.metadataLogger('error', this.definition.type, 'saveResults', ex, originalKey);
             }
         }
         if (filterCounter) {

package/lib/metadataTypes/Query.js

@@ -144,9 +144,8 @@ class Query extends MetadataType {
         } catch (ex) {
             throw new Error(`Query '${metadata.key}': ${ex.message}`);
         }
-        metadata.targetUpdateTypeId = this.definition.targetUpdateTypeMapping[
-            metadata.targetUpdateTypeName
-        ];
+        metadata.targetUpdateTypeId =
+            this.definition.targetUpdateTypeMapping[metadata.targetUpdateTypeName];
         return metadata;
     }
 

package/lib/metadataTypes/Role.js

@@ -107,14 +107,19 @@ class Role extends MetadataType {
             return;
         }
         if (!metadata) {
-            metadata = this.readBUMetadataForType(
-                File.normalizePath([
-                    this.properties.directories.retrieve,
-                    buObject.credential,
-                    Util.parentBuName,
-                ]),
-                true
-            ).role;
+            try {
+                metadata = this.readBUMetadataForType(
+                    File.normalizePath([
+                        this.properties.directories.retrieve,
+                        buObject.credential,
+                        Util.parentBuName,
+                    ]),
+                    true
+                ).role;
+            } catch (ex) {
+                Util.logger.error(ex.message);
+                return;
+            }
         }
         const directory = this.properties.directories.roles;
 

package/lib/metadataTypes/Script.js

@@ -178,12 +178,12 @@ class Script extends MetadataType {
         templateName
     ) {
         // get SSJS from filesystem
-        let code = this._mergeCode(metadata, templateDir, templateName);
-
+        let code = await this._mergeCode(metadata, templateDir, templateName);
         // replace template variables with their values
         try {
             code = Mustache.render(code, variables);
         } catch (ex) {
+            Util.logger.debug('script.buildDefinitionForExtracts: ' + ex.message);
             throw new Error(
                 `${this.definition.type}:: Error applying template variables on ${
                     metadata[this.definition.keyField] + '.' + this.definition.type