mcdev 5.2.0 → 6.0.0

package/lib/metadataTypes/TransactionalSMS.js

@@ -1,11 +1,11 @@
 'use strict';
 
-const TYPE = require('../../types/mcdev.d');
-const TransactionalMessage = require('./TransactionalMessage');
-const Util = require('../util/util');
-const File = require('../util/file');
-const beautifier = require('beauty-amp-core');
-const cache = require('../util/cache');
+import TYPE from '../../types/mcdev.d.js';
+import TransactionalMessage from './TransactionalMessage.js';
+import { Util } from '../util/util.js';
+import File from '../util/file.js';
+import beautifier from 'beauty-amp-core';
+import cache from '../util/cache.js';
 
 /**
  * TransactionalSMS MetadataType
@@ -341,6 +341,7 @@ class TransactionalSMS extends TransactionalMessage {
 }
 
 // Assign definition to static attributes
-TransactionalSMS.definition = require('../MetadataTypeDefinitions').transactionalSMS;
+import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
+TransactionalSMS.definition = MetadataTypeDefinitions.transactionalSMS;
 
-module.exports = TransactionalSMS;
+export default TransactionalSMS;

package/lib/metadataTypes/TriggeredSend.js

@@ -1,15 +1,12 @@
 'use strict';
 
-const TYPE = require('../../types/mcdev.d');
-const MetadataType = require('./MetadataType');
-const Util = require('../util/util');
-const cache = require('../util/cache');
-
-const cacheTypes = {
-    asset: require('./Asset'),
-    folder: require('./Folder'),
-    list: require('./List'),
-};
+import TYPE from '../../types/mcdev.d.js';
+import MetadataType from './MetadataType.js';
+import { Util } from '../util/util.js';
+import cache from '../util/cache.js';
+import asset from './Asset.js';
+import folder from './Folder.js';
+import list from './List.js';
 
 /**
  * MessageSendActivity MetadataType
@@ -273,6 +270,11 @@ class TriggeredSend extends MetadataType {
             asset: ['message'],
             list: null,
         };
+        const cacheTypes = {
+            asset,
+            folder,
+            list,
+        };
         for (const [type, subTypeArr] of Object.entries(requiredCache)) {
             if (type === 'asset' && assetLoaded) {
                 continue;
@@ -406,6 +408,7 @@ class TriggeredSend extends MetadataType {
 }
 
 // Assign definition to static attributes
-TriggeredSend.definition = require('../MetadataTypeDefinitions').triggeredSend;
+import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
+TriggeredSend.definition = MetadataTypeDefinitions.triggeredSend;
 
-module.exports = TriggeredSend;
+export default TriggeredSend;

package/lib/metadataTypes/User.js

@@ -1,10 +1,10 @@
 'use strict';
 
-const TYPE = require('../../types/mcdev.d');
-const MetadataType = require('./MetadataType');
-const Util = require('../util/util');
-const File = require('../util/file');
-const cache = require('../util/cache');
+import TYPE from '../../types/mcdev.d.js';
+import MetadataType from './MetadataType.js';
+import { Util } from '../util/util.js';
+import File from '../util/file.js';
+import cache from '../util/cache.js';
 
 /**
  * MetadataType
@@ -1169,6 +1169,7 @@ class User extends MetadataType {
 }
 
 // Assign definition to static attributes
-User.definition = require('../MetadataTypeDefinitions').user;
+import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
+User.definition = MetadataTypeDefinitions.user;
 
-module.exports = User;
+export default User;

package/lib/metadataTypes/Verification.js

@@ -0,0 +1,230 @@
+'use strict';
+
+import Automation from './Automation.js';
+import MetadataType from './MetadataType.js';
+import TYPE from '../../types/mcdev.d.js';
+import { Util } from '../util/util.js';
+import cache from '../util/cache.js';
+/**
+ * Verification MetadataType
+ *
+ * @augments MetadataType
+ */
+class Verification extends MetadataType {
+    /**
+     * Retrieves Metadata of Data Verification Activity.
+     *
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {void} [_] unused parameter
+     * @param {void} [__] unused parameter
+     * @param {string} key customer key of single item to retrieve
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
+     */
+    static async retrieve(retrieveDir, _, __, key) {
+        let paramArr = [];
+        if (key?.startsWith('id:')) {
+            paramArr = [key.slice(3)];
+        } else if (key) {
+            paramArr = [key];
+        }
+        if (!paramArr.length) {
+            // there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities
+            Util.logger.info(` - Caching dependent Metadata: automation`);
+            Automation.client = this.client;
+            Automation.buObject = this.buObject;
+            Automation.properties = this.properties;
+            Automation._skipNotificationRetrieve = true;
+            delete Automation._cachedMetadataMap;
+            const automationsMapObj = await Automation.retrieve();
+            delete Automation._skipNotificationRetrieve;
+            if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) {
+                if (!key) {
+                    // if we are not retrieving a single item, cache the automations for later use during retrieval of automations
+                    Automation._cachedMetadataMap = automationsMapObj?.metadata;
+                }
+                // automations found, lets iterate over their activities to find the dataVerification items
+                const dataVerificationIds = [];
+                for (const automation of Object.values(automationsMapObj.metadata)) {
+                    if (automation.steps) {
+                        for (const step of automation.steps) {
+                            for (const activity of step.activities) {
+                                if (
+                                    activity.objectTypeId === 1000 &&
+                                    activity.activityObjectId &&
+                                    activity.activityObjectId !==
+                                        '00000000-0000-0000-0000-000000000000'
+                                ) {
+                                    dataVerificationIds.push(activity.activityObjectId);
+                                }
+                            }
+                        }
+                    }
+                }
+                if (dataVerificationIds.length) {
+                    paramArr.push(...dataVerificationIds);
+                }
+            }
+        }
+        const results = {};
+        if (paramArr.length) {
+            const response = await this.retrieveRESTcollection(
+                paramArr.map((id) => ({ id, uri: '/automation/v1/dataverifications/' + id })),
+                undefined,
+                !key
+            );
+            if (response?.metadata) {
+                Object.assign(results, response.metadata);
+            }
+        }
+        if (retrieveDir) {
+            const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
+            Util.logger.info(
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
+                    Util.getKeysString(key)
+            );
+        }
+
+        return {
+            metadata: results,
+            type: this.definition.type,
+        };
+    }
+    /**
+     * helper for {@link this.retrieveRESTcollection}
+     *
+     * @param {Error} ex exception
+     * @param {string} id id or key of item
+     * @returns {null} -
+     */
+    static handleRESTErrors(ex, id) {
+        if (ex.message === 'Not Found' || ex.message === 'Request failed with status code 400') {
+            // if the ID is too short, the system will throw the 400 error
+            Util.logger.debug(
+                ` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`
+            );
+        } else {
+            // if we do get here, we should log the error and continue instead of failing to download all automations
+            Util.logger.error(
+                ` ☇ skipping ${this.definition.type} ${id}: ${ex.message} ${ex.code}`
+            );
+        }
+        return null;
+    }
+
+    /**
+     * Retrieves Metadata of  Data Extract Activity for caching
+     *
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
+     */
+    static async retrieveForCache() {
+        return this.retrieve();
+    }
+
+    /**
+     * Creates a single Data Extract
+     *
+     * @param {TYPE.VerificationItem} metadata a single Data Extract
+     * @returns {Promise} Promise
+     */
+    static create(metadata) {
+        return super.createREST(metadata, '/automation/v1/dataverifications/');
+    }
+
+    /**
+     * helper for {@link MetadataType.createREST}
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @param {TYPE.MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
+     * @returns {void}
+     */
+    static async postCreateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {
+        if (!apiResponse?.[this.definition.idField]) {
+            return;
+        }
+        Util.logger.warn(
+            ` - ${this.definition.type} ${
+                metadataEntryWithAllFields?.[this.definition.idField]
+            }: new key ${
+                apiResponse?.[this.definition.idField]
+            } automatically assigned during creation`
+        );
+        metadataEntry[this.definition.idField] = apiResponse?.[this.definition.idField];
+
+        // map structure: cred/bu --> type --> old key --> new key
+        const buName = this.buObject.credential + '/' + this.buObject.businessUnit;
+        Automation.createdKeyMap ||= {};
+        Automation.createdKeyMap[buName] ||= {};
+        Automation.createdKeyMap[buName][this.definition.type] ||= {};
+        Automation.createdKeyMap[buName][this.definition.type][
+            metadataEntryWithAllFields[this.definition.idField]
+        ] = metadataEntry[this.definition.idField];
+    }
+
+    /**
+     * Updates a single Data Extract
+     *
+     * @param {TYPE.VerificationItem} metadata a single Data Extract
+     * @returns {Promise} Promise
+     */
+    static update(metadata) {
+        return super.updateREST(
+            metadata,
+            '/automation/v1/dataverifications/' + metadata.dataVerificationDefinitionId
+        );
+    }
+
+    /**
+     * prepares a verification for deployment
+     *
+     * @param {TYPE.VerificationItem} metadata a single verification activity definition
+     * @returns {TYPE.VerificationItem} metadata object
+     */
+    static preDeployTasks(metadata) {
+        metadata.targetObjectId = cache.searchForField(
+            'dataExtension',
+            metadata.r__dataExtension_CustomerKey,
+            'CustomerKey',
+            'ObjectID'
+        );
+        delete metadata.r__dataExtension_CustomerKey;
+        return metadata;
+    }
+    /**
+     * parses retrieved Metadata before saving
+     *
+     * @param {TYPE.VerificationItem} metadata a single verification activity definition
+     * @returns {TYPE.VerificationItem} Array with one metadata object and one sql string
+     */
+    static postRetrieveTasks(metadata) {
+        try {
+            metadata.r__dataExtension_CustomerKey = cache.searchForField(
+                'dataExtension',
+                metadata.targetObjectId,
+                'ObjectID',
+                'CustomerKey'
+            );
+            delete metadata.targetObjectId;
+        } catch (ex) {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}`
+            );
+        }
+        return metadata;
+    }
+    /**
+     * Delete a metadata item from the specified business unit
+     *
+     * @param {string} key Identifier of item
+     * @returns {Promise.<boolean>} deletion success status
+     */
+    static deleteByKey(key) {
+        return super.deleteByKeyREST('/automation/v1/dataverifications/' + key, key);
+    }
+}
+
+// Assign definition to static attributes
+import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
+Verification.definition = MetadataTypeDefinitions.verification;
+
+export default Verification;

package/lib/metadataTypes/definitions/Asset.definition.js

@@ -1,5 +1,5 @@
 // asset types https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/base-asset-types.htm
-module.exports = {
+export default {
     bodyIteratorField: 'items',
     dependencies: ['folder-asset', 'folder-asset-shared'],
     folderType: 'asset',

package/lib/metadataTypes/definitions/AttributeGroup.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'attributeGroupDefinitions',
     dependencies: ['attributeSet'], // future may have dependency on Data Extensions
     hasExtended: false,
@@ -8,8 +8,8 @@ module.exports = {
     nameField: 'definitionName.value',
     restPagination: false, // Hub API does not support pagination and returns everything instead
     type: 'attributeGroup',
-    typeDescription: 'BETA: Groupings of Set Definitions (Data Extensions) in Data Designer.',
-    typeRetrieveByDefault: false,
+    typeDescription: 'Groupings of Attribute Sets (Data Extensions) in Data Designer.',
+    typeRetrieveByDefault: true,
     typeName: 'Data Designer Attribute Groups',
     fields: {
         applicationID: {

package/lib/metadataTypes/definitions/AttributeSet.definition.js

@@ -1,6 +1,15 @@
-module.exports = {
+export default {
     bodyIteratorField: 'setDefinition',
-    dependencies: ['folder-hidden', 'folder-dataextension', 'dataExtension'], // future may have dependency on Data Extensions
+    dependencies: [
+        'folder-hidden',
+        'folder-dataextension',
+        'folder-salesforcedataextension',
+        'folder-shared_data',
+        'folder-shared_dataextension',
+        'folder-shared_salesforcedataextension',
+        'folder-synchronizeddataextension',
+        'dataExtension',
+    ],
     hasExtended: false,
     idField: 'definitionID',
     keyIsFixed: null,
@@ -13,9 +22,9 @@ module.exports = {
     lastmodNameField: null,
     restPagination: false, // Hub API does not support pagination and returns everything instead
     type: 'attributeSet',
-    typeDescription: 'BETA: Data Extensions linked to Attribute Groups in Data Designer.',
-    typeRetrieveByDefault: false,
-    typeName: 'Data Designer Set Definitions',
+    typeDescription: 'Data Extensions linked together in Attribute Groups in Data Designer.',
+    typeRetrieveByDefault: true,
+    typeName: 'Data Designer Attribute Sets',
     fields: {
         applicationID: {
             isCreateable: null,
@@ -119,6 +128,12 @@ module.exports = {
             retrieving: true,
             template: null,
         },
+        'dataRetentionProperties.periodLength': {
+            isCreateable: null,
+            isUpdateable: null,
+            retrieving: true,
+            template: null,
+        },
         definitionID: {
             isCreateable: null,
             isUpdateable: null,
@@ -138,14 +153,14 @@ module.exports = {
             template: false,
         },
         'definitionName.value': {
-            // equal to 'name'; auto-populated by preDeployTasks
+            // equal to 'name'
             isCreateable: true,
             isUpdateable: true,
             retrieving: false,
             template: false,
         },
         fullyQualifiedName: {
-            // equal to 'name'; auto-populated by preDeployTasks
+            // equal to 'name'
             isCreateable: true,
             isUpdateable: true,
             retrieving: false,
@@ -602,9 +617,10 @@ module.exports = {
             template: null,
         },
         'valueDefinitions[].baseType': {
+            // "Numeric", "Text", ... valueDefinitions[].dataType is more relevant
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].customerDataID': {
@@ -613,6 +629,12 @@ module.exports = {
             retrieving: true,
             template: null,
         },
+        'valueDefinitions[].connectingID': {
+            isCreateable: null,
+            isUpdateable: null,
+            retrieving: false,
+            template: null,
+        },
         'valueDefinitions[].dataSourceID': {
             isCreateable: null,
             isUpdateable: null,
@@ -622,7 +644,7 @@ module.exports = {
         'valueDefinitions[].dataSourceName': {
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].dataType': {
@@ -638,9 +660,10 @@ module.exports = {
             template: null,
         },
         'valueDefinitions[].definitionID': {
+            // likely the main ID of the value definition. No use for simple checks on git though as long as we cannot update it
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].definitionKey': {
@@ -649,10 +672,11 @@ module.exports = {
             retrieving: true,
             template: null,
         },
-        'valueDefinitions[].definitionName.value': {
+        'valueDefinitions[].definitionName': {
+            // equal to valueDefinitions[].name
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].description': {
@@ -662,15 +686,17 @@ module.exports = {
             template: null,
         },
         'valueDefinitions[].displayOrder': {
+            // merely a numeric counter; equal to valueDefinitions[].ordinal; not given for isHidden:true entries
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].fullyQualifiedName': {
+            // dataExtension name + field name
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].identifierType': {
@@ -727,10 +753,11 @@ module.exports = {
             retrieving: true,
             template: null,
         },
-        'valueDefinitions[].localizedDescription.value': {
+        'valueDefinitions[].localizedDescription': {
+            // always equal to { value: "" }
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].name': {
@@ -739,6 +766,13 @@ module.exports = {
             retrieving: true,
             template: null,
         },
+        'valueDefinitions[].obfuscationProperties': {
+            // might become relevant when fields are encrypted but for most cases we should simply skip it
+            isCreateable: null,
+            isUpdateable: null,
+            retrieving: false,
+            template: null,
+        },
         'valueDefinitions[].obfuscationProperties.maskType': {
             isCreateable: null,
             isUpdateable: null,
@@ -770,9 +804,10 @@ module.exports = {
             template: null,
         },
         'valueDefinitions[].ordinal': {
+            // merely a numeric counter; equal to valueDefinitions[].displayOrder; not given for isHidden:true entries
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].parentDefinition': {
@@ -788,9 +823,10 @@ module.exports = {
             template: null,
         },
         'valueDefinitions[].parentType': {
+            // always "Set"
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].restrictionLookupListID': {
@@ -817,40 +853,51 @@ module.exports = {
             retrieving: false,
             template: null,
         },
-        'valueDefinitions[].setDefinitionName.value': {
+        'valueDefinitions[].setDefinitionName': {
+            isCreateable: null,
+            isUpdateable: null,
+            retrieving: false,
+            template: null,
+        },
+        'valueDefinitions[].storageFieldReferenceID': {
             isCreateable: null,
             isUpdateable: null,
             retrieving: false,
             template: null,
         },
         'valueDefinitions[].storageFieldReferenceID.type': {
+            // always "guid"
             isCreateable: null,
             isUpdateable: null,
             retrieving: true,
             template: null,
         },
         'valueDefinitions[].storageFieldReferenceID.value': {
+            // unknown GUID
             isCreateable: null,
             isUpdateable: null,
             retrieving: true,
             template: null,
         },
         'valueDefinitions[].storageName': {
+            // always equal valueDefinitions[].definitionKey, except for when that is CustomObjectKey - then this will be _CustomObjectKey (with an underscore)
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].valueDefinitionID': {
+            // equal to valueDefinitions[].definitionID
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         'valueDefinitions[].valueDefinitionKey': {
+            // equal to valueDefinitions[].definitionKey
             isCreateable: null,
             isUpdateable: null,
-            retrieving: true,
+            retrieving: false,
             template: null,
         },
         r__folder_Path: {
@@ -859,5 +906,11 @@ module.exports = {
             retrieving: true,
             template: null,
         },
+        r__dataExtension_CustomerKey: {
+            isCreateable: null,
+            isUpdateable: null,
+            retrieving: true,
+            template: null,
+        },
     },
 };

package/lib/metadataTypes/definitions/Automation.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     activityTypeMapping: {
         dataExtract: 73,
         dataFactoryUtility: 425,
@@ -31,6 +31,7 @@ module.exports = {
         'importFile',
         'query',
         'script',
+        'verification',
     ],
     folderType: 'automation',
     hasExtended: false,

package/lib/metadataTypes/definitions/Campaign.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'entry',
     dependencies: [],
     hasExtended: false,

package/lib/metadataTypes/definitions/ContentArea.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'Results',
     dependencies: ['folder'],
     hasExtended: false,

package/lib/metadataTypes/definitions/DataExtension.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'Results',
     dependencies: [
         'folder-hidden',

package/lib/metadataTypes/definitions/DataExtensionField.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'Results',
     dependencies: [],
     filter: {},

package/lib/metadataTypes/definitions/DataExtensionTemplate.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'Results',
     dependencies: [],
     filter: {},

package/lib/metadataTypes/definitions/DataExtract.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: 'items',
     dependencies: ['dataExtension', 'dataExtractType'],
     hasExtended: true,

package/lib/metadataTypes/definitions/DataExtractType.definition.js

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     bodyIteratorField: [],
     dependencies: [],
     hasExtended: false,