mcdev 7.9.0 → 7.10.1

package/lib/metadataTypes/Verification.js

@@ -29,6 +29,7 @@ import cache from '../util/cache.js';
  * @augments MetadataType
  */
 class Verification extends MetadataType {
+    static verificationIdKeyMap;
     /**
      * Retrieves Metadata of Data Verification Activity.
      *
@@ -39,51 +40,63 @@ class Verification extends MetadataType {
      * @returns {Promise.<MetadataTypeMapObj>} Promise of metadata
      */
     static async retrieve(retrieveDir, _, __, key) {
-        let paramArr = [];
-        if (key?.startsWith('id:')) {
-            paramArr = [key.slice(3)];
-        } else if (key) {
-            paramArr = [key];
+        const paramArr = [];
+        let automationKey;
+        if (key) {
+            const regex = /^(.*?)__s\d{1,3}\.\d{1,3}$/;
+            const match = key.match(regex);
+            if (match) {
+                // automation key found
+                automationKey = match[1];
+            } else {
+                // invalid key, unset it
+                Util.logger.error(`Invalid key: ${key}`);
+                return;
+            }
         }
-        if (!paramArr.length) {
-            // there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities
-            Util.logger.info(` - Caching dependent Metadata: automation`);
-            Automation.client = this.client;
-            Automation.buObject = this.buObject;
-            Automation.properties = this.properties;
-            Automation._skipNotificationRetrieve = true;
-            delete Automation._cachedMetadataMap;
-            const automationsMapObj = await Automation.retrieve();
-            delete Automation._skipNotificationRetrieve;
-            if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) {
-                if (!key) {
-                    // if we are not retrieving a single item, cache the automations for later use during retrieval of automations
-                    Automation._cachedMetadataMap = automationsMapObj?.metadata;
-                }
-                // automations found, lets iterate over their activities to find the dataVerification items
-                const dataVerificationIds = [];
-                for (const automation of Object.values(automationsMapObj.metadata)) {
-                    if (automation.steps) {
-                        for (const step of automation.steps) {
-                            for (const activity of step.activities) {
-                                if (
-                                    activity.objectTypeId === 1000 &&
-                                    activity.activityObjectId &&
-                                    activity.activityObjectId !==
-                                        '00000000-0000-0000-0000-000000000000'
-                                ) {
-                                    dataVerificationIds.push(activity.activityObjectId);
-                                }
+        const results = {};
+        // there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities
+        Util.logger.info(` - Caching dependent Metadata: automation`);
+        Automation.client = this.client;
+        Automation.buObject = this.buObject;
+        Automation.properties = this.properties;
+        Automation._skipNotificationRetrieve = true;
+        delete Automation._cachedMetadataMap;
+        const automationsMapObj = automationKey
+            ? await Automation.retrieve(undefined, undefined, undefined, automationKey)
+            : await Automation.retrieve();
+        delete Automation._skipNotificationRetrieve;
+        if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) {
+            if (!key) {
+                // if we are not retrieving a single item, cache the automations for later use during retrieval of automations
+                Automation._cachedMetadataMap = automationsMapObj?.metadata;
+            }
+            // automations found, lets iterate over their activities to find the dataVerification items
+            this.verificationIdKeyMap = {};
+            for (const automation of Object.values(automationsMapObj.metadata)) {
+                if (automation.steps) {
+                    for (const step of automation.steps) {
+                        // ideally one would use activity.displayOrder here but that doesnt always start at 1 nor is it always sequential. To avoid cross-BU issues, we use a custom order
+                        let order = 1;
+                        for (const activity of step.activities) {
+                            if (
+                                activity.objectTypeId === 1000 &&
+                                activity.activityObjectId &&
+                                activity.activityObjectId !== '00000000-0000-0000-0000-000000000000'
+                            ) {
+                                // log the verification id
+                                this.verificationIdKeyMap[activity.activityObjectId] =
+                                    `${automation.key}__s${step.step}.${order}`;
                             }
+                            order++;
                         }
                     }
                 }
-                if (dataVerificationIds.length) {
-                    paramArr.push(...dataVerificationIds);
-                }
+            }
+            if (Object.keys(this.verificationIdKeyMap).length) {
+                paramArr.push(...Object.keys(this.verificationIdKeyMap));
             }
         }
-        const results = {};
         if (paramArr.length) {
             const response = await this.retrieveRESTcollection(
                 paramArr.map((id) => ({ id, uri: '/automation/v1/dataverifications/' + id })),
@@ -131,7 +144,7 @@ class Verification extends MetadataType {
     }
 
     /**
-     * Retrieves Metadata of  Data Extract Activity for caching
+     * Retrieves Metadata of item for caching
      *
      * @returns {Promise.<MetadataTypeMapObj>} Promise of metadata
      */
@@ -140,9 +153,9 @@ class Verification extends MetadataType {
     }
 
     /**
-     * Creates a single Data Extract
+     * Creates a single item
      *
-     * @param {VerificationItem} metadata a single Data Extract
+     * @param {VerificationItem} metadata a single item
      * @returns {Promise} Promise
      */
     static create(metadata) {
@@ -161,31 +174,41 @@ class Verification extends MetadataType {
         if (!apiResponse?.[this.definition.idField]) {
             return;
         }
-        Util.logger.warn(
-            ` - ${this.definition.type} ${
-                metadataEntryWithAllFields?.[this.definition.idField]
-            }: new key ${
-                apiResponse?.[this.definition.idField]
-            } automatically assigned during creation`
-        );
+        // update apiResponse to ensure the new metadata is saved correctly on disk
+        apiResponse[this.definition.keyField] =
+            metadataEntryWithAllFields?.[this.definition.keyField];
+
+        // update info on metadataEntry to allow for proper logs
+        metadataEntry[this.definition.keyField] =
+            metadataEntryWithAllFields?.[this.definition.keyField];
         metadataEntry[this.definition.idField] = apiResponse?.[this.definition.idField];
 
-        // map structure: cred/bu --> type --> old key --> new key
-        const buName = this.buObject.credential + '/' + this.buObject.businessUnit;
-        Automation.createdKeyMap ||= {};
-        Automation.createdKeyMap[buName] ||= {};
-        Automation.createdKeyMap[buName][this.definition.type] ||= {};
-        Automation.createdKeyMap[buName][this.definition.type][
-            metadataEntryWithAllFields[this.definition.idField]
-        ] = metadataEntry[this.definition.idField];
+        return apiResponse;
+    }
+    /**
+     * helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP}
+     *
+     * @param {MetadataTypeItem} metadataEntry a single metadata Entry
+     * @param {object} apiResponse varies depending on the API call
+     * @param {MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
+     * @returns {Promise.<object>} apiResponse, potentially modified
+     */
+    static postUpdateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {
+        // update apiResponse to ensure the new metadata is saved correctly on disk
+        apiResponse[this.definition.keyField] =
+            metadataEntryWithAllFields?.[this.definition.keyField];
 
+        // update info on metadataEntry to allow for proper logs
+        metadataEntry[this.definition.keyField] =
+            metadataEntryWithAllFields?.[this.definition.keyField];
+        metadataEntry[this.definition.idField] = apiResponse?.[this.definition.idField];
         return apiResponse;
     }
 
     /**
-     * Updates a single Data Extract
+     * Updates a single item
      *
-     * @param {VerificationItem} metadata a single Data Extract
+     * @param {VerificationItem} metadata a single item
      * @returns {Promise} Promise
      */
     static update(metadata) {
@@ -209,9 +232,22 @@ class Verification extends MetadataType {
             'ObjectID'
         );
         delete metadata.r__dataExtension_key;
+
         return metadata;
     }
 
+    /**
+     * helper for {@link parseResponseBody} that creates a custom key field for this type based on mobileCode and keyword
+     *
+     * @param {MetadataTypeItem} metadata single item
+     */
+    static createCustomKeyField(metadata) {
+        if (this.verificationIdKeyMap[metadata[this.definition.idField]]) {
+            metadata[this.definition.keyField] =
+                this.verificationIdKeyMap[metadata[this.definition.idField]];
+        }
+    }
+
     /**
      * parses retrieved Metadata before saving
      *
@@ -257,7 +293,7 @@ class Verification extends MetadataType {
      * @returns {Promise.<boolean>} deletion success status
      */
     static deleteByKey(key) {
-        return super.deleteByKeyREST('/automation/v1/dataverifications/' + key, key);
+        return super.deleteByKeyREST('/automation/v1/dataverifications/' + key, key, 400);
     }
 }
 

package/lib/metadataTypes/definitions/Automation.definition.js

@@ -53,10 +53,10 @@ export default {
     keyField: 'key',
     nameField: 'name',
     folderIdField: 'categoryId',
-    createdDateField: 'createdDate', // only returned by upsert
-    createdNameField: 'createdByName', // only returned by upsert
-    lastmodDateField: 'lastSavedDate', // only returned by upsert
-    lastmodNameField: 'lastSavedByName', // only returned by upsert
+    createdDateField: 'createdDate', // only returned by upsert & legacy
+    createdNameField: 'createdName', // only returned by upsert & legacy
+    lastmodDateField: 'modifiedDate', // only returned by upsert & legacy
+    lastmodNameField: 'modifiedName', // only returned by upsert & legacy
     restPagination: true,
     maxKeyLength: 200, // confirmed max length
     scheduleTypeMapping: {
@@ -371,16 +371,46 @@ export default {
             retrieving: false, // only returned by upsert
             template: false,
         },
-        createdDate: {
+        createdByName: {
             isCreateable: false,
             isUpdateable: false,
             retrieving: false, // only returned by upsert
             template: false,
         },
-        createdByName: {
+        createdDate: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true, // only returned by upsert + legacy
+            template: false,
+        },
+        createdName: {
             isCreateable: false,
             isUpdateable: false,
-            retrieving: false, // only returned by upsertt
+            retrieving: true, // only returned by legacy
+            template: false,
+        },
+        modifiedDate: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true, // only returned by legacy
+            template: false,
+        },
+        modifiedName: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true, // only returned by legacy
+            template: false,
+        },
+        pausedDate: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true, // only returned by legacy
+            template: false,
+        },
+        pausedName: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true, // only returned by legacy
             template: false,
         },
         updateInProgress: {
@@ -515,6 +545,12 @@ export default {
             retrieving: false,
             template: false,
         },
+        'schedule.scheduleTypeId': {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
         status: {
             isCreateable: true,
             isUpdateable: true,

package/lib/metadataTypes/definitions/DataExtract.definition.js

@@ -1,7 +1,9 @@
 export default {
     bodyIteratorField: 'items',
-    dependencies: ['dataExtractType', 'user'],
-    dependencyGraph: null, // dataExtractType cannot be deployed
+    dependencies: ['dataExtension', 'dataExtractType', 'user'],
+    dependencyGraph: {
+        dataExtension: ['r__dataExtension_key'],
+    },
     hasExtended: true,
     idField: 'dataExtractDefinitionId',
     keyIsFixed: false,
@@ -100,6 +102,17 @@ export default {
             retrieving: true,
             template: true,
         },
-        r__dataExtractType_name: { skipValidation: true },
+        r__dataExtractType_name: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        r__dataExtension_key: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
     },
 };

package/lib/metadataTypes/definitions/DomainVerification.definition.js

@@ -16,7 +16,7 @@ export default {
     type: 'domainVerification',
     typeDescription: 'Domains emails that are verified for sending',
     typeRetrieveByDefault: true,
-    typeCdpByDefault: false,
+    typeCdpByDefault: true,
     typeName: 'Domain Verification',
     fields: {
         domain: {

package/lib/metadataTypes/definitions/SenderProfile.definition.js

@@ -1,7 +1,9 @@
 export default {
     bodyIteratorField: 'Results',
     dependencies: ['user', 'domainVerification'],
-    dependencyGraph: null,
+    dependencyGraph: {
+        domainVerification: ['FallbackFromAddress', 'FromAddress'],
+    },
     filter: {},
     hasExtended: false,
     idField: 'ObjectID',

package/lib/metadataTypes/definitions/Verification.definition.js

@@ -5,12 +5,12 @@ export default {
     hasExtended: false,
     idField: 'dataVerificationDefinitionId',
     keyIsFixed: true,
-    keyField: 'dataVerificationDefinitionId',
+    keyField: 'c__automation_step',
     createdDateField: null,
     createdNameField: 'createdBy',
     lastmodDateField: null,
     lastmodNameField: null,
-    nameField: 'dataVerificationDefinitionId',
+    nameField: 'c__automation_step',
     restPagination: false,
     maxKeyLength: 36, // confirmed max length
     type: 'verification',
@@ -86,5 +86,11 @@ export default {
             retrieving: true,
             template: true,
         },
+        c__automation_step: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
     },
 };

package/lib/util/util.js

@@ -1317,6 +1317,18 @@ export const Util = {
                 return acc;
             }, {});
     },
+    /**
+     * helper for MetadataType class to issue a similar error message for unsupported methods
+     *
+     * @param {any} definition type definition object
+     * @param {string} method name of the method thats not supported
+     * @param {MetadataTypeItem} [item] metadata item
+     */
+    notSupportedError: function (definition, method, item) {
+        Util.logger.error(
+            ` ☇ skipping ${item ? Util.getTypeKeyName(definition, item) : definition.type}: ${method} is not supported yet for ${definition.type}`
+        );
+    },
 };
 
 Util.startLogger(false, true);

package/lib/util/validations.js

@@ -111,7 +111,9 @@ export default async function validation(definition, item, targetDir, codeExtrac
                 const doNotEvaluate = ['automation', 'attributeSet', 'list'];
                 if (
                     doNotEvaluate.includes(definition.type) ||
-                    (definition.type === 'asset' && item?.assetType?.name === 'webpage')
+                    (definition.type === 'asset' && item?.assetType?.name === 'webpage') ||
+                    (definition.type === 'dataExtension' &&
+                        item.r__folder_Path === 'Synchronized Data Extensions')
                 ) {
                     // this subtype is not visible in the interface and hence always technically sits in the root
                     return true;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcdev",
-    "version": "7.9.0",
+    "version": "7.10.1",
     "description": "Accenture Salesforce Marketing Cloud DevTools",
     "author": "Accenture: joern.berkefeld, douglas.midgley, robert.zimmermann, maciej.barnas",
     "license": "MIT",

package/test/general.test.js

@@ -378,7 +378,7 @@ describe('GENERAL', () => {
                 );
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
-                    6,
+                    9,
                     'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
                 );
             });
@@ -512,7 +512,7 @@ describe('GENERAL', () => {
                 assert.equal(deployedTypes[0], 'dataExtract', 'deploy should have returned 1 type');
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
-                    10,
+                    13,
                     'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
                 );
             });
@@ -777,7 +777,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU', ['automation', 'query']);
 
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 35;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -882,7 +882,7 @@ describe('GENERAL', () => {
             });
 
             it('buildTemplate + buildDefinition for multiple types with keys and --retrieve', async () => {
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 32;
 
                 // preparation
                 const argvMetadata = [
@@ -987,7 +987,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU');
 
-                const expectedApiCallsRetrieve = 93;
+                const expectedApiCallsRetrieve = 97;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1025,15 +1025,16 @@ describe('GENERAL', () => {
                     Object.keys(templateResult),
                     [
                         'automation',
-                        'query',
+                        'dataExtension',
                         'dataExtract',
+                        'domainVerification',
                         'emailSend',
-                        'dataExtension',
-                        'sendClassification',
-                        'senderProfile',
                         'fileTransfer',
                         'importFile',
+                        'query',
                         'script',
+                        'sendClassification',
+                        'senderProfile',
                         'verification',
                     ],
                     'did not create deployment packages for all relevant types'
@@ -1145,15 +1146,16 @@ describe('GENERAL', () => {
                     Object.keys(templateResult),
                     [
                         'automation',
-                        'query',
+                        'dataExtension',
                         'dataExtract',
+                        'domainVerification',
                         'emailSend',
-                        'dataExtension',
-                        'sendClassification',
-                        'senderProfile',
                         'fileTransfer',
                         'importFile',
+                        'query',
                         'script',
+                        'sendClassification',
+                        'senderProfile',
                         'verification',
                     ],
                     'did not create deployment packages for all relevant types'
@@ -1227,7 +1229,7 @@ describe('GENERAL', () => {
                     await testUtils.getActualDeployFile('testTemplated_query', 'query', 'sql')
                 ).to.equal(await testUtils.getExpectedFile('9999999', 'query', 'build', 'sql'));
 
-                const expectedApiCallsRetrieve = 97;
+                const expectedApiCallsRetrieve = 101;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1239,7 +1241,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU', ['automation', 'query']);
 
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 35;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1431,7 +1433,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU', ['automation', 'query']);
 
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 35;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1502,7 +1504,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU', ['automation', 'query']);
 
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 35;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1601,7 +1603,7 @@ describe('GENERAL', () => {
                 // download everything before we test buildTemplate
                 await handler.retrieve('testInstance/testBU');
 
-                const expectedApiCallsRetrieve = 93;
+                const expectedApiCallsRetrieve = 97;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1660,15 +1662,16 @@ describe('GENERAL', () => {
                     Object.keys(definitionResult),
                     [
                         'automation',
-                        'query',
+                        'dataExtension',
                         'dataExtract',
+                        'domainVerification',
                         'emailSend',
-                        'dataExtension',
-                        'sendClassification',
-                        'senderProfile',
                         'fileTransfer',
                         'importFile',
+                        'query',
                         'script',
+                        'sendClassification',
+                        'senderProfile',
                         'verification',
                     ],
                     'did not create deployment packages for all relevant types'
@@ -1767,15 +1770,16 @@ describe('GENERAL', () => {
                     Object.keys(definitionResult),
                     [
                         'automation',
-                        'query',
+                        'dataExtension',
                         'dataExtract',
+                        'domainVerification',
                         'emailSend',
-                        'dataExtension',
-                        'sendClassification',
-                        'senderProfile',
                         'fileTransfer',
                         'importFile',
+                        'query',
                         'script',
+                        'sendClassification',
+                        'senderProfile',
                         'verification',
                     ],
                     'did not create deployment packages for all relevant types'
@@ -1810,7 +1814,7 @@ describe('GENERAL', () => {
                     await testUtils.getActualDeployFile('testTemplated_query', 'query', 'sql')
                 ).to.equal(await testUtils.getExpectedFile('9999999', 'query', 'build', 'sql'));
 
-                const expectedApiCallsRetrieve = 97;
+                const expectedApiCallsRetrieve = 101;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -1822,7 +1826,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU', ['automation', 'query']);
 
-                const expectedApiCallsRetrieve = 31;
+                const expectedApiCallsRetrieve = 35;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -2001,7 +2005,7 @@ describe('GENERAL', () => {
                 // download first before we test buildTemplate
                 await handler.retrieve('testInstance/testBU');
 
-                const expectedApiCallsRetrieve = 93;
+                const expectedApiCallsRetrieve = 97;
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
                     expectedApiCallsRetrieve,
@@ -2061,15 +2065,16 @@ describe('GENERAL', () => {
                     Object.keys(definitionResult),
                     [
                         'automation',
-                        'query',
+                        'dataExtension',
                         'dataExtract',
+                        'domainVerification',
                         'emailSend',
-                        'dataExtension',
-                        'sendClassification',
-                        'senderProfile',
                         'fileTransfer',
                         'importFile',
+                        'query',
                         'script',
+                        'sendClassification',
+                        'senderProfile',
                         'verification',
                     ],
                     'did not create deployment packages for all relevant types'
@@ -2503,7 +2508,7 @@ describe('GENERAL', () => {
                 );
                 assert.equal(
                     testUtils.getAPIHistoryLength(),
-                    9,
+                    12,
                     'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
                 );
             });

package/test/mockRoot/.mcdevrc.json

@@ -175,5 +175,5 @@
             "verification"
         ]
     },
-    "version": "7.9.0"
+    "version": "7.10.1"
 }