mcdev 7.0.3 → 7.1.0

package/lib/metadataTypes/Automation.js

@@ -19,6 +19,7 @@ import pLimit from 'p-limit';
  * @typedef {import('../../types/mcdev.d.js').SDKError} SDKError
  * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
  * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
  */
 /**
  * @typedef {import('../../types/mcdev.d.js').AutomationItem} AutomationItem
@@ -1620,6 +1621,39 @@ class Automation extends MetadataType {
         // delete local copy: retrieve/cred/bu/.../...-doc.md
         await super.postDeleteTasks(customerKey, [`${this.definition.type}-doc.md`]);
     }
+
+    /**
+     * helper for {@link MetadataType.getNestedValue} - adjusted for automation activities
+     *
+     * @param {any} obj the metadataItem to search in (or the result)
+     * @param {string[]} nestedKeyParts key in dot-notation split into parts
+     * @param {string} dependentType used for types that need custom handling
+     * @returns {(string) | (string)[]} result
+     */
+    static getNestedValueHelper(obj, nestedKeyParts, dependentType) {
+        if (nestedKeyParts.length == 0) {
+            // key was found; append '' to ensure we always return a string
+            return obj + '';
+        }
+        // get most left key
+        const key = nestedKeyParts.shift();
+        if (!obj[key]) {
+            // key was not found
+            return;
+        }
+        if (Array.isArray(obj[key])) {
+            return obj[key].flatMap((x) =>
+                this.getNestedValueHelper(x, [...nestedKeyParts], dependentType)
+            );
+        } else {
+            if (nestedKeyParts.length == 0) {
+                // key was found if length is 0 and search field (r__type) is equal the current type, else, keep going
+                return obj[key] === dependentType ? obj.r__key : undefined;
+            } else {
+                return this.getNestedValueHelper(obj[key], [...nestedKeyParts], dependentType);
+            }
+        }
+    }
 }
 
 // Assign definition to static attributes

package/lib/metadataTypes/DataExtension.js

@@ -10,7 +10,7 @@ import File from '../util/file.js';
 import auth from '../util/auth.js';
 import cache from '../util/cache.js';
 import pLimit from 'p-limit';
-import inquirer from 'inquirer';
+import { checkbox } from '@inquirer/prompts';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -33,7 +33,7 @@ import inquirer from 'inquirer';
  * @augments MetadataType
  */
 class DataExtension extends MetadataType {
-    /** @type {DataExtensionFieldMap} */
+    /** @type {Object.<string, DataExtensionFieldMap>} key: deKey, value: deFieldMap */
     static oldFields;
     /**
      * Upserts dataExtensions after retrieving them from source and target to compare
@@ -100,16 +100,26 @@ class DataExtension extends MetadataType {
             .filter((r) => r !== undefined && r !== null)
             .filter(this.#filterUpsertResults);
 
+        if (Util.OPTIONS.noUpdate && metadataToUpdate.length > 0) {
+            Util.logger.info(
+                ` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
+            );
+        }
+
         const updateLimit = pLimit(10);
-        const updateResults = (
-            await Promise.allSettled(
-                metadataToUpdate
-                    .filter((r) => r !== undefined && r !== null)
-                    .map((metadataEntry) => updateLimit(() => this.update(metadataEntry.after)))
-            )
-        )
-            .filter((r) => r !== undefined && r !== null)
-            .filter(this.#filterUpsertResults);
+        const updateResults = Util.OPTIONS.noUpdate
+            ? []
+            : (
+                  await Promise.allSettled(
+                      metadataToUpdate
+                          .filter((r) => r !== undefined && r !== null)
+                          .map((metadataEntry) =>
+                              updateLimit(() => this.update(metadataEntry.after))
+                          )
+                  )
+              )
+                  .filter((r) => r !== undefined && r !== null)
+                  .filter(this.#filterUpsertResults);
 
         const successfulResults = [...createResults, ...updateResults];
         Util.logger.info(
@@ -119,8 +129,8 @@ class DataExtension extends MetadataType {
         let upsertResults;
         if (successfulResults.length > 0) {
             const metadataResults = successfulResults
-                // @ts-expect-error not sure why this produces a type error. looks 100% correct
-                .map((r) => r.value.Results[0].Object)
+                .map((r) => (r.status === 'fulfilled' ? r.value.Results[0].Object : null))
+                .filter(Boolean)
                 .map((r) => {
                     // if only one fields added will return object otherwise array
                     if (Array.isArray(r?.Fields?.Field)) {
@@ -172,7 +182,6 @@ class DataExtension extends MetadataType {
             DataExtensionField.client = this.client;
             DataExtensionField.properties = this.properties;
             DataExtension.oldFields ||= {};
-            // @ts-expect-error not sure why this produces a type error. looks 100% correct
             DataExtension.oldFields[metadataMap[metadataKey][this.definition.keyField]] =
                 await DataExtensionField.prepareDeployColumnsOnUpdate(
                     metadataMap[metadataKey].Fields,
@@ -278,10 +287,11 @@ class DataExtension extends MetadataType {
      * Updates a single dataExtension. Also updates their columns in 'dataExtension.columns'
      *
      * @param {DataExtensionItem} metadata single metadata entry
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {Promise} Promise
      */
-    static async update(metadata) {
-        return super.updateSOAP(metadata);
+    static async update(metadata, handleOutside) {
+        return super.updateSOAP(metadata, handleOutside);
     }
     /**
      * Gets executed after deployment of metadata type
@@ -485,24 +495,19 @@ class DataExtension extends MetadataType {
             value: name,
             checked: fixBuPreselected.includes(name),
         }));
-        const questions = [
-            {
-                type: 'checkbox',
-                name: 'businessUnits',
+        let answer = null;
+
+        try {
+            answer = await checkbox({
                 message:
                     'Please select BUs that have access to the updated Shared Data Extensions:',
                 pageSize: 10,
                 choices: buList,
-            },
-        ];
-        let responses = null;
-
-        try {
-            responses = await inquirer.prompt(questions);
+            });
         } catch (ex) {
             Util.logger.info(ex);
         }
-        return responses.businessUnits;
+        return answer;
     }
 
     /**
@@ -678,7 +683,6 @@ class DataExtension extends MetadataType {
                 ],
             },
         };
-        // @ts-expect-error SFMC-SDK typing error
         await this.client.soap.update(Util.capitalizeFirstLetter(soapType), payload, null);
         return randomSuffix;
     }
@@ -821,9 +825,10 @@ class DataExtension extends MetadataType {
         // get shared folders to match our shared / synched Data Extensions
         const subTypeArr = this.definition.dependencies
             .filter((item) => item.startsWith('folder-'))
-            .map((item) => item.slice(7));
+            .map((item) => item.slice(7))
+            .filter((item) => Folder.definition.folderTypesFromParent.includes(item));
         Util.logger.info(' - Caching dependent Metadata: folder (shared via _ParentBU_)');
-        Util.logSubtypes(subTypeArr);
+        Util.logSubtypes(subTypeArr, '  ');
         Folder.client = this.client;
         Folder.buObject = this.buObject;
         Folder.properties = this.properties;
@@ -1325,7 +1330,7 @@ class DataExtension extends MetadataType {
                         newJsonElement[field] = element[field];
                     }
                 }
-                // @ts-ignore hard to tell ts that this is in fact a DataExtensionFieldItem
+                // @ts-ignore for document the fields have less values and that leads to an error here.
                 newJson.push(newJsonElement);
             }
             fieldsJson = newJson;

package/lib/metadataTypes/DataExtensionField.js

@@ -97,7 +97,7 @@ class DataExtensionField extends MetadataType {
      *
      * @param {DataExtensionFieldItem[]} deployColumns Columns of data extension that will be deployed
      * @param {string} deKey external/customer key of Data Extension
-     * @returns {Promise.<Object.<string, DataExtensionFieldItem>>} existing fields by their original name to allow re-adding FieldType after update
+     * @returns {Promise.<DataExtensionFieldMap>} existing fields by their original name to allow re-adding FieldType after update
      */
     static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
         // create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic
@@ -132,7 +132,7 @@ class DataExtensionField extends MetadataType {
         const fieldsObj = response.metadata;
 
         // ensure fields can be updated properly by their adding ObjectId based on Name-matching
-        /** @type {Object.<string, DataExtensionFieldItem>} */
+        /** @type {DataExtensionFieldMap} */
         const existingFieldByName = {};
 
         for (const key of Object.keys(fieldsObj)) {
@@ -315,7 +315,6 @@ class DataExtensionField extends MetadataType {
         };
         try {
             // ! we really do need to delete from DataExtension not DataExtensionField here!
-            // @ts-expect-error wrong jsdoc for 2nd parameter in SFMC-SDK
             this.client.soap.delete('DataExtension', keyObj, null);
 
             if (!fieldId) {

package/lib/metadataTypes/Event.js

@@ -4,6 +4,7 @@ import MetadataType from './MetadataType.js';
 import { Util } from '../util/util.js';
 import File from '../util/file.js';
 import cache from '../util/cache.js';
+import deepEqual from 'deep-equal';
 
 /**
  * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
@@ -24,6 +25,7 @@ import cache from '../util/cache.js';
  * @augments MetadataType
  */
 class Event extends MetadataType {
+    static reCacheDataExtensions = [];
     /**
      * Retrieves Metadata of Event Definition.
      * Endpoint /interaction/v1/eventDefinitions return all Event Definitions with all details.
@@ -92,30 +94,26 @@ class Event extends MetadataType {
                 );
             } else if (event?.length === 1) {
                 const originalKey = event[0][this.definition.keyField];
-                const eventDef = JSON.parse(
-                    Util.replaceByObject(
-                        JSON.stringify(this.postRetrieveTasks(event[0])),
-                        templateVariables
-                    )
+                const metadataItemTemplated = Util.replaceByObject(
+                    this.postRetrieveTasks(event[0]),
+                    templateVariables
                 );
-                if (!eventDef.dataExtensionId) {
+
+                if (!metadataItemTemplated.r__dataExtension_key) {
                     throw new Error(
-                        `Event.postRetrieveTasks:: ` +
-                            `No Data Extension found for ` +
-                            `event: ${eventDef.name}. ` +
-                            `This cannot be templated`
+                        `Event.postRetrieveTasks:: No Data Extension found for ${this.definition.type}: ${metadataItemTemplated.name}. This cannot be templated.`
                     );
                 }
 
                 // remove all fields listed in Definition for templating
-                this.keepTemplateFields(eventDef);
+                this.keepTemplateFields(metadataItemTemplated);
                 await File.writeJSONToFile(
                     [templateDir, this.definition.type].join('/'),
                     originalKey + '.' + this.definition.type + '-meta',
-                    JSON.parse(Util.replaceByObject(JSON.stringify(eventDef), templateVariables))
+                    metadataItemTemplated
                 );
-                Util.logger.info(`- templated ${this.definition.type}: ${name}`);
-                return { metadata: eventDef, type: this.definition.type };
+                Util.logger.info(` - templated ${this.definition.type}: ${name}`);
+                return { metadata: metadataItemTemplated, type: this.definition.type };
             } else {
                 throw new Error(
                     `Encountered unknown error when retrieveing ${
@@ -175,7 +173,8 @@ class Event extends MetadataType {
     static async update(metadataEntry) {
         return super.updateREST(
             metadataEntry,
-            '/interaction/v1/eventDefinitions/' + metadataEntry.id,
+            '/interaction/v1/eventDefinitions/key:' +
+                encodeURIComponent(metadataEntry[this.definition.keyField]),
             'put'
         );
     }
@@ -187,33 +186,216 @@ class Event extends MetadataType {
      * @returns {MetadataTypeItem} parsed version
      */
     static preDeployTasks(metadata) {
-        if (metadata.r__dataExtension_key) {
-            metadata.dataExtensionId = cache.searchForField(
-                'dataExtension',
-                metadata.r__dataExtension_key,
-                'CustomerKey',
-                'ObjectID'
-            );
-            metadata.dataExtensionName = cache.searchForField(
-                'dataExtension',
-                metadata.r__dataExtension_key,
-                'CustomerKey',
-                'Name'
-            );
-            metadata.arguments.dataExtensionId = metadata.dataExtensionId;
-            if (metadata.schema) {
-                metadata.schema.id = metadata.dataExtensionId;
-                metadata.schema.name = metadata.dataExtensionName;
-            }
-        }
+        // Note: lots has to be done in createOrUpdate based on what action is required
+        metadata.arguments ||= {};
         metadata.arguments.eventDefinitionKey = metadata.eventDefinitionKey;
 
         // standard values
         metadata.isVisibleInPicker ||= false;
-        metadata.isPlatformObject = false;
-        metadata.mode = 'Production'; // potentially needs to be set to "1" instead
+        if (metadata.isVisibleInPicker && !metadata.sourceApplicationExtensionId) {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.keyField]}: isVisibleInPicker is true but sourceApplicationExtensionId is missing. Setting isVisibleInPicker to false.`
+            );
+            metadata.isVisibleInPicker = false;
+        }
+        metadata.isPlatformObject ||= false;
+        metadata.mode ||= 'Production';
+        switch (metadata.type) {
+            case 'APIEvent': {
+                metadata.entrySourceGroupConfigUrl ||=
+                    'jb:///data/entry/api-event/entrysourcegroupconfig.json';
+                metadata.iconUrl ||= '/images/icon_journeyBuilder-event-api-blue.svg';
+                break;
+            }
+            case 'SalesforceObjectTriggerV2': {
+                metadata.iconUrl ||= '/jbint-events/events/SalesforceData/images/SF-Event-Icon.svg';
+                break;
+            }
+        }
+
         return metadata;
     }
+    /**
+     * helper for {@link MetadataType.upsert}
+     *
+     * @param {MetadataTypeMap} metadataMap list of metadata
+     * @param {string} metadataKey key of item we are looking at
+     * @param {boolean} hasError error flag from previous code
+     * @param {MetadataTypeItemDiff[]} metadataToUpdate list of items to update
+     * @param {MetadataTypeItem[]} metadataToCreate list of items to create
+     * @returns {Promise.<'create'|'update'|'skip'>} action to take
+     */
+    static async createOrUpdate(
+        metadataMap,
+        metadataKey,
+        hasError,
+        metadataToUpdate,
+        metadataToCreate
+    ) {
+        const createOrUpdateAction = await super.createOrUpdate(
+            metadataMap,
+            metadataKey,
+            hasError,
+            metadataToUpdate,
+            metadataToCreate
+        );
+        const metadataItem = metadataMap[metadataKey];
+        if (createOrUpdateAction === 'update') {
+            if (metadataItem.r__dataExtension_key) {
+                metadataItem.dataExtensionId = cache.searchForField(
+                    'dataExtension',
+                    metadataItem.r__dataExtension_key,
+                    'CustomerKey',
+                    'ObjectID'
+                );
+                metadataItem.dataExtensionName = cache.searchForField(
+                    'dataExtension',
+                    metadataItem.r__dataExtension_key,
+                    'CustomerKey',
+                    'Name'
+                );
+                metadataItem.arguments.dataExtensionId = metadataItem.dataExtensionId;
+                if (metadataItem.schema) {
+                    metadataItem.schema.id = metadataItem.dataExtensionId;
+                    metadataItem.schema.name = metadataItem.dataExtensionName;
+                }
+            }
+            if (metadataItem.schema?.fields?.length) {
+                const normalizedKey = File.reverseFilterIllegalFilenames(
+                    metadataMap[metadataKey][this.definition.keyField]
+                );
+                const cachedVersion = cache.getByKey(this.definition.type, normalizedKey);
+                if (cachedVersion?.schema?.fields?.length) {
+                    const cacheClone = structuredClone(cachedVersion);
+                    cacheClone.schema.fields = cacheClone.schema.fields.map((field) => {
+                        delete field.isDevicePreference;
+                        return field;
+                    });
+                    if (!deepEqual(metadataItem?.schema?.fields, cacheClone?.schema?.fields)) {
+                        Util.logger.warn(
+                            ` - ${this.definition.type} ${metadataItem[this.definition.keyField]}: schema fields differ from server version. Resetting as this will not be reflected on dataExtension.`
+                        );
+                        metadataItem.schema.fields = cacheClone.schema.fields;
+                    }
+                }
+            }
+        } else if (createOrUpdateAction === 'create') {
+            try {
+                if (metadataItem.r__dataExtension_key) {
+                    metadataItem.dataExtensionId = cache.searchForField(
+                        'dataExtension',
+                        metadataItem.r__dataExtension_key,
+                        'CustomerKey',
+                        'ObjectID'
+                    );
+                    metadataItem.dataExtensionName = cache.searchForField(
+                        'dataExtension',
+                        metadataItem.r__dataExtension_key,
+                        'CustomerKey',
+                        'Name'
+                    );
+                    if (metadataItem.schema) {
+                        delete metadataItem.schema;
+                        Util.logger.info(
+                            ` - ${this.definition.type} ${metadataItem[this.definition.keyField]}: dataExtension ${metadataItem.r__dataExtension_key} found, ignoring schema-section in ${this.definition.type} json`
+                        );
+                    }
+                }
+            } catch {
+                // no action
+            }
+            if (metadataItem.schema) {
+                if (metadataItem.r__dataExtension_key) {
+                    metadataItem.schema.name = metadataItem.r__dataExtension_key;
+                }
+                Util.logger.warn(
+                    `Data Extension ${metadataItem.schema.name || metadataItem[this.definition.keyField]} not found on BU. Creating it automatically based on schema-definition.`
+                );
+                // we want the event api to create the DE for us based on the schema
+                this.reCacheDataExtensions.push({
+                    eventKey: metadataItem[this.definition.keyField],
+                    deKey: metadataItem.schema.name || metadataItem[this.definition.keyField],
+                });
+            }
+        }
+        return createOrUpdateAction;
+    }
+    /**
+     * Gets executed after deployment of metadata type
+     *
+     * @param {MetadataTypeMap} upsertResults metadata mapped by their keyField as returned by update/create
+     * @param {MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
+     * @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
+     * @returns {Promise.<void>} -
+     */
+    static async postDeployTasks(upsertResults, originalMetadata, createdUpdated) {
+        // CREATE ONLY - if dataExtensions were auto-
+        if (this.reCacheDataExtensions.length && createdUpdated.created > 0) {
+            Util.logger.warn(' - Re-caching dependent Metadata: dataExtension');
+            const deRetrieve = await DataExtension.retrieveForCache();
+            cache.setMetadata('dataExtension', deRetrieve.metadata);
+            const reDownloadDeKeys = [];
+            // try to update key & name of the auto-generated dataExtension
+            for (const { eventKey, deKey } of this.reCacheDataExtensions) {
+                if (!upsertResults[eventKey]) {
+                    continue;
+                }
+                const eventItem = upsertResults[eventKey];
+                const newDeKey = cache.searchForField(
+                    'dataExtension',
+                    eventItem.dataExtensionId,
+                    'ObjectID',
+                    'CustomerKey'
+                );
+                // get dataExtension from cache which conveniently already has the ObjectID set
+                const deObj = cache.getByKey('dataExtension', newDeKey);
+                const oldName = deObj[DataExtension.definition.nameField];
+                // prepare a clone of the DE to update name & key to match the event
+                const clone = structuredClone(deObj);
+                clone[DataExtension.definition.keyField] = deKey;
+                clone[DataExtension.definition.nameField] = deKey;
+                try {
+                    // update DE on server
+                    await DataExtension.update(clone, true);
+                    Util.logger.info(
+                        ` - changed dataExtension ${newDeKey} (${oldName}) key/name to ${deKey}`
+                    );
+                    // update cache
+                    deObj[DataExtension.definition.keyField] = deKey;
+                    deObj[DataExtension.definition.nameField] = deKey;
+
+                    reDownloadDeKeys.push(deObj[DataExtension.definition.keyField]);
+                } catch {
+                    // fallback, set DE key to value of DE name
+                    const clone = structuredClone(deObj);
+                    clone[DataExtension.definition.keyField] = oldName;
+                    try {
+                        // update DE on server
+                        await DataExtension.update(clone, true);
+                        Util.logger.info(
+                            ` - changed dataExtension ${newDeKey} (${oldName}) key to ${oldName}`
+                        );
+                        // update cache
+                        deObj[DataExtension.definition.keyField] =
+                            deObj[DataExtension.definition.nameField];
+
+                        reDownloadDeKeys.push(deObj[DataExtension.definition.keyField]);
+                    } catch {
+                        Util.logger.debug(
+                            ` - failed to change dataExtension ${newDeKey} (${oldName}) key/name`
+                        );
+                    }
+                }
+            }
+            this.reCacheDataExtensions.length = 0;
+
+            // ensure we have downloaded auto-created DEs
+            if (reDownloadDeKeys.length) {
+                const retriever = new Retriever(this.properties, this.buObject);
+                await retriever.retrieve(['dataExtension'], reDownloadDeKeys);
+            }
+        }
+    }
 
     /**
      * parses retrieved Metadata before saving
@@ -234,7 +416,6 @@ class Event extends MetadataType {
             delete metadata.arguments.dataExtensionId;
             if (metadata.schema) {
                 delete metadata.schema.id;
-                delete metadata.schema.name;
             }
         } catch (ex) {
             Util.logger.verbose(
@@ -243,12 +424,21 @@ class Event extends MetadataType {
                 }): ${ex.message}.`
             );
         }
+        if (!metadata.isPlatformObject) {
+            delete metadata.isPlatformObject;
+        }
+        if (metadata.mode === 'Production') {
+            delete metadata.mode;
+        }
+
         return metadata;
     }
 }
 
 // Assign definition to static attributes
 import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
+import DataExtension from './DataExtension.js';
+import Retriever from './../Retriever.js';
 Event.definition = MetadataTypeDefinitions.event;
 
 export default Event;