mcdev 3.1.1 → 4.0.0

package/lib/metadataTypes/Folder.js

@@ -1,25 +1,33 @@
 'use strict';
 
+const TYPE = require('../../types/mcdev.d');
 const MetadataType = require('./MetadataType');
 const toposort = require('toposort');
 const Util = require('../util/util');
 const File = require('../util/file');
+const cache = require('../util/cache');
 
 /**
  * Folder MetadataType
+ *
  * @augments MetadataType
  */
 class Folder extends MetadataType {
     /**
      * Retrieves metadata of metadata type into local filesystem. executes callback with retrieved metadata
-     * @param {String} retrieveDir Directory where retrieved metadata directory will be saved
-     * @param {String[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Object} buObject properties for auth
+     *
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @param {TYPE.BuObject} buObject properties for auth
+     * @param {void} [___] unused parameter
+     * @param {string} [key] customer key of single item to retrieve
      * @returns {Promise} Promise
      */
-    static async retrieve(retrieveDir, additionalFields, buObject) {
+    static async retrieve(retrieveDir, additionalFields, buObject, ___, key) {
+        if (key) {
+            Util.logger.error(`Folder.retrieve() does not support key parameter`);
+        }
         const queryAllFolders = await this.retrieveHelper(additionalFields, true);
-        // if this is the parent, no need to query twice as QueryAllAccounts works.
 
         if (buObject.eid !== buObject.mid) {
             queryAllFolders.push(...(await this.retrieveHelper(additionalFields, false)));
@@ -36,6 +44,8 @@ class Folder extends MetadataType {
             else if (!val.CustomerKey) {
                 val.CustomerKey = [buObject.eid, val.ID].join('-');
             }
+            // ensure name is a string and not a number (SFMC-SDK workaround)
+            val.Name = val.Name + '';
 
             idMap[val.ID] = val;
         }
@@ -47,12 +57,9 @@ class Folder extends MetadataType {
         for (const id of sortPairs) {
             if (!idMap[id]) {
                 Util.logger.debug(`Error: id ${id} not found in idMap-obj but in sortPairs-array.`);
-                if (Util.logger.level === 'debug') {
-                    console.log(`Error: id ${id} not found in idMap-ob but in sortPairs-array.`);
-                }
             } else if (
                 idMap[id].ParentFolder &&
-                idMap[id].ParentFolder.ID &&
+                Number.isSafeInteger(idMap[id].ParentFolder.ID) &&
                 idMap[id].Name !== '<ROOT>'
             ) {
                 // if the parent folder can be found by ID
@@ -69,16 +76,16 @@ class Folder extends MetadataType {
                         idMap[id].Path = idMap[id].Name;
                     }
                 } else {
-                    Util.logger.error(
-                        `Skipping - Cannot find parent folder (${idMap[id].ParentFolder.ID}) of: ${idMap[id].Name} (${id}, type: ${idMap[id].ContentType})`
+                    Util.logger.warn(
+                        ` - Skipping folder ${idMap[id].Name} (${id}, type: ${idMap[id].ContentType}): Cannot find parent folder (${idMap[id].ParentFolder.ID})`
                     );
                 }
             }
             // All folders except the artificial root have ParentFolder attribute. If they dont something else is wrong
             else if (idMap[id].Name !== '<ROOT>') {
                 idMap[id].Path = '';
-                Util.logger.error(
-                    `Skipping - Folder ${idMap[id].Name} does not have a parent folder (type: ${idMap[id].ContentType})`
+                Util.logger.warn(
+                    ` - Skipping folder ${idMap[id].Name} (${id}, type: ${idMap[id].ContentType}): Does not have a parent folder`
                 );
             }
         }
@@ -97,8 +104,7 @@ class Folder extends MetadataType {
         for (const id in idMap) {
             // remove keys which are listed in other BUs and skip root
             if (
-                idMap[id].Client &&
-                idMap[id].Client.ID &&
+                idMap[id].Client?.ID &&
                 (buObject.mid == idMap[id].Client.ID ||
                     this.definition.folderTypesFromParent.includes(
                         idMap[id].ContentType.toLowerCase()
@@ -142,7 +148,8 @@ class Folder extends MetadataType {
 
     /**
      * Retrieves folder metadata for caching
-     * @param {Object} buObject properties for auth
+     *
+     * @param {TYPE.BuObject} buObject properties for auth
      * @returns {Promise} Promise
      */
     static retrieveForCache(buObject) {
@@ -153,12 +160,14 @@ class Folder extends MetadataType {
      * Folder upsert (copied from Metadata Upsert), after retrieving from target
      * and comparing to check if create or update operation is needed.
      * Copied due to having a dependency on itself, meaning the created need to be serial
-     * @param {Object} metadata metadata mapped by their keyField
-     * @returns {Promise<Object>} Promise of saved metadata
+     *
+     * @param {TYPE.MetadataTypeMap} metadata metadata mapped by their keyField
+     * @returns {Promise.<object>} Promise of saved metadata
      */
     static async upsert(metadata) {
         let updateCount = 0;
         let createCount = 0;
+        let filteredByPreDeploy = 0;
         const upsertResults = {};
         const sortPairs = toposort(
             Object.keys(metadata).map((customerKey) => [
@@ -180,18 +189,61 @@ class Folder extends MetadataType {
                             let existingId;
                             try {
                                 // perform a secondary check based on path
-                                existingId = Util.getFromCache(
-                                    this.cache,
+                                existingId = cache.searchForField(
                                     'folder',
                                     deployableMetadata.Path,
                                     'Path',
                                     'ID'
                                 );
-                            } catch (ex) {
+                                const cachedVersion = cache.getByKey(
+                                    'folder',
+                                    cache.searchForField(
+                                        'folder',
+                                        deployableMetadata.Path,
+                                        'Path',
+                                        this.definition.keyField
+                                    )
+                                );
+                                if (
+                                    existingId &&
+                                    !this.hasChangedGeneric(
+                                        cachedVersion,
+                                        metadata[metadataKey],
+                                        null,
+                                        true
+                                    )
+                                ) {
+                                    Util.logger.verbose(
+                                        ` - skipping ${this.definition.type} ${
+                                            cachedVersion?.Path ||
+                                            metadata[metadataKey][this.definition.nameField]
+                                        }: no change detected`
+                                    );
+                                    filteredByPreDeploy++;
+                                    continue;
+                                }
+                            } catch {
                                 // In case no path matching, then try to use CustomerKey
-                                existingId = this.cache.folder[normalizedKey]
-                                    ? this.cache.folder[normalizedKey].ID
-                                    : null;
+                                const cachedVersion = cache.getByKey('folder', normalizedKey);
+                                existingId = cachedVersion?.ID;
+                                if (
+                                    existingId &&
+                                    !this.hasChangedGeneric(
+                                        cachedVersion,
+                                        metadata[metadataKey],
+                                        null,
+                                        true
+                                    )
+                                ) {
+                                    Util.logger.verbose(
+                                        ` - skipping ${this.definition.type} ${
+                                            cachedVersion?.Path ||
+                                            metadata[metadataKey][this.definition.nameField]
+                                        }: no change detected`
+                                    );
+                                    filteredByPreDeploy++;
+                                    continue;
+                                }
                             }
 
                             let result;
@@ -206,48 +258,48 @@ class Folder extends MetadataType {
                                 result = await this.create(deployableMetadata);
                                 createCount++;
                             }
-                            if (result && result.body && result.body.Results) {
+                            if (result?.Results) {
                                 const parsed = this.parseResponseBody({
-                                    Results: [result.body.Results[0].Object],
+                                    Results: [result.Results[0].Object],
                                 });
-                                if (
-                                    !result.body.Results[0].Object.CustomerKey &&
-                                    parsed['undefined']
-                                ) {
+                                if (!result.Results[0].Object.CustomerKey && parsed['undefined']) {
                                     // when inserting folders without specifying a CustomerKey, this happens in parseResponseBody()
                                     parsed[normalizedKey] = parsed['undefined'];
                                     delete parsed['undefined'];
                                 }
-                                this.cache.folder[normalizedKey] = Object.assign(
+                                const newObject = {};
+                                newObject[normalizedKey] = Object.assign(
                                     beforeMetadata,
                                     parsed[normalizedKey]
                                 );
+                                cache.mergeMetadata('folder', newObject);
 
                                 upsertResults[metadataKey] = beforeMetadata;
                             } else {
                                 Util.logger.debug(result);
                                 throw new Error(
-                                    `'${beforeMetadata[metadataKey].Path}' was not deployed correctly`
+                                    `'${beforeMetadata.Path}' was not deployed correctly`
                                 );
                             }
                         }
                     } catch (ex) {
-                        Util.metadataLogger('error', 'folder', 'upsert', ex, metadataKey);
+                        Util.logger.errorStack(
+                            ex,
+                            `Upserting ${this.definition.type} failed: ${ex.message}`
+                        );
                     }
                 }
             }
         }
         // Logging
-        Util.metadataLogger(
-            'info',
-            this.definition.type,
-            'upsert',
-            `${createCount} created / ${updateCount} updated`
+        Util.logger.info(
+            `${this.definition.type} upsert: ${createCount} created / ${updateCount} updated` +
+                (filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
         );
 
         if (updateCount) {
             Util.logger.warn(
-                `Folders are recognized for updates based on their CustomerKey or, if that is not given, their folder-path.`
+                ` - Folders are recognized for updates based on their CustomerKey or, if that is not given, their folder-path.`
             );
         }
         return upsertResults;
@@ -255,36 +307,37 @@ class Folder extends MetadataType {
 
     /**
      * creates a folder based on metatadata
-     * @param {Object} metadata metadata of the folder
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry metadata of the folder
      * @returns {Promise} Promise
      */
-    static async create(metadata) {
-        if (metadata.Parent && metadata.Parent.ID && metadata.Parent.ID === 0) {
+    static async create(metadataEntry) {
+        if (metadataEntry?.Parent?.ID === 0) {
             Util.logger.error(
-                `${this.definition.type}-${metadata.ContentType}.create:: Cannot create Root Folder: ${metadata.Name}`
+                `${this.definition.type}-${metadataEntry.ContentType}.create:: Cannot create Root Folder: ${metadataEntry.Name}`
             );
             return {};
         }
-        const path = metadata.Path;
+        const path = metadataEntry.Path;
         try {
-            const response = await super.createSOAP(metadata, 'DataFolder', true);
+            const response = await super.createSOAP(metadataEntry, 'DataFolder', true);
             if (response) {
-                response.body.Results[0].Object = metadata;
-                response.body.Results[0].Object.ID = response.body.Results[0].NewID;
-                response.body.Results[0].Object.CustomerKey = metadata.CustomerKey;
-                delete response.body.Results[0].Object.$;
+                response.Results[0].Object = metadataEntry;
+                response.Results[0].Object.ID = response.Results[0].NewID;
+                response.Results[0].Object.CustomerKey = metadataEntry.CustomerKey;
+                delete response.Results[0].Object.$;
 
-                Util.logger.info(`- created folder: ${path}`);
+                Util.logger.info(` - created folder: ${path}`);
                 return response;
             }
         } catch (ex) {
-            if (ex && ex.results) {
+            if (ex?.results) {
                 Util.logger.error(
-                    `${this.definition.type}-${metadata.ContentType}.create:: error creating: '${path}'. ${ex.results[0].StatusMessage}`
+                    `${this.definition.type}-${metadataEntry.ContentType}.create:: error creating: '${path}'. ${ex.results[0].StatusMessage}`
                 );
             } else if (ex) {
                 Util.logger.error(
-                    `${this.definition.type}-${metadata.ContentType}.create:: error creating: '${path}'. ${ex.message}`
+                    `${this.definition.type}-${metadataEntry.ContentType}.create:: error creating: '${path}'. ${ex.message}`
                 );
             }
         }
@@ -292,28 +345,29 @@ class Folder extends MetadataType {
 
     /**
      * Updates a single Folder.
-     * @param {Object} metadata single metadata entry
+     *
+     * @param {TYPE.MetadataTypeItem} metadataEntry single metadata entry
      * @returns {Promise} Promise
      */
-    static async update(metadata) {
-        const path = metadata.Path;
+    static async update(metadataEntry) {
+        const path = metadataEntry.Path;
         try {
-            const response = await super.updateSOAP(metadata, 'DataFolder', true);
+            const response = await super.updateSOAP(metadataEntry, 'DataFolder', true);
             if (response) {
-                response.body.Results[0].Object = metadata;
-                response.body.Results[0].Object.CustomerKey = metadata.CustomerKey;
-                delete response.body.Results[0].Object.$;
-                Util.logger.info(`- updated folder: ${path}`);
+                response.Results[0].Object = metadataEntry;
+                response.Results[0].Object.CustomerKey = metadataEntry.CustomerKey;
+                delete response.Results[0].Object.$;
+                Util.logger.info(` - updated folder: ${path}`);
                 return response;
             }
         } catch (ex) {
-            if (ex && ex.results) {
+            if (ex?.results) {
                 Util.logger.error(
-                    `${this.definition.type}-${metadata.ContentType}.update:: error updating: '${path}'. ${ex.results[0].StatusMessage}`
+                    `${this.definition.type}-${metadataEntry.ContentType}.update:: error updating: '${path}'. ${ex.results[0].StatusMessage}`
                 );
             } else if (ex) {
                 Util.logger.error(
-                    `${this.definition.type}-${metadata.ContentType}.update:: error updating: '${path}'. ${ex.message}`
+                    `${this.definition.type}-${metadataEntry.ContentType}.update:: error updating: '${path}'. ${ex.message}`
                 );
             }
         }
@@ -321,13 +375,14 @@ class Folder extends MetadataType {
 
     /**
      * prepares a folder for deployment
-     * @param {Object} metadata a single folder definition
-     * @returns {Promise} Promise
+     *
+     * @param {TYPE.MetadataTypeItem} metadata a single folder definition
+     * @returns {Promise.<TYPE.MetadataTypeItem>} Promise of parsed folder metadata
      */
     static async preDeployTasks(metadata) {
         if (!this.definition.deployFolderTypes.includes(metadata.ContentType.toLowerCase())) {
             Util.logger.warn(
-                `Folder ${metadata.Name} is of a type which does not support deployment (Type: ${metadata.ContentType}). Please create this manually in the Web-Interface.`
+                ` - Folder ${metadata.Name} is of a type which does not support deployment (Type: ${metadata.ContentType}). Please create this manually in the Web-Interface.`
             );
             return;
         }
@@ -353,9 +408,8 @@ class Folder extends MetadataType {
             return;
         }
         // retreive ID based on the matching Path of the parent folder
-        else if (metadata.ParentFolder && metadata.ParentFolder.Path) {
-            metadata.ParentFolder.ID = Util.getFromCache(
-                this.cache,
+        else if (metadata?.ParentFolder?.Path) {
+            metadata.ParentFolder.ID = cache.searchForField(
                 'folder',
                 metadata.ParentFolder.Path,
                 'Path',
@@ -375,9 +429,10 @@ class Folder extends MetadataType {
 
     /**
      * Returns file contents mapped to their filename without '.json' ending
-     * @param {String} dir directory that contains '.json' files to be read
+     *
+     * @param {string} dir directory that contains '.json' files to be read
      * @param {boolean} [listBadKeys=false] do not print errors, used for badKeys()
-     * @returns {Object} fileName => fileContent map
+     * @returns {TYPE.MetadataTypeMap} fileName => fileContent map
      */
     static getJsonFromFS(dir, listBadKeys) {
         try {
@@ -407,9 +462,7 @@ class Folder extends MetadataType {
                                 fileContent.ParentFolder = {
                                     Path: standardSubDir,
                                 };
-                                const key = fileContent.CustomerKey
-                                    ? fileContent.CustomerKey
-                                    : `new-${++newCounter}`;
+                                const key = fileContent.CustomerKey || `new-${++newCounter}`;
                                 if (fileName2FileContent[key]) {
                                     Util.logger.error(
                                         `Your have multiple folder-JSONs with the same CustomerKey '${key}' - skipping ${fileName}`
@@ -445,58 +498,38 @@ class Folder extends MetadataType {
 
     /**
      * Helper to retrieve the folders as promise
-     * @param {String[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
-     * @param {Boolean} [queryAllAccounts] which queryAllAccounts setting to use
-     * @returns {Promise<Object>} soap object
+     *
+     * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
+     * @param {boolean} [queryAllAccounts] which queryAllAccounts setting to use
+     * @returns {Promise.<object>} soap object
      */
     static async retrieveHelper(additionalFields, queryAllAccounts) {
-        const options = { queryAllAccounts: !!queryAllAccounts };
-        let status = 'MoreDataAvailable';
-        const Results = [];
-        do {
-            let response;
-            await Util.retryOnError(`Retrying ${this.definition.type}`, async () => {
-                response = await new Promise((resolve, reject) => {
-                    // filtered out path as we need them to be stored locally, but do not want to try and retrieve
-                    this.client.SoapClient.retrieve(
-                        'DataFolder',
-                        this.getFieldNamesToRetrieve(additionalFields).filter(
-                            (field) => !field.includes('Path')
-                        ),
-                        options,
-                        (ex, response) => (ex ? reject(ex) : resolve(response))
-                    );
-                });
-            });
-            // merge results with existing
-            Results.push(...response.body.Results);
-            // set status for calling again if required
-            status = response.body.OverallStatus;
-            options.continueRequest = response.body.RequestID;
-            if (status === 'MoreDataAvailable') {
-                Util.logger.info(
-                    '- more than ' +
-                        Results.length +
-                        ' folders in Business Unit - loading next batch...'
-                );
-            }
-        } while (status === 'MoreDataAvailable');
-        return Results;
+        const options = { QueryAllAccounts: !!queryAllAccounts };
+        const response = await this.client.soap.retrieveBulk(
+            'DataFolder',
+            this.getFieldNamesToRetrieve(additionalFields).filter(
+                (field) => !field.includes('Path')
+            ),
+            options
+        );
+        return response.Results;
     }
     /**
      * Gets executed after retreive of metadata type
-     * @param {Object} metadata metadata mapped by their keyField
-     * @returns {Object[]} cloned metadata
+     *
+     * @param {TYPE.MetadataTypeItem} metadata metadata mapped by their keyField
+     * @returns {TYPE.MetadataTypeItem} cloned metadata
      */
     static postRetrieveTasks(metadata) {
         return JSON.parse(JSON.stringify(metadata));
     }
     /**
      * Helper for writing Metadata to disk, used for Retrieve and deploy
-     * @param {Object} results metadata results from deploy
-     * @param {String} retrieveDir directory where metadata should be stored after deploy/retrieve
-     * @param {Number} mid current mid for this credential / business unit
-     * @returns {Promise<Object>} Promise of saved metadata
+     *
+     * @param {object} results metadata results from deploy
+     * @param {string} retrieveDir directory where metadata should be stored after deploy/retrieve
+     * @param {number} mid current mid for this credential / business unit
+     * @returns {Promise.<object>} Promise of saved metadata
      */
     static async saveResults(results, retrieveDir, mid) {
         const savedResults = {};
@@ -556,6 +589,5 @@ class Folder extends MetadataType {
 
 // Assign definition to static attributes
 Folder.definition = require('../MetadataTypeDefinitions').folder;
-Folder.client = undefined;
 
 module.exports = Folder;

package/lib/metadataTypes/FtpLocation.js

@@ -1,34 +1,40 @@
 'use strict';
 
+const TYPE = require('../../types/mcdev.d');
 const MetadataType = require('./MetadataType');
 
 /**
  * ImportFile MetadataType
+ *
  * @augments MetadataType
  */
 class FtpLocation extends MetadataType {
     /**
      * Retrieves Metadata of FtpLocation
      * Endpoint /automation/v1/ftplocations/ return all FtpLocations
-     * @param {String} retrieveDir Directory where retrieved metadata directory will be saved
-     * @returns {Promise} Promise
+     *
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {void} [_] unused parameter
+     * @param {void} [__] unused parameter
+     * @param {void} [___] unused parameter
+     * @param {string} [key] customer key of single item to retrieve
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise
      */
-    static retrieve(retrieveDir) {
-        return super.retrieveREST(retrieveDir, '/automation/v1/ftplocations/', null);
+    static retrieve(retrieveDir, _, __, ___, key) {
+        return super.retrieveREST(retrieveDir, '/automation/v1/ftplocations/', null, null, key);
     }
 
     /**
      * Retrieves folder metadata into local filesystem. Also creates a uniquePath attribute for each folder.
-     * @returns {Promise} Promise
+     *
+     * @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise
      */
     static async retrieveForCache() {
-        return super.retrieveREST(null, '/automation/v1/ftplocations/', null);
+        return super.retrieveREST(null, '/automation/v1/ftplocations/');
     }
 }
 
 // Assign definition to static attributes
 FtpLocation.definition = require('../MetadataTypeDefinitions').ftpLocation;
-FtpLocation.cache = {};
-FtpLocation.client = undefined;
 
 module.exports = FtpLocation;