mcdev 6.0.2 → 7.0.0

package/lib/util/replaceContentBlockReference.js

@@ -0,0 +1,278 @@
+'use strict';
+
+import auth from './auth.js';
+import cache from './cache.js';
+import { Util } from '../util/util.js';
+
+import Folder from '../metadataTypes/Folder.js';
+import Asset from '../metadataTypes/Asset.js';
+
+/**
+ * @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject
+ * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
+ * @typedef {import('../../types/mcdev.d.js').Cache} Cache
+ * @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
+ * @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
+ * @typedef {import('../../types/mcdev.d.js').DeltaPkgItem} DeltaPkgItem
+ * @typedef {import('../../types/mcdev.d.js').McdevLogger} McdevLogger
+ * @typedef {import('../../types/mcdev.d.js').Logger} Logger
+ * @typedef {import('../../types/mcdev.d.js').Mcdevrc} Mcdevrc
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
+ * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeList} MultiMetadataTypeList
+ * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeMap} MultiMetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').SkipInteraction} SkipInteraction
+ * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
+ * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
+ * @typedef {import('../../types/mcdev.d.js').SDKError} SDKError
+ * @typedef {import('../../types/mcdev.d.js').AssetItemSimple} AssetItemSimple
+ * @typedef {import('../../types/mcdev.d.js').AssetItemSimpleMap} AssetItemSimpleMap
+ * @typedef {import('../../types/mcdev.d.js').AssetItemIdSimpleMap} AssetItemIdSimpleMap
+ * @typedef {import('../../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
+ */
+
+/**
+ * Util that contains logger and simple util methods
+ */
+export default class ReplaceContentBlockReference {
+    /** @type {{id: AssetItemIdSimpleMap, key: AssetItemSimpleMap, name: AssetItemSimpleMap}} */
+    static assetCacheMap = {
+        id: {},
+        key: {},
+        name: {},
+    };
+    /** @type {{id: RegExp[], key: RegExp[], name: RegExp[]}} */
+    static #regexBy = {
+        // TODO: handle cases in which variables or functions are passed into ContentBlockByX
+
+        amp: {
+            id: [
+                /ContentBlockById\(\s*"([0-9]+)"\s*\)/gim,
+                /ContentBlockById\(\s*'([0-9]+)'\s*\)/gim,
+                /ContentBlockById\(\s*([0-9]+)\s*\)/gim,
+            ],
+            key: [
+                /ContentBlockByKey\(\s*"([a-z0-9-/._]+)"\s*\)/gim,
+                /ContentBlockByKey\(\s*'([a-z0-9-/._]+)'\s*\)/gim,
+            ],
+            name: [
+                /ContentBlockByName\(\s*"([ a-z0-9-\\._)(]+)"\s*\)/gim,
+                /ContentBlockByName\(\s*'([ a-z0-9-\\._)(]+)'\s*\)/gim,
+            ],
+        },
+        ssjs: {
+            id: [
+                /Platform.Function.ContentBlockById\(\s*"([0-9]+)"\s*\)/gim,
+                /Platform.Function.ContentBlockById\(\s*'([0-9]+)'\s*\)/gim,
+                /Platform.Function.ContentBlockById\(\s*([0-9]+)\s*\)/gim,
+            ],
+            key: [
+                /Platform.Function.ContentBlockByKey\(\s*"([a-z0-9-/._]+)"\s*\)/gim,
+                /Platform.Function.ContentBlockByKey\(\s*'([a-z0-9-/._]+)'\s*\)/gim,
+            ],
+            name: [
+                /Platform.Function.ContentBlockByName\(\s*"([ a-z0-9-\\._)(]+)"\s*\)/gim,
+                /Platform.Function.ContentBlockByName\(\s*'([ a-z0-9-\\._)(]+)'\s*\)/gim,
+            ],
+        },
+    };
+
+    /**
+     * used to equalize the reference in the code to whatever is set in the "to" field
+     *
+     * @param {string} str full code string
+     * @param {string} parentName name of the object that was passed in; used in error message only
+     * @param {ContentBlockConversionTypes[]} [fromList] what to replace
+     * @param {ContentBlockConversionTypes} [to] what to replace with
+     * @returns {string} replaced string
+     */
+    static replaceReference(str, parentName, fromList, to) {
+        if (!str) {
+            const ex = new Error('No string provided');
+            ex.code = 200;
+            throw ex;
+        }
+        fromList ||= Util.OPTIONS.referenceFrom;
+        to ||= Util.OPTIONS.referenceTo;
+        let result = str;
+        let changes = 0;
+        const languages = [
+            { name: 'ssjs', isSsjs: true },
+            { name: 'amp', isSsjs: false },
+        ];
+        for (const from of fromList) {
+            for (const lang of languages) {
+                for (const regex of this.#regexBy[lang.name][from]) {
+                    result = result.replaceAll(regex, (match, identifier) => {
+                        const referencedAsset = this.#getAssetBy(
+                            from,
+                            identifier,
+                            parentName,
+                            lang.isSsjs
+                        );
+                        if (referencedAsset && referencedAsset[to]) {
+                            // make sure we not only found the asset but also have a replacement for it (folder issue could block swap to ContentBlockByName)
+                            changes++;
+                            return this.#replaceWith(referencedAsset, to, lang.isSsjs);
+                        } else {
+                            if (referencedAsset && !referencedAsset[to]) {
+                                // this is expected to only happen if to=="name"
+                                Util.logger.error(
+                                    ` - ${parentName}: Asset ${from} ${identifier} has no valid ${to} reference`
+                                );
+                            }
+                            return match;
+                        }
+                    });
+                }
+            }
+        }
+        if (!changes) {
+            const ex = new Error('No changes made to the code.');
+            ex.code = 200;
+            throw ex;
+        }
+        return result;
+    }
+    /**
+     *
+     * @private
+     * @param {ContentBlockConversionTypes} from replace with
+     * @param {string|number} identifier id, key or name of asset
+     * @param {string} parentName name of the object that was passed in; used in error message only
+     * @param {boolean} [isSsjs] replaces backslashes with double backslashes in name if true
+     * @returns {AssetItemSimple} asset object
+     */
+    static #getAssetBy(from, identifier, parentName, isSsjs = false) {
+        let reference;
+        switch (from) {
+            case 'id': {
+                reference = ReplaceContentBlockReference.assetCacheMap.id[identifier];
+                break;
+            }
+            case 'key': {
+                reference = ReplaceContentBlockReference.assetCacheMap.key[identifier];
+                break;
+            }
+            case 'name': {
+                if (isSsjs) {
+                    identifier = identifier.replaceAll('\\\\', '\\');
+                }
+                reference = ReplaceContentBlockReference.assetCacheMap.name[identifier];
+                break;
+            }
+        }
+        if (!reference) {
+            Util.logger.error(` - ${parentName}: Asset not found for ${from} ${identifier}`);
+        }
+        return reference;
+    }
+
+    /**
+     *
+     * @param {AssetItemSimple} asset asset object
+     * @param {ContentBlockConversionTypes} to replace with
+     * @param {boolean} [isSsjs] replaces backslashes with double backslashes in name if true
+     * @returns {string} replaced string
+     */
+    static #replaceWith(asset, to, isSsjs = false) {
+        switch (to) {
+            case 'id': {
+                return `${isSsjs ? 'Platform.Function.' : ''}ContentBlockById(${asset.id})`;
+            }
+            case 'key': {
+                return `${isSsjs ? 'Platform.Function.' : ''}ContentBlockByKey("${asset.key}")`;
+            }
+            case 'name': {
+                return `${isSsjs ? 'Platform.Function.' : ''}ContentBlockByName("${isSsjs ? asset.name.replaceAll('\\', '\\\\') : asset.name}")`;
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {Mcdevrc} properties properties for auth
+saved
+     * @param {BuObject} buObject properties for auth
+     */
+    static async createCacheMap(properties, buObject) {
+        const client = auth.getSDK(buObject);
+
+        if (!cache.getCache()) {
+            cache.initCache(buObject);
+        }
+        Util.logger.info('Caching folders');
+        Folder.buObject = buObject;
+        Folder.properties = properties;
+        Folder.client = client;
+        const resultFolder = await Folder.retrieveForCache(null, ['asset', 'asset-shared']);
+        cache.setMetadata('folder', resultFolder.metadata);
+
+        Util.logger.info('Caching assets');
+        Asset.buObject = buObject;
+        Asset.properties = properties;
+        Asset.client = client;
+        const resultAsset = await Asset.retrieveForCache(undefined, [
+            'asset',
+            'code',
+            'textfile',
+            'block',
+            'other',
+        ]);
+        for (const element of Object.values(resultAsset.metadata)) {
+            // ensure we got the folder-path in our cache
+            Asset.setFolderPath(element);
+
+            // create actual cache map
+            const simpleAsset = {
+                id: element.id,
+                key: element.customerKey,
+                name: element.r__folder_Path
+                    ? element.r__folder_Path.replaceAll('/', '\\') + '\\' + element.name
+                    : null,
+            };
+            // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
+            ReplaceContentBlockReference.assetCacheMap.id[simpleAsset.id] = simpleAsset;
+            ReplaceContentBlockReference.assetCacheMap.key[simpleAsset.key] = simpleAsset;
+            if (simpleAsset.name) {
+                // while asset without path could still be found via search, it would no longer referencable via ContentBlockByName
+                ReplaceContentBlockReference.assetCacheMap.name[simpleAsset.name] = simpleAsset;
+            }
+        }
+
+        cache.setMetadata('asset', resultAsset.metadata);
+
+        Util.logger.info('Caching shared assets');
+        const resultAssetShared = await Asset.retrieveForCache(
+            undefined,
+            ['asset', 'code', 'textfile', 'block', 'other'],
+            true
+        );
+        for (const element of Object.values(resultAssetShared.metadata)) {
+            // ensure we got the folder-path in our cache
+            Asset.setFolderPath(element);
+
+            // create actual cache map
+            const simpleAsset = {
+                id: element.id,
+                key: element.customerKey,
+                name: element.r__folder_Path
+                    ? element.r__folder_Path.replaceAll('/', '\\') + '\\' + element.name
+                    : null,
+            };
+            // ! note that ContentBlockByName expects backslashes between folders and file name, not forward slashes
+            ReplaceContentBlockReference.assetCacheMap.id[simpleAsset.id] = simpleAsset;
+            ReplaceContentBlockReference.assetCacheMap.key[simpleAsset.key] = simpleAsset;
+            if (simpleAsset.name) {
+                // while asset without path could still be found via search, it would no longer referencable via ContentBlockByName
+                ReplaceContentBlockReference.assetCacheMap.name[simpleAsset.name] = simpleAsset;
+            }
+        }
+
+        // lets not put the shared assets into our cache to avoid confusing the system...
+    }
+}

package/lib/util/util.js

@@ -1,6 +1,5 @@
 'use strict';
 
-import TYPE from '../../types/mcdev.d.js';
 import MetadataDefinitions from './../MetadataTypeDefinitions.js';
 import process from 'node:process';
 import toposort from 'toposort';
@@ -12,17 +11,45 @@ import { readJsonSync } from 'fs-extra/esm';
 import { fileURLToPath } from 'node:url';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
+/**
+ * @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject
+ * @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
+ * @typedef {import('../../types/mcdev.d.js').Cache} Cache
+ * @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
+ * @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
+ * @typedef {import('../../types/mcdev.d.js').DeltaPkgItem} DeltaPkgItem
+ * @typedef {import('../../types/mcdev.d.js').McdevLogger} McdevLogger
+ * @typedef {import('../../types/mcdev.d.js').Logger} Logger
+ * @typedef {import('../../types/mcdev.d.js').Mcdevrc} Mcdevrc
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
+ * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeList} MultiMetadataTypeList
+ * @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeMap} MultiMetadataTypeMap
+ * @typedef {import('../../types/mcdev.d.js').SkipInteraction} SkipInteraction
+ * @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
+ * @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
+ * @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
+ * @typedef {import('../../types/mcdev.d.js').SDKError} SDKError
+ */
+
 /**
  * Util that contains logger and simple util methods
  */
 export const Util = {
+    isRunViaVSCodeExtension:
+        process.env.VSCODE_AMD_ENTRYPOINT === 'vs/workbench/api/node/extensionHostProcess' || // run via VSCode extension
+        process.env.VSCODE_CRASH_REPORTER_PROCESS_TYPE === 'extensionHost',
     authFileName: '.mcdev-auth.json',
     boilerplateDirectory: '../../boilerplate',
     configFileName: '.mcdevrc.json',
+    defaultGitBranch: 'main',
     parentBuName: '_ParentBU_',
     standardizedSplitChar: '/',
-    /** @type {TYPE.skipInteraction} */
-    skipInteraction: false,
+    /** @type {SkipInteraction} */
+    skipInteraction: null,
     packageJsonMcdev: readJsonSync(path.join(__dirname, '../../package.json')),
     OPTIONS: {},
     changedKeysMap: {},
@@ -68,7 +95,7 @@ export const Util = {
      * check if a market name exists in current mcdev config
      *
      * @param {string} market market localizations
-     * @param {TYPE.Mcdevrc} properties local mcdev config
+     * @param {Mcdevrc} properties local mcdev config
      * @returns {boolean} found market or not
      */
     checkMarket(market, properties) {
@@ -89,7 +116,7 @@ export const Util = {
      * ensure provided MarketList exists and it's content including markets and BUs checks out
      *
      * @param {string} mlName name of marketList
-     * @param {TYPE.Mcdevrc} properties General configuration to be used in retrieve
+     * @param {Mcdevrc} properties General configuration to be used in retrieve
      */
     verifyMarketList(mlName, properties) {
         if (properties.marketList[mlName]) {
@@ -135,7 +162,7 @@ export const Util = {
      * @returns {void}
      */
     signalFatalError() {
-        Util.logger.debug('Util.signalFataError() sets process.exitCode = 1 unless already set');
+        // Util.logger.debug('Util.signalFataError() sets process.exitCode = 1 unless already set');
         process.exitCode ||= 1;
     },
     /**
@@ -162,12 +189,12 @@ export const Util = {
     /**
      * helper for Mcdev.retrieve, Mcdev.retrieveAsTemplate and Mcdev.deploy
      *
-     * @param {TYPE.SupportedMetadataTypes} selectedType type or type-subtype
+     * @param {string} selectedType type or type-subtype
      * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
      * @returns {boolean} type ok or not
      */
     _isValidType(selectedType, handleOutside) {
-        const [type, subType] = Util.getTypeAndSubType(selectedType);
+        const { type, subType } = Util.getTypeAndSubType(selectedType);
         if (type && !MetadataDefinitions[type]) {
             if (!handleOutside) {
                 Util.logger.error(`:: '${type}' is not a valid metadata type`);
@@ -186,27 +213,51 @@ export const Util = {
         return true;
     },
 
+    /**
+     * helper for Mcdev.retrieve, Mcdev.retrieveAsTemplate and Mcdev.deploy
+     *
+     * @param {Mcdevrc} properties javascript object in .mcdevrc.json
+     * @param {string} businessUnit name of BU
+     * @param {boolean} [handleOutside] if the API reponse is irregular this allows you to handle it outside of this generic method
+     * @returns {boolean} bu found or not
+     */
+    _isValidBU(properties, businessUnit, handleOutside) {
+        const [cred, bu] = businessUnit ? businessUnit.split('/') : [null, null];
+        if (!properties.credentials[cred]) {
+            if (!handleOutside) {
+                Util.logger.error(`Credential not found`);
+            }
+            return false;
+        } else if (!properties.credentials[cred].businessUnits[bu]) {
+            if (!handleOutside) {
+                Util.logger.error(`BU not found in credential`);
+            }
+            return false;
+        }
+        return true;
+    },
+
     /**
      * helper that deals with extracting type and subtype
      *
      * @param {string} selectedType "type" or "type-subtype"
-     * @returns {string[]} first elem is type, second elem is subType
+     * @returns {{type:string, subType:string}} first elem is type, second elem is subType
      */
     getTypeAndSubType(selectedType) {
         if (selectedType) {
             const temp = selectedType.split('-');
             const type = temp.shift(); // remove first item which is the main typ
             const subType = temp.join('-'); // subType can include "-"
-            return [type, subType];
+            return { type, subType };
         } else {
-            return [];
+            return { type: null, subType: null };
         }
     },
 
     /**
      * helper for getDefaultProperties()
      *
-     * @returns {TYPE.SupportedMetadataTypes[]} type choices
+     * @returns {string[]} type choices
      */
     getRetrieveTypeChoices() {
         const typeChoices = [];
@@ -251,8 +302,9 @@ export const Util = {
         //   silly: 6
         // }
         if (
-            process.env.VSCODE_AMD_ENTRYPOINT === 'vs/workbench/api/node/extensionHostProcess' ||
-            process.env.VSCODE_CRASH_REPORTER_PROCESS_TYPE === 'extensionHost'
+            this.isRunViaVSCodeExtension || // run via VSCode extension
+            process.env.FORK_PROCESS_ID || // run via Git-Fork
+            process.env.PATH.toLowerCase().includes('sourcetree') // run via Atlassian SourceTree
         ) {
             Util.OPTIONS.noLogColors = true;
         }
@@ -287,20 +339,23 @@ export const Util = {
                     )
                 ),
             });
-            transports.fileError = new winston.transports.File({
-                // Write logs to additional error-logfile for better visibility of errors
-                filename: 'logs/' + logFileName + '-errors.log',
-                level: 'error', // only log errors
-                lazy: true, // if true, log files will be created on demand, not at the initialization time.
-                format: winston.format.combine(
-                    winston.format.uncolorize(),
-                    winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
-                    winston.format.simple(),
-                    winston.format.printf(
-                        (info) => `${info.timestamp} ${info.level}: ${info.message}`
-                    )
-                ),
-            });
+            if (Util.OPTIONS.errorLog) {
+                // used by CI/CD solutions like Copado to quickly show the error message to admins/users
+                transports.fileError = new winston.transports.File({
+                    // Write logs to additional error-logfile for better visibility of errors
+                    filename: 'logs/' + logFileName + '-errors.log',
+                    level: 'error', // only log errors
+                    lazy: true, // if true, log files will be created on demand, not at the initialization time.
+                    format: winston.format.combine(
+                        winston.format.uncolorize(),
+                        winston.format.timestamp({ format: 'HH:mm:ss.SSS' }),
+                        winston.format.simple(),
+                        winston.format.printf(
+                            (info) => `${info.timestamp} ${info.level}: ${info.message}`
+                        )
+                    ),
+                });
+            }
         }
         return transports;
     },
@@ -309,7 +364,7 @@ export const Util = {
     /**
      * Logger that creates timestamped log file in 'logs/' directory
      *
-     * @type {TYPE.Logger}
+     * @type {Logger}
      */
     logger: null,
     /**
@@ -339,11 +394,12 @@ export const Util = {
             transports: Object.values(Util.loggerTransports),
         });
         const winstonError = myWinston.error;
+        /** @type {McdevLogger} */
         const winstonExtension = {
             /**
              * helper that prints better stack trace for errors
              *
-             * @param {Error} ex the error
+             * @param {SDKError} ex the error
              * @param {string} [message] optional custom message to be printed as error together with the exception's message
              * @returns {void}
              */
@@ -428,7 +484,7 @@ export const Util = {
      * key-value pairs (obj)
      *
      * @param {string | object} str JSON object or its stringified version, which has values to be replaced
-     * @param {TYPE.TemplateMap} obj key value object which contains keys to be replaced and values to be replaced with
+     * @param {TemplateMap} obj key value object which contains keys to be replaced and values to be replaced with
      * @returns {string | object} replaced version of str
      */
     replaceByObject: function (str, obj) {
@@ -448,7 +504,7 @@ export const Util = {
 
         sortable.sort((a, b) => b[1].length - a[1].length);
         for (const pair of sortable) {
-            const escVal = pair[1].toString().replaceAll(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+            const escVal = pair[1].toString().replaceAll(/[-/\\^$*+?.()|[\]{}]/g, String.raw`\$&`);
             const regString = new RegExp(escVal, 'g');
             str = str.replace(regString, '{{{' + pair[0] + '}}}');
         }
@@ -462,7 +518,7 @@ export const Util = {
      * get key of an object based on the first matching value
      *
      * @param {object} objs object of objects to be searched
-     * @param {string} val value to be searched for
+     * @param {string | number} val value to be searched for
      * @returns {string} key
      */
     inverseGet: function (objs, val) {
@@ -548,40 +604,45 @@ export const Util = {
 
         // sort list & remove the undefined dependencies
         const flatList = toposort(dependencies).filter((a) => !!a);
-        const finalList = {};
+        /** @type {Object.<string, null | Set.<string>>} */
+        const setList = {};
         // group subtypes per type
-        for (const type of flatList) {
-            if (type.includes('-')) {
-                const [key, subKey] = Util.getTypeAndSubType(type);
-                if (finalList[key] === null) {
+        for (const flatType of flatList) {
+            if (flatType.includes('-')) {
+                const { type, subType } = Util.getTypeAndSubType(flatType);
+                if (setList[type] === null) {
                     // if main type is already required, then don't filter by subtypes
                     continue;
-                } else if (finalList[key] && subKey) {
+                } else if (setList[type] && subType) {
                     // add another subtype to the set
-                    finalList[key].add(subKey);
+                    setList[type].add(subType);
                     continue;
                 } else {
                     // create a new set with the first subtype; subKey will be always set here
-                    finalList[key] = new Set([subKey]);
+                    setList[type] = new Set([subType]);
                 }
-                if (subTypeDeps[key]) {
+                if (subTypeDeps[type]) {
                     // check if there are depndent subtypes that need to be added
-                    const temp = [...subTypeDeps[key]].map((a) => {
+                    /** @type {string[]} */
+                    const temp = [...subTypeDeps[type]].map((a) => {
                         const temp = a.split('-');
                         temp.shift(); // remove first item which is the main type
                         return temp.join('-'); // subType can include "-"
                     });
-                    finalList[key].add(...temp);
+                    for (const item of temp) {
+                        setList[type].add(item);
+                    }
                 }
             } else {
-                finalList[type] = null;
+                setList[flatType] = null;
             }
         }
         // convert sets into arrays
-        for (const type of Object.keys(finalList)) {
-            if (finalList[type] instanceof Set) {
-                finalList[type] = [...finalList[type]];
-            }
+        /** @type {Object.<string, string[]>} */
+        const finalList = {};
+
+        for (const type of Object.keys(setList)) {
+            finalList[type] = setList[type] instanceof Set ? [...setList[type]] : null;
         }
 
         return finalList;
@@ -630,10 +691,10 @@ export const Util = {
     /**
      * standardize check to ensure only one result is returned from template search
      *
-     * @param {TYPE.MetadataTypeItem[]} results array of metadata
+     * @param {MetadataTypeItem[]} results array of metadata
      * @param {string} keyToSearch the field which contains the searched value
      * @param {string} searchValue the value which is being looked for
-     * @returns {TYPE.MetadataTypeItem} metadata to be used in building template
+     * @returns {MetadataTypeItem} metadata to be used in building template
      */
     templateSearchResult(results, keyToSearch, searchValue) {
         const matching = results.filter((item) => item[keyToSearch] === searchValue);
@@ -838,8 +899,8 @@ export const Util = {
         }
         // Remove special chars
         search = search.replaceAll(
-            new RegExp('([\\.\\\\\\+\\*\\?\\[\\^\\]\\$\\(\\)\\{\\}\\=\\!\\<\\>\\|\\:\\-])', 'g'),
-            '\\$1'
+            new RegExp(String.raw`([\.\\\+\*\?\[\^\]\$\(\)\{\}\=\!\<\>\|\:\-])`, 'g'),
+            String.raw`\$1`
         );
         // Replace % and _ with equivalent regex
         search = search.replaceAll('%', '.*').replaceAll('_', '.');
@@ -849,7 +910,7 @@ export const Util = {
     /**
      * returns true if no LIKE filter is defined or if all filters match
      *
-     * @param {TYPE.MetadataTypeItem} metadata a single metadata item
+     * @param {MetadataTypeItem} metadata a single metadata item
      * @param {object} [filters] only used in recursive calls
      * @returns {boolean} true if no LIKE filter is defined or if all filters match
      */
@@ -892,6 +953,43 @@ export const Util = {
     capitalizeFirstLetter(str) {
         return str.charAt(0).toUpperCase() + str.slice(1);
     },
+    /**
+     * helper for Retriever and Deployer class
+     *
+     * @param {string[]} typeArr -
+     * @param {string[]} keyArr -
+     * @param {boolean} [returnEmpty] returns array with null element if false/not set; Retriever needs this to be false; Deployer needs it to be true
+     * @returns {TypeKeyCombo} -
+     */
+    createTypeKeyCombo(typeArr, keyArr, returnEmpty = false) {
+        if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) {
+            // no keys were provided, ensure we retrieve all
+            keyArr = returnEmpty ? null : [null];
+        }
+        /** @type {TypeKeyCombo} */
+        const typeKeyMap = {};
+        // no keys or array of keys was provided (likely called via CLI or to retrieve all)
+        // transform into TypeKeyCombo to iterate over it
+        for (const type of typeArr) {
+            typeKeyMap[type] = keyArr;
+        }
+        return typeKeyMap;
+    },
+    /**
+     * async version of Array.find()
+     * returns the first element in the provided array that satisfies the provided testin function
+     *
+     * @param {Array} arr your test array
+     * @param {Function} asyncCallback callback
+     * @returns {Promise.<any | undefined>} first element that passed the test
+     */
+    async findAsync(arr, asyncCallback) {
+        for (const element of arr) {
+            if (await asyncCallback(element)) {
+                return element;
+            }
+        }
+    },
 };
 
 Util.startLogger(false, true);