mcdev 9.0.2 → 10.0.0

package/lib/metadataTypes/MetadataType.js

@@ -10,7 +10,7 @@
 import { Util } from '../util/util.js';
 import File from '../util/file.js';
 import cache from '../util/cache.js';
-import deepEqual from 'deep-equal';
+import { isDeepEqual } from '../util/isDeepEqual.js';
 import pLimit from 'p-limit';
 import Mustache from 'mustache';
 import MetadataTypeInfo from '../MetadataTypeInfo.js';
@@ -76,7 +76,7 @@ class MetadataType {
 
                         // ensure filename includes extended metadata extension
                         const regex = new RegExp(/\.(\w|-)+-meta.json/);
-                        const errorDir = dir.split('\\').join('/');
+                        const errorDir = dir.replaceAll('\\', '/');
                         if (regex.test(fileName)) {
                             const fileNameWithoutEnding = File.reverseFilterIllegalFilenames(
                                 fileName.split(regex)[0]
@@ -240,10 +240,9 @@ class MetadataType {
      *
      * @param {MetadataTypeItem} metadata a single item
      * @param {string} targetDir folder where retrieves should be saved
-     * @param {boolean} [isTemplating] signals that we are retrieving templates
      * @returns {MetadataTypeItem} cloned metadata
      */
-    static postRetrieveTasks(metadata, targetDir, isTemplating) {
+    static postRetrieveTasks(metadata, targetDir) {
         return structuredClone(metadata);
     }
 
@@ -328,32 +327,58 @@ class MetadataType {
     }
 
     /**
-     * Gets metadata cache with limited fields and does not store value to disk
+     * helper for buildTemplate
      *
-     * @deprecated Use `retrieve` followed by `build` instead. `retrieveAsTemplate` will be removed in a future version.
-     * @param {string} templateDir Directory where retrieved metadata directory will be saved
-     * @param {string} name name of the metadata file
-     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
-     * @param {string} [subType] optionally limit to a single subtype
-     * @returns {Promise.<MetadataTypeItemObj>} metadata
+     * @param {unknown} value convert anything to string
+     * @returns {string|unknown[]|object} stringified value
      */
-    static async retrieveAsTemplate(templateDir, name, templateVariables, subType) {
-        Util.logNotSupported(this.definition, 'retrieveAsTemplate');
-        Util.logDeprecated('retrieveAsTemplate', `'retrieve' followed by 'build'`);
-        return { metadata: null, type: this.definition.type };
+    static stringifyValue(value) {
+        if (Array.isArray(value)) {
+            for (let i = 0; i < value.length; i++) {
+                value[i] = MetadataType.stringifyValue(value[i]);
+            }
+            return value;
+        } else if ('object' === typeof value) {
+            for (const key in value) {
+                value[key] = MetadataType.stringifyValue(value[key]);
+            }
+            return value;
+        }
+        return value.toString();
     }
 
     /**
-     * Retrieve a specific Script by Name
+     * helper for buildTemplate
      *
-     * @param {string} templateDir Directory where retrieved metadata directory will be saved
-     * @param {string} uri rest endpoint for GET
-     * @param {TemplateMap} templateVariables variables to be replaced in the metadata
-     * @param {string} name name (not key) of the metadata item
-     * @returns {Promise.<{metadata: MetadataTypeItem, type: string}>} Promise
+     * @param {object|unknown[]} target -
+     * @param {string[]} fieldPathParts -
+     * @returns {void} .
      */
-    static async retrieveTemplateREST(templateDir, uri, templateVariables, name) {
-        return this.retrieveREST(templateDir, uri, templateVariables, name);
+    static stringifyFieldByPath(target, fieldPathParts) {
+        if (!target) {
+            return;
+        }
+        const [currentPart, ...rest] = fieldPathParts;
+        if (currentPart === '%') {
+            if (Array.isArray(target)) {
+                for (const item of target) {
+                    MetadataType.stringifyFieldByPath(item, rest);
+                }
+            } else if ('object' === typeof target) {
+                for (const key in target) {
+                    MetadataType.stringifyFieldByPath(target[key], rest);
+                }
+            }
+            return;
+        }
+
+        if (rest.length) {
+            MetadataType.stringifyFieldByPath(target[currentPart], rest);
+        } else {
+            if (target[currentPart]) {
+                target[currentPart] = MetadataType.stringifyValue(target[currentPart]);
+            }
+        }
     }
 
     /**
@@ -411,20 +436,7 @@ class MetadataType {
             // numeric fields are returned as numbers by the SDK/API. If we try to replace them in buildTemplate it would break the JSON format - but not if we stringify them first because then the {{{var}}} is in quotes
             const metadataTemp = JSON.parse(metadataStr);
             for (const field of this.definition.stringifyFieldsBeforeTemplate) {
-                if (metadataTemp[field]) {
-                    if (Array.isArray(metadataTemp[field])) {
-                        for (let i = 0; i < metadataTemp[field].length; i++) {
-                            metadataTemp[field][i] = metadataTemp[field][i].toString();
-                        }
-                    } else if ('object' === typeof metadataTemp[field]) {
-                        for (const subField in metadataTemp[field]) {
-                            metadataTemp[field][subField] =
-                                metadataTemp[field][subField].toString();
-                        }
-                    } else {
-                        metadataTemp[field] = metadataTemp[field].toString();
-                    }
-                }
+                MetadataType.stringifyFieldByPath(metadataTemp, field.split('.'));
             }
             metadataStr = JSON.stringify(metadataTemp);
         }
@@ -444,11 +456,12 @@ class MetadataType {
         try {
             // write to file
             await File.writeJSONToFile([templateDir, ...typeDirArr], fileName, metadata);
-            Util.logger.info(
-                ` - templated ${this.definition.type}: ${key} (${
-                    metadata[this.definition.nameField]
-                })`
+            // we have to fake this ab it here be
+            const keyName = Util.getKeyName(
+                { type: this.definition.type, keyField: 'key', nameField: 'name' },
+                { key: key, name: metadata[this.definition.nameField] }
             );
+            Util.logger.info(` - templated ${this.definition.type}: ${keyName}`);
 
             return { metadata: metadata, type: this.definition.type };
         } catch (ex) {
@@ -472,7 +485,7 @@ class MetadataType {
      *
      * @param {string} key key of the item
      * @param {string} name name of the item
-     * @param {{ type: string; key: string; name: any; }[]} namesInFolder names of the items in the same folder
+     * @param {{ type: string; key: string; name: string; }[]} namesInFolder names of the items in the same folder
      * @param {string} [subtype] itemType-name
      * @returns {string} new name
      */
@@ -734,7 +747,7 @@ class MetadataType {
                     );
                     return true;
                 }
-            } else if (deepEqual(clonedMetada[prop], cachedVersion[prop])) {
+            } else if (!isDeepEqual(clonedMetada[prop], cachedVersion[prop])) {
                 // test complex objects here
                 Util.logger.debug(
                     `${this.definition.type}:: ${
@@ -1308,29 +1321,29 @@ class MetadataType {
      * @returns {Promise.<void>} -
      */
     static async _postChangeKeyTasks(metadataEntry, keepMap = false) {
-        if (
+        if (!(
             (Util.OPTIONS.changeKeyField || Util.OPTIONS.changeKeyValue) &&
             Util.changedKeysMap?.[this.definition.type]?.[metadataEntry[this.definition.keyField]]
-        ) {
-            const oldKey =
-                Util.changedKeysMap?.[this.definition.type]?.[
-                    metadataEntry[this.definition.keyField]
-                ];
+        )) {
+            return;
+        }
 
-            // delete file(s) of old key
-            await this.postDeleteTasks(oldKey);
+        const oldKey =
+            Util.changedKeysMap?.[this.definition.type]?.[metadataEntry[this.definition.keyField]];
 
-            // fix key in cache
-            const typeCache = cache.getCache()[this.definition.type];
-            typeCache[metadataEntry[this.definition.keyField]] = typeCache[oldKey];
-            delete typeCache[oldKey];
+        // delete file(s) of old key
+        await this.postDeleteTasks(oldKey);
 
-            if (!keepMap) {
-                // clean entry from to-do list
-                delete Util.changedKeysMap?.[this.definition.type]?.[
-                    metadataEntry[this.definition.keyField]
-                ];
-            }
+        // fix key in cache
+        const typeCache = cache.getCache()[this.definition.type];
+        typeCache[metadataEntry[this.definition.keyField]] = typeCache[oldKey];
+        delete typeCache[oldKey];
+
+        if (!keepMap) {
+            // clean entry from to-do list
+            delete Util.changedKeysMap?.[this.definition.type]?.[
+                metadataEntry[this.definition.keyField]
+            ];
         }
     }
 
@@ -1449,13 +1462,13 @@ class MetadataType {
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
      * @param {string} uri rest endpoint for GET
-     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
      * @param {string} [singleRetrieve] key of single item to filter by
-     * @returns {Promise.<{metadata: (MetadataTypeMap | MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
+     * @returns {Promise.<{metadata: MetadataTypeMap, type: string}>} Promise of item map
      */
-    static async retrieveREST(retrieveDir, uri, templateVariables, singleRetrieve) {
+    static async retrieveREST(retrieveDir, uri, singleRetrieve) {
         const response =
-            this.definition.restPagination && !singleRetrieve
+            (this.definition.restPagination && !singleRetrieve) ||
+            (this.definition.restPaginationForKey && singleRetrieve)
                 ? await this.client.rest.getBulk(uri, this.definition.restPageSize || 500)
                 : await this.client.rest.get(uri);
         const results = this.parseResponseBody(response, singleRetrieve);
@@ -1498,12 +1511,7 @@ class MetadataType {
         }
 
         if (retrieveDir) {
-            const savedMetadata = await this.saveResults(
-                results,
-                retrieveDir,
-                null,
-                templateVariables
-            );
+            const savedMetadata = await this.saveResults(results, retrieveDir);
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
                     Util.getKeysString(singleRetrieve)
@@ -1516,7 +1524,7 @@ class MetadataType {
         }
 
         return {
-            metadata: templateVariables ? Object.values(results)[0] : results,
+            metadata: results,
             type: this.definition.type,
         };
     }
@@ -1526,7 +1534,7 @@ class MetadataType {
      * @param {object[]} urlArray {uri: string, id: string} combo of URL and ID/key of metadata
      * @param {number} [concurrentRequests] optionally set a different amount of concurrent requests
      * @param {boolean} [logAmountOfUrls] if true, prints an info message about to-be loaded amount of metadata
-     * @returns {Promise.<{metadata: (MetadataTypeMap | MetadataTypeItem), type: string}>} Promise of item map (single item for templated result)
+     * @returns {Promise.<{metadata: MetadataTypeMap, type: string}>} Promise of item map
      */
     static async retrieveRESTcollection(urlArray, concurrentRequests = 10, logAmountOfUrls = true) {
         if (logAmountOfUrls) {
@@ -1571,7 +1579,7 @@ class MetadataType {
      * @param {RestError} ex exception
      * @param {string} key id or key of item
      * @param {string} url url to call for retry
-     * @returns {Promise.<any>} -
+     * @returns {Promise.<null>} no retry result
      */
     static async handleRESTErrors(ex, key, url) {
         // if the ID is too short, the system will throw the 400 error
@@ -2025,13 +2033,11 @@ class MetadataType {
      * @param {MetadataTypeMap} results metadata results from deploy
      * @param {string} retrieveDir directory where metadata should be stored after deploy/retrieve
      * @param {string} [overrideType] for use when there is a subtype (such as folder-queries)
-     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
      * @returns {Promise.<MetadataTypeMap>} Promise of saved metadata
      */
-    static async saveResults(results, retrieveDir, overrideType, templateVariables) {
+    static async saveResults(results, retrieveDir, overrideType) {
         const savedResults = {};
         let filterCounter = 0;
-        let subtypeExtension;
         for (const originalKey in results) {
             if (this.definition.type === 'asset') {
                 overrideType =
@@ -2043,7 +2049,7 @@ class MetadataType {
                         )
                     );
             }
-            subtypeExtension = '.' + (overrideType || this.definition.type) + '-meta';
+            const subtypeExtension = '.' + (overrideType || this.definition.type) + '-meta';
 
             try {
                 if (
@@ -2059,8 +2065,7 @@ class MetadataType {
                 const baseDir = [retrieveDir, ...(overrideType || this.definition.type).split('-')];
                 results[originalKey] = await this.postRetrieveTasks(
                     results[originalKey],
-                    retrieveDir,
-                    templateVariables ? true : false
+                    retrieveDir
                 );
                 if (!results[originalKey] || results[originalKey] === null) {
                     // we encountered a situation in our postRetrieveTasks that made us want to filter this record
@@ -2092,8 +2097,7 @@ class MetadataType {
                     results,
                     originalKey,
                     baseDir,
-                    subtypeExtension,
-                    templateVariables
+                    subtypeExtension
                 );
                 if (!savedResults[originalKey]) {
                     // happens in case like-filter struck
@@ -2124,25 +2128,12 @@ class MetadataType {
      * @param {string} originalKey key of metadata
      * @param {string[]} baseDir [retrieveDir, ...overrideType.split('-')]
      * @param {string} [subtypeExtension] e.g. ".asset-meta" or ".query-meta"
-     * @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
      * @returns {Promise.<MetadataTypeItem>} saved metadata
      */
-    static async saveToDisk(results, originalKey, baseDir, subtypeExtension, templateVariables) {
+    static async saveToDisk(results, originalKey, baseDir, subtypeExtension) {
         subtypeExtension ||= '.' + this.definition.type + '-meta';
         // for complex types like asset, script, query we need to save the scripts that were extracted from the JSON
         if (results[originalKey].json && results[originalKey].codeArr) {
-            // replace market values with template variable placeholders (do not do it on .codeArr)
-            if (templateVariables) {
-                results[originalKey].json = Util.replaceByObject(
-                    results[originalKey].json,
-                    templateVariables
-                );
-                results[originalKey].subFolder = Util.replaceByObject(
-                    results[originalKey].subFolder,
-                    templateVariables
-                );
-            }
-
             const postRetrieveData = results[originalKey];
             // normalize results[metadataEntry]
             results[originalKey] = postRetrieveData.json;
@@ -2168,20 +2159,11 @@ class MetadataType {
                         dir,
                         script.fileName + subtypeExtension,
                         script.fileExt,
-                        script.content,
-                        templateVariables
+                        script.content
                     );
                 })
             );
         } else {
-            // not a complex type, run the the entire JSON through templating
-            // replace market values with template variable placeholders
-            if (templateVariables) {
-                results[originalKey] = Util.replaceByObject(
-                    results[originalKey],
-                    templateVariables
-                );
-            }
             if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey], this.definition)) {
                 Util.logger.debug(`Filtered ${originalKey} because of --like option`);
                 return;
@@ -2197,11 +2179,7 @@ class MetadataType {
             delete saveClone[this.definition.idField];
         }
 
-        if (templateVariables) {
-            this.keepTemplateFields(saveClone);
-        } else {
-            this.keepRetrieveFields(saveClone);
-        }
+        this.keepRetrieveFields(saveClone);
 
         await File.writeJSONToFile(
             // manage subtypes
@@ -2209,12 +2187,6 @@ class MetadataType {
             originalKey + subtypeExtension,
             saveClone
         );
-        if (templateVariables) {
-            Util.logger.info(
-                `- templated ${this.definition.type}: ${saveClone[this.definition.nameField]}`
-            );
-        }
-
         return saveClone;
     }
 
@@ -2227,8 +2199,14 @@ class MetadataType {
      * @returns {string} code with markets applied
      */
     static applyTemplateValues(code, templateVariables) {
-        // replace template variables with their values
-        return Mustache.render(code, templateVariables, {}, ['{{{', '}}}']);
+        // Normalize regex entries without changing the shared market configuration.
+        const values = Object.fromEntries(
+            Object.entries(templateVariables).map(([key, value]) => [
+                key,
+                value && typeof value === 'object' ? value.replace : value,
+            ])
+        );
+        return Mustache.render(code, values, {}, ['{{{', '}}}']);
     }
 
     /**
@@ -2757,10 +2735,12 @@ class MetadataType {
                             dependentKeyArr.push(...foundKeys);
                         }
                     }
-                    if (dependentKeyArr.length) {
-                        dependentTypeKeyCombo[depType] ||= [];
-                        dependentTypeKeyCombo[depType].push(...dependentKeyArr);
+                    if (!dependentKeyArr.length) {
+                        continue;
                     }
+
+                    dependentTypeKeyCombo[depType] ||= [];
+                    dependentTypeKeyCombo[depType].push(...dependentKeyArr);
                 }
             }
             this.getDependentFilesExtra(metadataItem, dependentTypeKeyCombo);
@@ -2785,14 +2765,17 @@ class MetadataType {
             (key) => !multiTypeKeyList[this.definition.type].includes(key)
         );
         if (notFound && notFound.length) {
-            Util.logger.warn(
-                Util.getGrayMsg(
-                    `    ☇ skipping ${this.definition.type} dependenc${notFound.length === 1 ? 'y' : 'ies'} ${notFound.join(', ')}: Not found on your in your project folder.`
-                )
-            );
-            // make sure we don't search for it twice
-            notFoundList[this.definition.type] ||= [];
-            notFoundList[this.definition.type].push(...notFound);
+            const remainingNotFound = await this.handleNotFoundDependencies(notFound);
+            if (remainingNotFound.length) {
+                Util.logger.warn(
+                    Util.getGrayMsg(
+                        `    ☇ skipping ${this.definition.type} dependenc${remainingNotFound.length === 1 ? 'y' : 'ies'} ${remainingNotFound.join(', ')}: Not found on your in your project folder.`
+                    )
+                );
+                // make sure we don't search for it twice
+                notFoundList[this.definition.type] ||= [];
+                notFoundList[this.definition.type].push(...remainingNotFound);
+            }
         }
         return multiTypeKeyList;
     }
@@ -2805,6 +2788,19 @@ class MetadataType {
      */
     static getDependentFilesExtra(metadataItem, dependentTypeKeyCombo) {}
 
+    /**
+     * Hook called when dependency keys were not found in the primary retrieve folder.
+     * Override in subtypes to show type-specific warnings (e.g. shared/synchronized dataExtensions).
+     * Keys returned by this method still receive the generic "not found" warning.
+     * Used by {@link MetadataType.getDependentFiles}.
+     *
+     * @param {string[]} notFound keys that could not be found in the retrieve folder
+     * @returns {Promise.<string[]>} keys that should still trigger the default "not found" warning
+     */
+    static async handleNotFoundDependencies(notFound) {
+        return notFound;
+    }
+
     /**
      * helper for {@link MetadataType.getDependentFiles}
      *
@@ -2826,7 +2822,7 @@ class MetadataType {
     /**
      * helper for {@link MetadataType.getNestedValue}
      *
-     * @param {any} obj the metadataItem to search in (or the result)
+     * @param {ReturnType<JSON['parse']>} obj the metadataItem to search in (or the result)
      * @param {string[]} nestedKeyParts key in dot-notation split into parts
      * @param {string} dependentType used for types that need custom handling
      * @returns {(string) | (string)[]} result
@@ -2999,60 +2995,62 @@ class MetadataType {
         let fixed = false;
         for (const rule of Object.keys(validationRules)) {
             if (
-                validationConfig[rule] &&
-                validationConfig[rule] !== 'off' &&
-                !this.definition.skipValidation?.[rule]
+                !validationConfig[rule] ||
+                validationConfig[rule] === 'off' ||
+                this.definition.skipValidation?.[rule]
             ) {
-                const fixable = Object.prototype.hasOwnProperty.call(validationRules[rule], 'fix');
-                if (!fixable && validationConfig[rule] === 'fix') {
-                    // reduce from fix to error if not fixable
-                    validationConfig[rule] = 'error';
-                }
-                const mode = Util.OPTIONS.fix && fixable ? 'fix' : validationConfig[rule];
-                let passed;
+                continue;
+            }
 
-                try {
-                    passed = await (!Util.OPTIONS.skipValidation && mode === 'fix'
-                        ? validationRules[rule].fix()
-                        : validationRules[rule].passed());
-                } catch (ex) {
-                    Util.logger.errorStack(
-                        ex,
-                        `Error while running validation rule ${rule} (check log file for details!)`
-                    );
-                    continue;
-                }
-                if (mode === 'fix') {
-                    // potentially fixed really
-                    fixed = true;
-                }
-                if (!passed) {
-                    if (!Util.OPTIONS.skipValidation && mode === 'fix') {
-                        if (passed === false) {
-                            info.push('😎 auto-fixed: ' + validationRules[rule].failedMsg);
-                        } else if (passed === null) {
-                            // ensure this item no longer exists outside of the validation rule
-                            item = null;
-                            warnings.length = 0;
-                            info.length = 0;
-                            info.push(
-                                `🚫 ☇ filtered item via validation rule fix: ` +
-                                    validationRules[rule].failedMsg
-                            );
-                            break;
-                        }
-                    } else if (
-                        !Util.OPTIONS.skipValidation &&
-                        validationConfig[rule] === 'error' &&
-                        passed === false
-                    ) {
-                        throw new Error(validationRules[rule].failedMsg);
-                    } else if (
-                        (Util.OPTIONS.skipValidation || validationConfig[rule] === 'warn') &&
-                        passed === false
-                    ) {
-                        warnings.push(validationRules[rule].failedMsg);
+            const fixable = Object.prototype.hasOwnProperty.call(validationRules[rule], 'fix');
+            if (!fixable && validationConfig[rule] === 'fix') {
+                // reduce from fix to error if not fixable
+                validationConfig[rule] = 'error';
+            }
+            const mode = Util.OPTIONS.fix && fixable ? 'fix' : validationConfig[rule];
+            let passed;
+
+            try {
+                passed = await (!Util.OPTIONS.skipValidation && mode === 'fix'
+                    ? validationRules[rule].fix()
+                    : validationRules[rule].passed());
+            } catch (ex) {
+                Util.logger.errorStack(
+                    ex,
+                    `Error while running validation rule ${rule} (check log file for details!)`
+                );
+                continue;
+            }
+            if (mode === 'fix') {
+                // potentially fixed really
+                fixed = true;
+            }
+            if (!passed) {
+                if (!Util.OPTIONS.skipValidation && mode === 'fix') {
+                    if (passed === false) {
+                        info.push('😎 auto-fixed: ' + validationRules[rule].failedMsg);
+                    } else if (passed === null) {
+                        // ensure this item no longer exists outside of the validation rule
+                        item = null;
+                        warnings.length = 0;
+                        info.length = 0;
+                        info.push(
+                            `🚫 ☇ filtered item via validation rule fix: ` +
+                                validationRules[rule].failedMsg
+                        );
+                        break;
                     }
+                } else if (
+                    !Util.OPTIONS.skipValidation &&
+                    validationConfig[rule] === 'error' &&
+                    passed === false
+                ) {
+                    throw new Error(validationRules[rule].failedMsg);
+                } else if (
+                    (Util.OPTIONS.skipValidation || validationConfig[rule] === 'warn') &&
+                    passed === false
+                ) {
+                    warnings.push(validationRules[rule].failedMsg);
                 }
             }
         }

package/lib/metadataTypes/MobileCode.js

@@ -35,7 +35,6 @@ class MobileCode extends MetadataType {
         return super.retrieveREST(
             retrieveDir,
             '/legacy/v1/beta/mobile/code/' + (key ? `?$where=keyword%20eq%20%27${key}%27%20` : ''),
-            null,
             key
         );
     }