mcdev 9.0.2 → 10.0.0

package/lib/index.js

@@ -16,6 +16,8 @@ import MetadataTypeDefinitions from './MetadataTypeDefinitions.js';
 import Retriever from './Retriever.js';
 import cache from './util/cache.js';
 import ReplaceContentBlockReference from './util/replaceContentBlockReference.js';
+import { migrateAssetV10Tree } from './util/migrations/assetV10.js';
+import fs from 'fs-extra';
 import pLimit from 'p-limit';
 import path from 'node:path';
 
@@ -41,12 +43,13 @@ import { confirm } from '@inquirer/prompts';
  * @typedef {import('../types/mcdev.d.js').ExplainType} ExplainType
  * @typedef {import('../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
  * @typedef {import('../types/mcdev.d.js').BuildFilter} BuildFilter
+ * @typedef {import('./util/migrations/assetV10.js').AssetV10MigrationReport} AssetV10MigrationReport
  */
 
 /**
  * main class
  */
-class Mcdev {
+export default class Mcdev {
     /**
      * @returns {string} current version of mcdev
      */
@@ -79,7 +82,6 @@ class Mcdev {
     }
     static knownOptions = [
         '_runningTest',
-        '_welcomeMessageShown',
         'api',
         'autoMidSuffix',
         'changeKeyField',
@@ -117,6 +119,7 @@ class Mcdev {
         'skipStatusCheck',
         'skipValidation',
         'validate',
+        'shareAsset',
     ];
 
     /**
@@ -185,6 +188,158 @@ class Mcdev {
         }
     }
 
+    /**
+     * Migrates historical v10 asset groupings in one selected BU retrieve tree.
+     * Pass `credential/*` to migrate every Business Unit of that credential sequentially.
+     *
+     * @param {string} [businessUnit] configured credential/BU; omit to select interactively; `credential/*` targets all BUs of one credential
+     * @returns {Promise.<AssetV10MigrationReport | Object.<string, AssetV10MigrationReport>>} per-BU reports, or the single report for a single BU
+     */
+    static async migrate(businessUnit) {
+        Util.startLogger();
+        const properties = await config.getProperties();
+        if (!properties) {
+            return;
+        }
+
+        const target = businessUnit ? businessUnit.split('/') : [null, null];
+        const [cred, bu] = target;
+        if (businessUnit && (target.length > 2 || target.some((part) => !part))) {
+            Util.logger.error(
+                `Invalid migration target '${businessUnit}': expected credential/BU or credential/*.`
+            );
+            return;
+        }
+        if (bu === '*' && properties.credentials?.[cred]) {
+            // every BU of one credential; the BU list comes from the config, never from disk
+            return await this.#migrateAllBus(properties, cred);
+        }
+
+        const buObject = await Cli.getCredentialObject(properties, businessUnit);
+        if (!buObject) {
+            return;
+        }
+        const assetRoot = this.#getMigrationRoot(properties, buObject);
+        this.#logMigrationHeader();
+        const report = await migrateAssetV10Tree(assetRoot);
+        await this.#logMigrationReport(report);
+        return report;
+    }
+
+    /**
+     * Migrates the retrieve asset tree of every BU of one credential sequentially.
+     *
+     * @param {Mcdevrc} properties config properties
+     * @param {string} cred name of the credential whose BUs are migrated
+     * @returns {Promise.<Object.<string, AssetV10MigrationReport>>} migration report per `credential/BU`
+     */
+    static async #migrateAllBus(properties, cred) {
+        this.#logMigrationHeader();
+        /** @type {Object.<string, AssetV10MigrationReport>} */
+        const reports = {};
+        let conflictedBus = 0;
+        for (const buName in properties.credentials[cred].businessUnits) {
+            // resolve every BU through the same logic the single-BU command uses
+            const buObject = await Cli.getCredentialObject(properties, `${cred}/${buName}`);
+            if (!buObject) {
+                continue;
+            }
+            const buPath = `${buObject.credential}/${buObject.businessUnit}`;
+            try {
+                Util.logger.info(`Migrating assets on ${buPath}:`);
+                const report = await migrateAssetV10Tree(
+                    this.#getMigrationRoot(properties, buObject)
+                );
+                reports[buPath] = report;
+                if (report.conflicts.length) {
+                    conflictedBus++;
+                }
+                await this.#logMigrationReport(report, buPath);
+            } catch (ex) {
+                // a broken BU must not stop the remaining BUs from being migrated
+                Util.logger.error(`${buPath}: Asset v10 migration failed: ${ex.message}`);
+            }
+        }
+        Util.logger.info(
+            `Asset v10 migration: processed ${Object.keys(reports).length} BUs of ${cred}, ${conflictedBus} of them had conflicts.`
+        );
+        return reports;
+    }
+
+    /**
+     * Builds the retrieve asset root of one BU exactly as the single-BU migration does.
+     *
+     * @param {Mcdevrc} properties config properties
+     * @param {BuObject} buObject resolved credential/BU
+     * @returns {string} path to that BU's retrieve asset folder
+     */
+    static #getMigrationRoot(properties, buObject) {
+        return File.normalizePath([
+            properties.directories.retrieve,
+            buObject.credential,
+            buObject.businessUnit,
+            'asset',
+        ]);
+    }
+
+    /**
+     * Prints the messages shown once at the start of a migration run.
+     *
+     * @returns {void} -
+     */
+    static #logMigrationHeader() {
+        Util.logger.info('Migrating assets to their new folders ...');
+        Util.logger.info(
+            'This only reorganizes the local retrieve folder and makes no changes on the server.'
+        );
+        Util.logger.info('This may take a while ...');
+    }
+
+    /**
+     * Writes the report of one BU to the log; per-BU runs prefix every line with the BU path.
+     *
+     * @param {AssetV10MigrationReport} report migration report of one BU
+     * @param {string} [buPath] `credential/BU` label; omitted for the unchanged single-BU output
+     * @returns {Promise.<void>} -
+     */
+    static async #logMigrationReport(report, buPath) {
+        const prefix = buPath ? `${buPath}: ` : '';
+        Util.logger.info(
+            `${prefix}Asset v10 migration: ${report.moved.length} moved, ${report.conflicts.length} conflicts, ${report.skipped.length} skipped.`
+        );
+        for (const conflict of report.conflicts) {
+            Util.logger.warn(`${prefix}Asset v10 migration conflict: ${conflict}`);
+        }
+        if (report.unremovedEmptyDirs.length) {
+            Util.logger.warn(
+                `${prefix}Asset v10 migration: the operating system did not allow deleting these now-empty folders; please remove them manually: ${report.unremovedEmptyDirs.join(', ')}`
+            );
+        }
+        if (report.unremovedNonEmptyDirs.length) {
+            const remaining = [];
+            for (const dir of report.unremovedNonEmptyDirs) {
+                let entries = [];
+                try {
+                    entries = await fs.readdir(dir);
+                } catch {
+                    // the folder vanished after the migration report was built
+                }
+                remaining.push(entries.length ? `${dir} (contains: ${entries.join(', ')})` : dir);
+            }
+            Util.logger.warn(
+                `${prefix}Asset v10 migration: these folders still contain files and must NOT be deleted: ${remaining.join('; ')}`
+            );
+        }
+        const unmappedSummary = Object.entries(report.unmapped)
+            .map(([assetTypeName, owners]) => `${assetTypeName} (${owners.length})`)
+            .join(', ');
+        if (unmappedSummary) {
+            Util.logger.warn(
+                `${prefix}Asset v10 migration: unmapped asset types still inside: ${unmappedSummary}`
+            );
+        }
+    }
+
     /**
      * @returns {Promise} .
      */
@@ -222,40 +377,6 @@ class Mcdev {
         return Init.upgradeProject(properties, false);
     }
 
-    /**
-     * helper to show an off-the-logs message to users
-     */
-    static #welcomeMessage() {
-        if (Util.OPTIONS._welcomeMessageShown) {
-            // ensure we don't spam the user in case methods are called multiple times
-            return;
-        }
-        Util.OPTIONS._welcomeMessageShown = true;
-
-        const color = Util.color;
-        /* eslint-disable no-console */
-        if (process.env['USERDNSDOMAIN'] === 'DIR.SVC.ACCENTURE.COM') {
-            // Accenture internal message
-            console.log(
-                `\n` +
-                    `    Thank you for using Accenture SFMC DevTools on your Accenture laptop!\n` +
-                    `    We are trying to understand who is using mcdev across the globe and would therefore appreciate it if you left a message\n` +
-                    `    in our Accenture Teams channel ${color.bgWhite}telling us about your journey with mcdev${color.reset}: ${color.fgBlue}https://go.accenture.com/mcdevTeams${color.reset}.\n` +
-                    `\n` +
-                    `    For any questions or concerns, please feel free to create a ticket in GitHub: ${color.fgBlue}https://bit.ly/mcdev-support${color.reset}.\n`
-            );
-        } else {
-            // external message
-            console.log(
-                `\n` +
-                    `    Thank you for using Accenture SFMC DevTools!\n` +
-                    `\n` +
-                    `    For any questions or concerns, please feel free to create a ticket in GitHub: ${color.fgBlue}https://bit.ly/mcdev-support${color.reset}.\n`
-            );
-        }
-        /* eslint-enable no-console */
-    }
-
     /**
      * Retrieve all metadata from the specified business unit into the local file system.
      *
@@ -266,7 +387,6 @@ class Mcdev {
      * @returns {Promise.<object>} -
      */
     static async retrieve(businessUnit, selectedTypesArr, keys, changelogOnly) {
-        this.#welcomeMessage();
         console.time('Time'); // eslint-disable-line no-console
         Util.startLogger();
         Util.logger.info('mcdev:: Retrieve');
@@ -471,7 +591,9 @@ class Mcdev {
                 await File.remove(File.normalizePath([properties.directories.retrieve, cred, bu]));
                 // removes subtypes and removes duplicates
                 retrieveTypesArr.push(
-                    ...new Set(properties.metaDataTypes.retrieve.map((type) => type.split('-')[0]))
+                    ...new Set(
+                        properties.metaDataTypes.retrieve.map((type) => type.split('-', 1)[0])
+                    )
                 );
                 for (const selectedType of retrieveTypesArr) {
                     const test = Util._isValidType(selectedType);
@@ -490,7 +612,6 @@ class Mcdev {
                 const retrieveChangelog = await retriever.retrieve(
                     retrieveTypesArr,
                     Array.isArray(selectedTypesArr) ? keys : selectedTypesArr,
-                    null,
                     changelogOnly
                 );
                 return retrieveChangelog;
@@ -509,7 +630,6 @@ class Mcdev {
      * @returns {Promise.<Object.<string, MultiMetadataTypeMap>>} deployed metadata per BU (first key: bu name, second key: metadata type)
      */
     static async deploy(businessUnit, selectedTypesArr, keyArr) {
-        this.#welcomeMessage();
         console.time('Time'); // eslint-disable-line no-console
         Util.startLogger();
         const deployResult = await Deployer.deploy(businessUnit, selectedTypesArr, keyArr);
@@ -592,7 +712,7 @@ class Mcdev {
             const parentBUOnlyTypes = ['user', 'role'];
             const buObject = await Cli.getCredentialObject(
                 properties,
-                parentBUOnlyTypes.includes(type) ? businessUnit.split('/')[0] : businessUnit,
+                parentBUOnlyTypes.includes(type) ? businessUnit.split('/', 1)[0] : businessUnit,
                 parentBUOnlyTypes.includes(type) ? Util.parentBuName : null
             );
             if (buObject !== null) {
@@ -621,10 +741,10 @@ class Mcdev {
         Util.startLogger();
         Util.logger.info('mcdev:: delete');
 
-        /** @typedef {string[]} */
+        /** @type {string[]} */
         let selectedTypesArr;
 
-        /** @typedef {TypeKeyCombo} */
+        /** @type {TypeKeyCombo} */
         let selectedTypesObj;
         let keyArr;
         keyArr = 'string' === typeof keys ? [keys] : keys;
@@ -847,18 +967,20 @@ class Mcdev {
                 let listEntries = '';
                 for (const entry in metadata[metadataType]) {
                     const metadataEntry = metadata[metadataType][entry];
-                    if (regex.test(entry)) {
-                        if (metadataType === 'query' && metadataEntry.Status === 'Inactive') {
-                            continue;
-                        }
-                        listEntries +=
-                            '- ' +
-                            entry +
-                            (metadataEntry.name || metadataEntry.Name
-                                ? ' => ' + (metadataEntry.name || metadataEntry.Name)
-                                : '') +
-                            '\n';
+                    if (!regex.test(entry)) {
+                        continue;
+                    }
+
+                    if (metadataType === 'query' && metadataEntry.Status === 'Inactive') {
+                        continue;
                     }
+                    listEntries +=
+                        '- ' +
+                        entry +
+                        (metadataEntry.name || metadataEntry.Name
+                            ? ' => ' + (metadataEntry.name || metadataEntry.Name)
+                            : '') +
+                        '\n';
                 }
                 if (listEntries !== '') {
                     output += '\n## ' + metadataType + '\n\n' + listEntries;
@@ -874,50 +996,6 @@ class Mcdev {
         }
     }
 
-    /**
-     * Retrieve a specific metadata file and templatise.
-     *
-     * @deprecated Use `retrieve` followed by `build` instead. `retrieveAsTemplate` will be removed in a future version.
-     * @param {string} businessUnit references credentials from properties.json
-     * @param {string} selectedType supported metadata type
-     * @param {string[]} name name of the metadata
-     * @param {string} market market which should be used to revert template
-     * @returns {Promise.<MultiMetadataTypeList>} -
-     */
-    static async retrieveAsTemplate(businessUnit, selectedType, name, market) {
-        Util.startLogger();
-        Util.logDeprecated('retrieveAsTemplate', `'retrieve' followed by 'build'`);
-        const properties = await config.getProperties();
-        if (!properties) {
-            return;
-        }
-
-        if (!Util._isValidType(selectedType)) {
-            return;
-        }
-        const { type, subType } = Util.getTypeAndSubType(selectedType);
-
-        let retrieveTypesArr;
-        if (
-            type &&
-            subType &&
-            MetadataTypeInfo[type] &&
-            MetadataTypeDefinitions[type].subTypes.includes(subType)
-        ) {
-            retrieveTypesArr = [selectedType];
-        } else if (type && MetadataTypeInfo[type]) {
-            retrieveTypesArr = [type];
-        }
-        const buObject = await Cli.getCredentialObject(properties, businessUnit);
-        if (buObject !== null) {
-            cache.initCache(buObject);
-            const retriever = new Retriever(properties, buObject);
-            if (Util.checkMarket(market, properties)) {
-                return retriever.retrieve(retrieveTypesArr, name, properties.markets[market]);
-            }
-        }
-    }
-
     /**
      * @param {string} businessUnit references credentials from properties.json
      * @param {TypeKeyCombo} typeKeyList limit retrieval to given metadata type
@@ -1004,12 +1082,12 @@ class Mcdev {
                     `Found ${additionalAssetDependencies.length - initialAssetNumber} additional assets linked via ContentBlockByX.`
                 );
             }
-            // reset cache in case this is used progammatically somehow
-            Asset.getJsonFromFSCache = null;
 
             // remove duplicates in main object after adding dependencies
             typeKeyList[depType] = [...new Set(typeKeyList[depType])];
         }
+        // reset cache regardless of whether assets were found, to avoid stale data in programmatic use
+        MetadataTypeInfo['asset'].getJsonFromFSCache = null;
 
         return typeKeyList;
     }
@@ -1214,8 +1292,6 @@ class Mcdev {
      * @returns {Promise.<MultiMetadataTypeList>} -
      */
     static async buildTemplate(businessUnit, selectedTypes, keyArr, marketArr, filter) {
-        this.#welcomeMessage();
-
         Util.startLogger();
         Util.logger.info('mcdev:: Build Template from retrieved files');
         const properties = await config.getProperties();
@@ -1248,43 +1324,41 @@ class Mcdev {
         ]);
         for (const type of Object.keys(typeKeyList)) {
             const keyArr = typeKeyList[type];
-            if (keyArr.some((key) => key.startsWith('name:'))) {
-                // at least one key was provided as a name -> load all files from disk to try and find that key
-                const builTemplateCache = Object.values(
-                    await MetadataTypeInfo[type].getJsonFromFS(retrieveDir + path.sep + type)
-                );
-                typeKeyList[type] = keyArr
-                    .map((key) => {
-                        if (key.startsWith('name:')) {
-                            // key was defined by name. try and find matching item on disk
-                            const name = key.slice(5);
-                            const foundKeysByName = builTemplateCache
-                                .filter(
-                                    (item) =>
-                                        name == item[MetadataTypeInfo[type].definition.nameField]
-                                )
-                                .map((item) => item[MetadataTypeInfo[type].definition.keyField]);
-                            if (foundKeysByName.length === 1) {
-                                key = foundKeysByName[0];
-                                Util.logger.debug(
-                                    `- found ${type} key '${key}' for name '${name}'`
-                                );
-                                return key;
-                            } else if (foundKeysByName.length > 1) {
-                                Util.logger.error(
-                                    `Found multiple keys (${foundKeysByName.join(', ')}) for name: ${key}`
-                                );
-                                return;
-                            } else {
-                                Util.logger.error(`Could not find any keys for name: ${name}`);
-                                return;
-                            }
-                        } else {
+            if (keyArr.every((key) => !key.startsWith('name:'))) {
+                continue;
+            }
+            // at least one key was provided as a name -> load all files from disk to try and find that key
+            const builTemplateCache = Object.values(
+                await MetadataTypeInfo[type].getJsonFromFS(retrieveDir + path.sep + type)
+            );
+            typeKeyList[type] = keyArr
+                .map((key) => {
+                    if (key.startsWith('name:')) {
+                        // key was defined by name. try and find matching item on disk
+                        const name = key.slice(5);
+                        const foundKeysByName = builTemplateCache
+                            .filter(
+                                (item) => name == item[MetadataTypeInfo[type].definition.nameField]
+                            )
+                            .map((item) => item[MetadataTypeInfo[type].definition.keyField]);
+                        if (foundKeysByName.length === 1) {
+                            key = foundKeysByName[0];
+                            Util.logger.debug(`- found ${type} key '${key}' for name '${name}'`);
                             return key;
+                        } else if (foundKeysByName.length > 1) {
+                            Util.logger.error(
+                                `Found multiple keys (${foundKeysByName.join(', ')}) for name: ${key}`
+                            );
+                            return;
+                        } else {
+                            Util.logger.error(`Could not find any keys for name: ${name}`);
+                            return;
                         }
-                    })
-                    .filter(Boolean);
-            }
+                    } else {
+                        return key;
+                    }
+                })
+                .filter(Boolean);
         }
 
         // if dependencies are enabled, we need to search for them and add them to our
@@ -1428,7 +1502,7 @@ class Mcdev {
                     const properties = await config.getProperties();
                     retrieveTypes = [
                         ...new Set(
-                            properties.metaDataTypes.retrieve.map((type) => type.split('-')[0])
+                            properties.metaDataTypes.retrieve.map((type) => type.split('-', 1)[0])
                         ),
                         ...Object.keys(defaultPlusTheseTypes),
                     ];
@@ -1448,15 +1522,10 @@ class Mcdev {
      * @returns {Promise.<MultiMetadataTypeList>} -
      */
     static async buildDefinition(businessUnit, selectedTypes, nameArr, marketArr) {
-        this.#welcomeMessage();
-
         Util.startLogger();
         Util.logger.info('mcdev:: Build Definition from Template');
         const properties = await config.getProperties();
-        if (!properties) {
-            return;
-        }
-        if (!Util.checkMarketList(marketArr, properties)) {
+        if (!properties || !Util.checkMarketList(marketArr, properties)) {
             return;
         }
         const typeKeyList = Util.checkAndPrepareTypeKeyCombo(
@@ -1498,8 +1567,6 @@ class Mcdev {
      * @returns {Promise.<object>} -
      */
     static async buildDefinitionBulk(listName, selectedTypes, nameArr) {
-        this.#welcomeMessage();
-
         Util.startLogger();
         Util.logger.info('mcdev:: Build Definition from Template Bulk');
 
@@ -1548,10 +1615,7 @@ class Mcdev {
         Util.startLogger();
         Util.logger.info('mcdev:: getFilesToCommit');
         const properties = await config.getProperties();
-        if (!properties) {
-            return;
-        }
-        if (!Util._isValidType(selectedType)) {
+        if (!properties || !Util._isValidType(selectedType)) {
             return;
         }
         if (selectedType.includes('-')) {
@@ -1691,14 +1755,11 @@ class Mcdev {
         this.setOptions({ referenceFrom, referenceTo });
 
         const properties = await config.getProperties();
-        if (!properties) {
-            return;
-        }
-        if (!Util.isValidBU(properties, businessUnit)) {
+        if (!properties || !Util.isValidBU(properties, businessUnit)) {
             return;
         }
 
-        /** @typedef {string[]} */
+        /** @type {string[]} */
         let selectedTypesArr;
         if (selectedTypesObj) {
             // check if types are valid
@@ -1712,7 +1773,7 @@ class Mcdev {
             selectedTypesArr = [
                 ...new Set(
                     properties.metaDataTypes.retrieve
-                        .map((type) => type.split('-')[0])
+                        .map((type) => type.split('-', 1)[0])
                         .filter((type) =>
                             Object.prototype.hasOwnProperty.call(
                                 MetadataTypeInfo[type],
@@ -1757,10 +1818,10 @@ class Mcdev {
         this.setOptions({ skipValidation: true });
         let reRetrieveAll = false;
 
-        /** @typedef {string[]} */
+        /** @type {string[]} */
         let selectedTypesArr;
 
-        /** @typedef {TypeKeyCombo} */
+        /** @type {TypeKeyCombo} */
         let selectedTypesObj;
         if (selectedTypes) {
             // check if types are valid
@@ -1781,7 +1842,7 @@ class Mcdev {
             selectedTypesArr = [
                 ...new Set(
                     properties.metaDataTypes.retrieve
-                        .map((type) => type.split('-')[0])
+                        .map((type) => type.split('-', 1)[0])
                         .filter(
                             (type) =>
                                 !MetadataTypeDefinitions[type].keyIsFixed &&
@@ -1821,7 +1882,7 @@ class Mcdev {
             );
             const buObject = await Cli.getCredentialObject(properties, businessUnit, null, true);
             const retriever = new Retriever(properties, buObject);
-            await retriever.retrieve(selectedTypesArr, null, null, false);
+            await retriever.retrieve(selectedTypesArr, null, false);
         }
         return response;
     }
@@ -1928,10 +1989,10 @@ class Mcdev {
             }
         }
 
-        /** @typedef {string[]} */
+        /** @type {string[]} */
         let selectedTypesArr;
 
-        /** @typedef {TypeKeyCombo} */
+        /** @type {TypeKeyCombo} */
         let selectedTypesObj;
         if (selectedTypes) {
             // check if types are valid
@@ -2252,7 +2313,7 @@ class Mcdev {
         try {
             Util.logger.info(`Retrieving latest versions of ${type} from server`);
             const retriever = new Retriever(properties, buObject);
-            const retrieved = await retriever.retrieve([type], keyArr, null, false);
+            const retrieved = await retriever.retrieve([type], keyArr, false);
 
             const metadataMap = Object.values(retrieved)[0][0];
             const keysForDeploy = MetadataTypeInfo[type].getKeysForFixing(metadataMap);
@@ -2288,7 +2349,7 @@ class Mcdev {
                             `Retrieving latest versions of ${dependentTypes.join(', ')} from server`
                         );
                         const retriever = new Retriever(properties, buObject);
-                        await retriever.retrieve(dependentTypes, null, null, false);
+                        await retriever.retrieve(dependentTypes, null, false);
                     }
                 } else {
                     Util.logger.info(
@@ -2490,5 +2551,3 @@ class Mcdev {
         return Object.keys(response).length >= 1 ? response : undefined;
     }
 }
-
-export default Mcdev;