@eui/tools 6.14.21 → 6.15.1

package/scripts/csdr/metadata/backend.js

@@ -0,0 +1,148 @@
+'use strict';
+
+// GLOBAL
+const path = require('path');
+
+// LOCAL
+const tools = require('../../utils/tools');
+
+// FETCH ARGS
+const { dryRun } = tools.getArgs();
+
+
+const storeBuildMetadataBackend = (pkg, pkgVersion, pkgMetadata, envTarget) => {
+  return Promise.resolve()
+    .then(() => {
+      tools.logInfo('Storing package BUILD metadata...');
+
+      let generatedMetadata = {};
+      let versions = [];
+
+      let pkgMetadataFile = path.join(pkg.paths.root, 'package-build-metadata.json');
+
+      if (tools.isFileExists(pkgMetadataFile)) {
+        generatedMetadata = require(pkgMetadataFile);
+        versions = generatedMetadata.versions;
+      }
+
+      versions.push({ name: pkgVersion, envTarget: envTarget, gitMetadata: pkgMetadata });
+      generatedMetadata.versions = versions;
+
+      tools.logInfo(`Write metadata on ${pkgMetadataFile}`);
+      if (dryRun) {
+        tools.logInfo('DRY-RUN...');
+        console.log(generatedMetadata);
+      } else {
+        tools.writeJsonFileSync(pkgMetadataFile, generatedMetadata);
+      }
+    })
+
+    .catch((e) => {
+      throw e;
+    });
+};
+
+const storeMetadataVersionsBackend = (pkg, pkgVersion, duration, envTarget) => {
+  return Promise.resolve()
+    .then(() => {
+      tools.logInfo('Storing package version history metadata...');
+
+      let generatedMetadata = {};
+      let versions = [];
+      let pkgMetadataFile = path.join(pkg.paths.root, 'package-versions-metadata.json');
+
+      if (tools.isFileExists(pkgMetadataFile)) {
+        generatedMetadata = require(pkgMetadataFile);
+        versions = generatedMetadata.versions;
+      }
+
+      const newVersionMetadata = {
+        name: pkg.name,
+        version: pkgVersion,
+        date: moment(new Date()).format("YYYYMMDD-HH:mm"),
+        npmPkg: pkg.npmPkg,
+        duration: duration,
+        envTarget: envTarget,
+      };
+
+      console.log(newVersionMetadata);
+      versions.push(newVersionMetadata);
+      generatedMetadata.versions = versions;
+
+      tools.logInfo(`Write metadata on ${pkgMetadataFile}`);
+
+      if (dryRun) {
+        tools.logInfo('DRY RUN...outputing newVersionMetadata');
+        console.log(newVersionMetadata);
+
+      } else {
+        tools.writeJsonFileSync(pkgMetadataFile, generatedMetadata);
+      }
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+}
+
+const storePipelineMetadata = (pkgMetadata) => {
+  return Promise.resolve()
+    .then(() => {
+      tools.logInfo('Storing package pipeline metadata into root variables...');
+
+      const variablesFile = path.join(process.cwd(), 'variables');
+
+      let fileContent;
+      if (tools.isFileExists(variablesFile)) {
+        fileContent = tools.getFileContent(variablesFile);
+      }
+
+      if (pkgMetadata && pkgMetadata.issues) {
+        const issuesList = pkgMetadata.issues.join(',');
+        tools.logInfo(`Unique issues found for this version : ${issuesList}`);
+
+        tools.logInfo(`Write metadata on ${variablesFile}`);
+        if (!dryRun) {
+          fileContent += `export ISSUES_LIST=${issuesList}\n`;
+          tools.writeFileContent(variablesFile, fileContent);
+        }
+      } else {
+        tools.logInfo('No issues found for this release');
+      }
+    })
+
+    .catch((e) => {
+      console.log(e);
+    });
+};
+
+
+module.exports.storeMetadataBackend = (pkg, pkgVersion, pkgMetadata, isMaster, duration, envTarget) => {
+  tools.logTitle('Storing Backend metadata...');
+
+  return Promise.resolve()
+    .then(() => {
+      return storeMetadataVersionsBackend(pkg, pkgVersion, duration, envTarget);
+    })
+
+    .then(() => {
+      if (isMaster) {
+        return storeBuildMetadataBackend(pkg, pkgVersion, pkgMetadata, envTarget);
+      }
+    })
+
+    .then(() => {
+      if (isMaster) {
+        return storePipelineMetadata(pkgMetadata);
+      }
+    })
+
+    .then(() => {
+      tools.logSuccess();
+    })
+
+    .catch((e) => {
+      throw e;
+    });
+};
+

package/scripts/csdr/metadata/metadata-utils.js

@@ -5,11 +5,10 @@ module.exports.appEnvs = require('./app-envs');
 module.exports.appHistory = require('./app-history');
 module.exports.appVersions = require('./app-versions');
 module.exports.app = require('./app');
+module.exports.backend = require('./backend');
 module.exports.commit = require('./commit');
 module.exports.common = require('./common');
-module.exports.package = require('./package');
-module.exports.packageHistory = require('./package-history');
-module.exports.packageEnvs = require('./package-envs');
-module.exports.packageVersions = require('./package-versions');
+module.exports.packageUtils = require('./package-utils');
 module.exports.remotePackage = require('./remote-package');
+module.exports.remoteHistory = require('./remote-history');
 module.exports.stats = require('./stats');

package/scripts/csdr/metadata/{package.js → package-utils.js}

@@ -10,174 +10,10 @@ const tools = require('../../utils/tools');
 const configUtils = require('../config/config-utils');
 const gitUtils = require('../../utils/git-utils');
 
-// INNER MODULES
-const innerPackageEnvs = require('./package-envs');
-const innerPackageVersions = require('./package-versions');
-
 // FETCH ARGS
 const { dryRun, debug } = tools.getArgs();
 
-module.exports.storeMetadata = (pkg, pkgVersion, pkgMetadata, branches, pkgCompositeDeps, duration, envTarget) => {
-  tools.logTitle('Storing central and package metadata...');
-
-  return Promise.resolve()
-    .then(() => {
-      if (pkg.remote) {
-        return innerPackageVersions.storeMetadata(pkg, pkgVersion, pkgCompositeDeps, duration, envTarget);
-      }
-    })
-
-    .then(() => {
-      if (pkg.remote) {
-        return innerPackageEnvs.storeMetadata(pkg, pkgVersion, pkgCompositeDeps, envTarget);
-      }
-    })
-
-    .then(() => {
-      if (branches.isMaster) {
-        return storeBuildMetadata(pkg, pkgVersion, pkgMetadata, envTarget);
-      }
-    })
-
-    .then(() => {
-      if (branches.isMaster) {
-        return storePipelineMetadata(pkgMetadata);
-      }
-    })
-
-    .then(() => {
-      tools.logSuccess();
-    })
-
-    .catch((e) => {
-      throw e;
-    });
-};
-
-const storeBuildMetadata = (pkg, pkgVersion, pkgMetadata, envTarget) => {
-  return Promise.resolve()
-    .then(() => {
-      tools.logInfo('Storing package BUILD metadata...');
-
-      const configOptions = configUtils.global.getConfigOptions();
-
-      let generatedMetadata = {};
-      let versions = [];
-
-      let pkgMetadataFile;
-
-      if (pkg.backend) {
-        pkgMetadataFile = path.join(pkg.paths.root, 'package-build-metadata.json');
-      } else {
-        pkgMetadataFile = path.join(configOptions.DEVOPS_METADATA_PATH, pkg.devopsMetadataFile);
-      }
-
-      if (tools.isFileExists(pkgMetadataFile)) {
-        generatedMetadata = require(pkgMetadataFile);
-        versions = generatedMetadata.versions;
-      }
-
-      versions.push({ name: pkgVersion, envTarget: envTarget, gitMetadata: pkgMetadata });
-      generatedMetadata.versions = versions;
-
-      tools.logInfo(`Write metadata on ${pkgMetadataFile}`);
-      if (dryRun) {
-        tools.logInfo('DRY-RUN...');
-        console.log(generatedMetadata);
-      } else {
-        tools.writeJsonFileSync(pkgMetadataFile, generatedMetadata);
-      }
-    })
-
-    .catch((e) => {
-      throw e;
-    });
-};
-
-module.exports.storeMetadataBackend = (pkg, pkgVersion, pkgMetadata, isMaster, duration, envTarget) => {
-  tools.logTitle('Storing Backend metadata...');
-
-  return Promise.resolve()
-
-    .then(() => {
-      return innerPackageVersions.storeMetadata(pkg, pkgVersion, {}, duration, envTarget);
-    })
-
-    .then(() => {
-      if (isMaster) {
-        return storeBuildMetadata(pkg, pkgVersion, pkgMetadata, envTarget);
-      }
-    })
-
-    .then(() => {
-      if (isMaster) {
-        return storePipelineMetadata(pkgMetadata);
-      }
-    })
-
-    .then(() => {
-      tools.logSuccess();
-    })
-
-    .catch((e) => {
-      throw e;
-    });
-};
-
-const storePipelineMetadata = (pkgMetadata) => {
-  return Promise.resolve()
-    .then(() => {
-      tools.logInfo('Storing package pipeline metadata into root variables...');
-
-      const variablesFile = path.join(process.cwd(), 'variables');
-
-      let fileContent;
-      if (tools.isFileExists(variablesFile)) {
-        fileContent = tools.getFileContent(variablesFile);
-      }
-
-      if (pkgMetadata && pkgMetadata.issues) {
-        const issuesList = pkgMetadata.issues.join(',');
-        tools.logInfo(`Unique issues found for this version : ${issuesList}`);
-
-        tools.logInfo(`Write metadata on ${variablesFile}`);
-        if (!dryRun) {
-          fileContent += `export ISSUES_LIST=${issuesList}\n`;
-          tools.writeFileContent(variablesFile, fileContent);
-        }
-      } else {
-        tools.logInfo('No issues found for this release');
-      }
-    })
 
-    .catch((e) => {
-      console.log(e);
-    });
-};
-
-module.exports.storeMetadataAssets = (pkg, pkgCompositeDeps) => {
-  return Promise.resolve()
-    .then(() => {
-      const pkgMetadataFile = path.join(pkg.paths.dist, 'bundles', 'package-metadata.json');
-      const pkgMetadataRootFile = path.join(pkg.paths.root, 'package-metadata.json');
-
-      const pkgMetadata = {
-        dependencies: pkgCompositeDeps,
-      };
-
-      tools.logInfo(`Creating ${pkgMetadataFile}`);
-      console.log(pkgMetadata);
-
-      if (!dryRun) {
-        tools.writeJsonFileSync(pkgMetadataFile, pkgMetadata);
-        tools.writeJsonFileSync(pkgMetadataRootFile, pkgMetadata);
-      }
-    })
-
-    .catch((e) => {
-      throw e;
-    });
-};
 
 const getPackageVersions = (module.exports.getPackageVersions = (pkg) => {
   let npmOutput;

package/scripts/csdr/metadata/remote-history.js

@@ -0,0 +1,353 @@
+'use strict';
+
+// GLOBAL
+const path = require('path');
+
+// LOCAL
+const tools = require('../../utils/tools');
+const configUtils = require('../config/config-utils');
+const gitUtils = require('../../utils/git-utils');
+const pipelineUtils = require('../../utils/pipeline-utils');
+
+// INNER MODULES
+const innerRemotePackage = require('./remote-package');
+const innerCommit = require('./commit');
+
+// ARGS
+const { dryRun } = tools.getArgs();
+
+
+const getDiffsFromDeps = (fromDeps, toDeps) => {
+  const diffs = [];
+  Object.keys(fromDeps).forEach((k) => {
+    if ({}.hasOwnProperty.call(toDeps, k) && toDeps[k] !== fromDeps[k]) {
+      diffs.push({ package: k, old: fromDeps[k], new: toDeps[k] });
+    }
+  });
+
+  // remove snapshots as they are not stored in the package metadata
+  const cleanDiffs = diffs.filter((item) => {
+    return item.old.indexOf('-snapshot') === -1 &&
+           item.new.indexOf('-snapshot') === -1 &&
+           item.package.indexOf('@eui') === -1;
+  })
+
+  tools.logInfo(JSON.stringify(cleanDiffs, null, 2));
+
+  return cleanDiffs;
+}
+
+
+const getDiffBetweenBuildVersions = (pkg, fromBuildMetadata, toBuildMetadata) => {
+  tools.logInfo('Get differences between versions metadata');
+  tools.logInfo('FROM : ');
+  console.log(fromBuildMetadata);
+
+  tools.logInfo('TO : ');
+  console.log(toBuildMetadata);
+
+  if (!fromBuildMetadata) {
+    tools.logWarning('No previous metadata found for package...skipping');
+    return null;
+  }
+
+  return Promise.resolve()
+    .then(() => {
+      // get differences from old in previous major release to new currently generated
+      const diffs = getDiffsFromDeps(fromBuildMetadata.dependencies, toBuildMetadata.dependencies);
+
+      // Iterate over the
+      const mappedPkg = diffs.map((item) => {
+
+        tools.logInfo(`parsing item : ${item.package}`);
+
+        const pkgItem = configUtils.packages.getPackageByNpmPkg(item.package, true);
+        let pkgName;
+        if (pkgItem) {
+          pkgName = pkgItem.name;
+
+          if (pkgItem.child) {
+            pkgName = pkgItem.parentPkg;
+          }
+        }
+
+        return { npmPkg: item.package, name: pkgName, old: item.old, new: item.new };
+      })
+
+      return mappedPkg;
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+}
+
+
+
+const getDiffByEnvTarget = (pkg, envTarget) => {
+  tools.logInfo(`Getting differences for ${pkg.name} - ${envTarget}`);
+
+  return Promise.resolve()
+    .then(() => {
+      return innerRemotePackage.getPackageVersionsLatest(pkg);
+    })
+
+    .then((pkgVersions) => {
+      const envVersions = pkgVersions.filter((v) => {
+        return v.envTarget === envTarget
+      });
+
+      const fromVersion = envVersions[envVersions.length - 2];
+      const toVersion = envVersions[envVersions.length - 1];
+
+      if (!fromVersion || !toVersion) {
+        tools.logWarning('WARNING: could not find versions matching for generating diff report');
+        return null;
+      } else {
+        return getDiffBetweenBuildVersions(pkg, fromVersion, toVersion);
+      }
+
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+}
+
+
+
+
+// get differences of package versions between last(current) and previous major
+
+const generateDiffReport = module.exports.generateDiffReport = (pkg, envTarget, fromVersion, toVersion) => {
+  tools.logTitle('Generate packages differences report from previous major version...');
+
+  if (envTarget && pkg.build && pkg.build.envTargetActive) {
+
+    return Promise.resolve()
+    .then(() => {
+      return getDiffByEnvTarget(pkg, envTarget);
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+
+
+  } else {
+    return Promise.resolve()
+    .then(() => {
+      return innerRemotePackage.getPackageVersionsLatest(pkg);
+    })
+
+    .then((pkgVersions) => {
+      if (fromVersion && toVersion) {
+        const fromVersionMetadata = pkgVersions.filter((v) => {
+          return v.version === fromVersion;
+        })[0];
+
+        const toVersionMetadata = pkgVersions.filter((v) => {
+          return v.version === toVersion;
+        })[0];
+
+        return getDiffBetweenBuildVersions(pkg, fromVersionMetadata, toVersionMetadata);
+
+      } else {
+        return getDiffBetweenBuildVersions(pkg, pkgVersions[pkgVersions.length - 2], pkgVersions[pkgVersions.length - 1]);
+      }
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+  }
+}
+
+
+module.exports.generateDiffCommitsMetadataReport = (pkg, envTarget, fromVersion, toVersion) => {
+  tools.logTitle('Generating metadata diffs report and extract commits');
+
+  let diffsMetadata;
+
+  return Promise.resolve()
+    .then(() => {
+      return generateDiffReport(pkg, envTarget, fromVersion, toVersion);
+    })
+
+    .then((metadata) => {
+      // save for later export
+      diffsMetadata = metadata;
+
+      // processing single pkg from diffs metadata found
+      const getDiffsMetadataForPkg = (item) => {
+        const pkg = configUtils.packages.getPackage(item.name, true);
+
+        tools.logTitle(`Extract commits for pkg: ${pkg.name}`);
+        tools.logInfo('Diff metadata found: ');
+        console.log(item);
+
+        return Promise.resolve()
+          .then(() => {
+            return gitUtils.cloneAndCheckout(pkg.repository, pkg.paths.root, 'master');
+          })
+          .then(() => {
+            return gitUtils.getTags(pkg.paths.root);
+          })
+          .then((tags) => {
+            const oldTag = tags.find(t => t.indexOf(item.old) > -1);
+            const newTag = tags.find(t => t.indexOf(item.new) > -1);
+
+            tools.logInfo('Processing tags found:');
+            console.log('oldTag:', oldTag, ' - newTag:', newTag);
+
+            if (oldTag && newTag) {
+              return innerCommit.getMetadata(pkg, {fromTag: oldTag, toTag: newTag});
+
+            } else {
+              tools.logWarning('Either old tag or new tag cannot be found - commit results will be empty for this pkg');
+              return [];
+            }
+          })
+          .then((commitsMetadata) => {
+            return {
+              pkgName: pkg.name,
+              metadata: commitsMetadata
+            }
+          })
+          .catch((e) => {
+            tools.logWarning('ERROR found processing item :');
+            console.log(item);
+            console.log(e);
+          })
+      }
+
+      // check if current item pkg is valid for this remote (either rootPkg or part of linkedPackages list of remote)
+      const isPkgIncluded = (pkg, item) => {
+        let isIncluded = false;
+
+        if (pkg.skeletonConfig && pkg.skeletonConfig.rootNpmPkg === item.npmPkg) {
+          isIncluded = true;
+        } else {
+          if (pkg.linkedPackages) {
+            if (pkg.linkedPackages.filter(p => p === item.name).length !== 0) {
+              isIncluded = true;
+            }
+          }
+        }
+        return isIncluded;
+      }
+
+
+      // extracting commits for diffs metadata found / looping on promise
+      const getDiffsMetadata = async (pkg, diffs) => {
+        let metadata = [];
+        for (const item of diffs) {
+          if (isPkgIncluded(pkg, item)) {
+            tools.logInfo(`Package ${item.name} is included from parsing, processing...`);
+            const result = await getDiffsMetadataForPkg(item);
+            metadata.push(result);
+          } else {
+            tools.logInfo(`Package ${item.name} excluded from parsing`);
+          }
+        }
+        return metadata;
+      }
+
+      // main function call
+      return getDiffsMetadata(pkg, metadata);
+    })
+
+
+    .then((pkgMetadata) => {
+      tools.logInfo('Commits metadata found :');
+      console.log(JSON.stringify(pkgMetadata, null, 2));
+
+      tools.logInfo('Processing metadata');
+
+      let commitsMetadata = [], finalIssues = [];
+
+      pkgMetadata.forEach(pm => {
+        if (pm.metadata.commits && pm.metadata.commits.length !== 0) {
+          pm.metadata.commits.forEach(c => {
+            commitsMetadata.push({ pkgName: pm.pkgName, date: c.date, hash: c.hash, subject: c.originalSubject });
+          })
+        }
+        if (pm.metadata.issues && pm.metadata.issues.length !== 0) {
+          finalIssues = [ ...finalIssues, ...pm.metadata.issues ];
+        }
+      })
+
+      finalIssues = tools.removeArrayDuplicates(finalIssues);
+
+      return { commits: commitsMetadata, issues: finalIssues};
+    })
+
+    .then((commitsMetadata) => {
+      tools.logInfo('Final metadata and issues generated: ');
+      console.log(JSON.stringify(commitsMetadata, null, 2));
+
+      return {
+        diffsMetadata: diffsMetadata,
+        commitsMetadata: commitsMetadata
+      }
+    })
+
+    .catch((e) => {
+      throw e;
+    })
+}
+
+
+module.exports.storeMetadataHistory = (pkg, metadata, newVersion) => {
+  tools.logTitle('Storing pkg history metadata');
+
+  const DEVOPS_METADATA_PATH = configUtils.global.getConfigOptions().DEVOPS_METADATA_PATH;
+
+  let historyMetadata = {};
+
+  return Promise.resolve()
+    .then(() => {
+      const historyFile = path.join(DEVOPS_METADATA_PATH, pkg.devopsHistoryMetadataFile);
+
+      var versions = [];
+
+      if (tools.isFileExists(historyFile)) {
+        historyMetadata = require(historyFile);
+        versions = historyMetadata.versions;
+      }
+
+      versions.push({
+        name: newVersion,
+        metadata: metadata,
+      });
+
+      historyMetadata.versions = versions;
+      if (dryRun) {
+        tools.logWarning('DryRun...skipping');
+      } else {
+        tools.writeJsonFileSync(historyFile, historyMetadata);
+      }
+    })
+    .catch((e) => {
+      throw e;
+    })
+}
+
+
+module.exports.generatePipelineIssuesList = (issuesList) => {
+  return Promise.resolve()
+    // storing issuesList gathered into gitlab variables for post CI processing
+    .then(() => {
+      tools.logInfo('Extracting Jira issues list to gitlab variables');
+
+      if (dryRun) {
+        tools.logWarning('DryRun...skipping');
+        console.log(`export ISSUES_LIST=${issuesList.join(',')}\n`);
+      } else {
+        return pipelineUtils.setVariables(process.cwd(), `export ISSUES_LIST=${issuesList.join(',')}\n`);
+      }
+    })
+    .catch((e) => {
+      throw e;
+    })
+}