@cyclonedx/cdxgen 9.8.9 → 9.9.0

package/display.js

@@ -1,3 +1,4 @@
+import { existsSync, readFileSync } from "fs";
 import { createStream, table } from "table";
 
 // https://github.com/yangshun/tree-node-cli/blob/master/src/index.js
@@ -277,3 +278,36 @@ const recursePrint = (depMap, subtree, level, shownList, treeGraphics) => {
     }
   }
 };
+
+export const printReachables = (sliceArtefacts) => {
+  const reachablesSlicesFile = sliceArtefacts.reachablesSlicesFile;
+  if (!existsSync(reachablesSlicesFile)) {
+    return;
+  }
+  const purlCounts = {};
+  const reachablesSlices = JSON.parse(
+    readFileSync(reachablesSlicesFile, "utf-8")
+  );
+  for (const areachable of reachablesSlices.reachables || []) {
+    const purls = areachable.purls || [];
+    for (const apurl of purls) {
+      purlCounts[apurl] = (purlCounts[apurl] || 0) + 1;
+    }
+  }
+  const sortedPurls = Object.fromEntries(
+    Object.entries(purlCounts).sort(([, a], [, b]) => b - a)
+  );
+  const data = [["Package URL", "Reachable Flows"]];
+  for (const apurl of Object.keys(sortedPurls)) {
+    data.push([apurl, "" + sortedPurls[apurl]]);
+  }
+  const config = {
+    header: {
+      alignment: "center",
+      content: "Reachable Components\nGenerated with \u2665 by cdxgen"
+    }
+  };
+  if (data.length > 1) {
+    console.log(table(data, config));
+  }
+};

package/docker.js

@@ -323,6 +323,9 @@ export const parseImageName = (fullImageName) => {
       fullImageName = fullImageName.replace(":" + nameObj.tag, "");
     }
   }
+  if (fullImageName && fullImageName.startsWith("library/")) {
+    fullImageName = fullImageName.replace("library/", "");
+  }
   // The left over string is the repo name
   nameObj.repo = fullImageName;
   return nameObj;
@@ -333,7 +336,9 @@ export const parseImageName = (fullImageName) => {
  */
 export const getImage = async (fullImageName) => {
   let localData = undefined;
+  let pullData = undefined;
   const { repo, tag, digest } = parseImageName(fullImageName);
+  let repoWithTag = `${repo}:${tag !== "" ? tag : ":latest"}`;
   // Fetch only the latest tag if none is specified
   if (tag === "" && digest === "") {
     fullImageName = fullImageName + ":latest";
@@ -379,6 +384,14 @@ export const getImage = async (fullImageName) => {
       }
     }
   }
+  try {
+    localData = await makeRequest(`images/${repoWithTag}/json`);
+    if (localData) {
+      return localData;
+    }
+  } catch (err) {
+    // ignore
+  }
   try {
     localData = await makeRequest(`images/${repo}/json`);
   } catch (err) {
@@ -397,7 +410,7 @@ export const getImage = async (fullImageName) => {
     }
     // If the data is not available locally
     try {
-      const pullData = await makeRequest(
+      pullData = await makeRequest(
         `images/create?fromImage=${fullImageName}`,
         "POST"
       );
@@ -415,15 +428,42 @@ export const getImage = async (fullImageName) => {
         return undefined;
       }
     } catch (err) {
-      // continue regardless of error
+      try {
+        if (DEBUG_MODE) {
+          console.log(`Re-trying the pull with the name ${repoWithTag}.`);
+        }
+        pullData = await makeRequest(
+          `images/create?fromImage=${repoWithTag}`,
+          "POST"
+        );
+      } catch (err) {
+        // continue regardless of error
+      }
     }
     try {
       if (DEBUG_MODE) {
-        console.log(`Trying with ${repo}`);
+        console.log(`Trying with ${repoWithTag}`);
+      }
+      localData = await makeRequest(`images/${repoWithTag}/json`);
+      if (localData) {
+        return localData;
       }
-      localData = await makeRequest(`images/${repo}/json`);
     } catch (err) {
       try {
+        if (DEBUG_MODE) {
+          console.log(`Trying with ${repo}`);
+        }
+        localData = await makeRequest(`images/${repo}/json`);
+        if (localData) {
+          return localData;
+        }
+      } catch (err) {
+        // continue regardless of error
+      }
+      try {
+        if (DEBUG_MODE) {
+          console.log(`Trying with ${fullImageName}`);
+        }
         localData = await makeRequest(`images/${fullImageName}/json`);
       } catch (err) {
         // continue regardless of error
@@ -701,7 +741,26 @@ export const exportImage = async (fullImageName) => {
         })
       );
     } catch (err) {
-      console.error(err);
+      if (localData && localData.Id) {
+        console.log(`Retrying with ${localData.Id}`);
+        try {
+          await stream.pipeline(
+            client.stream(`images/${localData.Id}/get`),
+            x({
+              sync: true,
+              preserveOwner: false,
+              noMtime: true,
+              noChmod: true,
+              strict: true,
+              C: tempDir,
+              portable: true,
+              onwarn: () => {}
+            })
+          );
+        } catch (err) {
+          console.log(err);
+        }
+      }
     }
   }
   // Continue with extracting the layers

package/evinser.js

@@ -71,7 +71,7 @@ export const prepareDB = async (options) => {
     }
   }
   for (const purl of Object.keys(purlsToSlice)) {
-    await createAndStoreSlice(purl, purlsJars, Usages);
+    await createAndStoreSlice(purl, purlsJars, Usages, options);
   }
   return { sequelize, Namespaces, Usages, DataFlows };
 };
@@ -148,8 +148,13 @@ export const catalogGradleDeps = async (dirPath, purlsJars, Namespaces) => {
   );
 };
 
-export const createAndStoreSlice = async (purl, purlsJars, Usages) => {
-  const retMap = createSlice(purl, purlsJars[purl], "usages");
+export const createAndStoreSlice = async (
+  purl,
+  purlsJars,
+  Usages,
+  options = {}
+) => {
+  const retMap = createSlice(purl, purlsJars[purl], "usages", options);
   let sliceData = undefined;
   if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
     sliceData = await Usages.findOrCreate({
@@ -166,7 +171,12 @@ export const createAndStoreSlice = async (purl, purlsJars, Usages) => {
   return sliceData;
 };
 
-export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
+export const createSlice = (
+  purlOrLanguage,
+  filePath,
+  sliceType = "usages",
+  options = {}
+) => {
   if (!filePath) {
     return;
   }
@@ -177,9 +187,18 @@ export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
   if (!language) {
     return undefined;
   }
-  const tempDir = fs.mkdtempSync(path.join(tmpdir(), `atom-${sliceType}-`));
-  const atomFile = path.join(tempDir, "app.atom");
-  const slicesFile = path.join(tempDir, `${sliceType}.slices.json`);
+  let sliceOutputDir = fs.mkdtempSync(
+    path.join(tmpdir(), `atom-${sliceType}-`)
+  );
+  if (options && options.output) {
+    sliceOutputDir =
+      fs.existsSync(options.output) &&
+      fs.lstatSync(options.output).isDirectory()
+        ? path.basename(options.output)
+        : path.dirname(options.output);
+  }
+  const atomFile = path.join(sliceOutputDir, "app.atom");
+  const slicesFile = path.join(sliceOutputDir, `${sliceType}.slices.json`);
   const args = [
     sliceType,
     "-l",
@@ -204,7 +223,7 @@ export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
     );
   }
   return {
-    tempDir,
+    tempDir: sliceOutputDir,
     slicesFile,
     atomFile
   };
@@ -260,8 +279,10 @@ export const analyzeProject = async (dbObjMap, options) => {
   const language = options.language;
   let usageSlice = undefined;
   let dataFlowSlice = undefined;
+  let reachablesSlice = undefined;
   let usagesSlicesFile = undefined;
   let dataFlowSlicesFile = undefined;
+  let reachablesSlicesFile = undefined;
   let dataFlowFrames = {};
   let servicesMap = {};
   let retMap = {};
@@ -278,7 +299,7 @@ export const analyzeProject = async (dbObjMap, options) => {
     usagesSlicesFile = options.usagesSlicesFile;
   } else {
     // Generate our own slices
-    retMap = createSlice(language, dirPath, "usages");
+    retMap = createSlice(language, dirPath, "usages", options);
     if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
       usageSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
       usagesSlicesFile = retMap.slicesFile;
@@ -310,7 +331,7 @@ export const analyzeProject = async (dbObjMap, options) => {
         fs.readFileSync(options.dataFlowSlicesFile, "utf-8")
       );
     } else {
-      retMap = createSlice(language, dirPath, "data-flow");
+      retMap = createSlice(language, dirPath, "data-flow", options);
       if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
         dataFlowSlicesFile = retMap.slicesFile;
         dataFlowSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
@@ -330,10 +351,36 @@ export const analyzeProject = async (dbObjMap, options) => {
       purlImportsMap
     );
   }
+  if (options.withReachables) {
+    if (
+      options.reachablesSlicesFile &&
+      fs.existsSync(options.reachablesSlicesFile)
+    ) {
+      reachablesSlicesFile = options.reachablesSlicesFile;
+      reachablesSlice = JSON.parse(
+        fs.readFileSync(options.reachablesSlicesFile, "utf-8")
+      );
+    } else {
+      retMap = createSlice(language, dirPath, "reachables", options);
+      if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
+        reachablesSlicesFile = retMap.slicesFile;
+        reachablesSlice = JSON.parse(
+          fs.readFileSync(retMap.slicesFile, "utf-8")
+        );
+        console.log(
+          `To speed up this step, cache ${reachablesSlicesFile} and invoke evinse with the --reachables-slices-file argument.`
+        );
+      }
+    }
+  }
+  if (reachablesSlice && Object.keys(reachablesSlice).length) {
+    dataFlowFrames = await collectReachableFrames(language, reachablesSlice);
+  }
   return {
     atomFile: retMap.atomFile,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames,
@@ -752,6 +799,7 @@ export const createEvinseFile = (sliceArtefacts, options) => {
     tempDir,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames
@@ -830,6 +878,14 @@ export const createEvinseFile = (sliceArtefacts, options) => {
         text: fs.readFileSync(dataFlowSlicesFile, "utf8")
       });
     }
+    if (reachablesSlicesFile && fs.existsSync(reachablesSlicesFile)) {
+      bomJson.annotations.push({
+        subjects: [bomJson.serialNumber],
+        annotator: { component: bomJson.metadata.tools.components[0] },
+        timestamp: new Date().toISOString(),
+        text: fs.readFileSync(reachablesSlicesFile, "utf8")
+      });
+    }
   }
   // Increment the version
   bomJson.version = (bomJson.version || 1) + 1;
@@ -973,6 +1029,49 @@ export const collectDataFlowFrames = async (
   return dfFrames;
 };
 
+/**
+ * Method to convert reachable slice into usable callstack frames
+ * Implemented based on the logic proposed here - https://github.com/AppThreat/atom/blob/main/specification/docs/slices.md#data-flow-slice
+ *
+ * @param {string} language Application language
+ * @param {object} reachablesSlice Reachables slice object from atom
+ */
+export const collectReachableFrames = async (language, reachablesSlice) => {
+  const reachableNodes = reachablesSlice?.reachables || [];
+  // purl key and an array of frames array
+  // CycloneDX 1.5 currently accepts only 1 frame as evidence
+  // so this method is more future-proof
+  const dfFrames = {};
+  for (const anode of reachableNodes) {
+    let aframe = [];
+    let referredPurls = new Set(anode.purls || []);
+    for (const fnode of anode.flows) {
+      if (!fnode.parentFileName || fnode.parentFileName === "<unknown>") {
+        continue;
+      }
+      aframe.push({
+        package: fnode.parentPackageName,
+        module: fnode.parentClassName || "",
+        function: fnode.parentMethodName || "",
+        line: fnode.lineNumber || undefined,
+        column: fnode.columnNumber || undefined,
+        fullFilename: fnode.parentFileName || ""
+      });
+    }
+    referredPurls = Array.from(referredPurls);
+    if (referredPurls.length) {
+      for (const apurl of referredPurls) {
+        if (!dfFrames[apurl]) {
+          dfFrames[apurl] = [];
+        }
+        // Store this frame as an evidence for this purl
+        dfFrames[apurl].push(aframe);
+      }
+    }
+  }
+  return dfFrames;
+};
+
 /**
  * Method to pick a callstack frame as an evidence. This method is required since CycloneDX 1.5 accepts only a single frame as evidence.
  *