@cyclonedx/cdxgen 9.8.10 → 9.9.1

package/evinser.js

@@ -7,7 +7,7 @@ import {
   collectMvnDependencies
 } from "./utils.js";
 import { tmpdir } from "node:os";
-import path from "node:path";
+import path, { basename } from "node:path";
 import fs from "node:fs";
 import * as db from "./db.js";
 import { PackageURL } from "packageurl-js";
@@ -22,10 +22,24 @@ const typePurlsCache = {};
  * @param {object} Command line options
  */
 export const prepareDB = async (options) => {
+  if (!options.dbPath.includes("memory") && !fs.existsSync(options.dbPath)) {
+    try {
+      fs.mkdirSync(options.dbPath, { recursive: true });
+    } catch (e) {
+      // ignore
+    }
+  }
   const dirPath = options._[0] || ".";
   const bomJsonFile = options.input;
   if (!fs.existsSync(bomJsonFile)) {
-    console.log("Bom file doesn't exist");
+    console.log(
+      "Bom file doesn't exist. Check if cdxgen was invoked with the correct type argument."
+    );
+    if (!process.env.CDXGEN_DEBUG_MODE) {
+      console.log(
+        "Set the environment variable CDXGEN_DEBUG_MODE to debug to troubleshoot the issue further."
+      );
+    }
     return;
   }
   const bomJson = JSON.parse(fs.readFileSync(bomJsonFile, "utf8"));
@@ -54,8 +68,6 @@ export const prepareDB = async (options) => {
     if ((!usagesSlice && !namespaceSlice) || options.force) {
       if (comp.purl.startsWith("pkg:maven")) {
         hasMavenPkgs = true;
-      } else if (isSlicingRequired(comp.purl)) {
-        purlsToSlice[comp.purl] = true;
       }
     }
   }
@@ -71,7 +83,7 @@ export const prepareDB = async (options) => {
     }
   }
   for (const purl of Object.keys(purlsToSlice)) {
-    await createAndStoreSlice(purl, purlsJars, Usages);
+    await createAndStoreSlice(purl, purlsJars, Usages, options);
   }
   return { sequelize, Namespaces, Usages, DataFlows };
 };
@@ -148,8 +160,13 @@ export const catalogGradleDeps = async (dirPath, purlsJars, Namespaces) => {
   );
 };
 
-export const createAndStoreSlice = async (purl, purlsJars, Usages) => {
-  const retMap = createSlice(purl, purlsJars[purl], "usages");
+export const createAndStoreSlice = async (
+  purl,
+  purlsJars,
+  Usages,
+  options = {}
+) => {
+  const retMap = createSlice(purl, purlsJars[purl], "usages", options);
   let sliceData = undefined;
   if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
     sliceData = await Usages.findOrCreate({
@@ -166,7 +183,12 @@ export const createAndStoreSlice = async (purl, purlsJars, Usages) => {
   return sliceData;
 };
 
-export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
+export const createSlice = (
+  purlOrLanguage,
+  filePath,
+  sliceType = "usages",
+  options = {}
+) => {
   if (!filePath) {
     return;
   }
@@ -177,9 +199,18 @@ export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
   if (!language) {
     return undefined;
   }
-  const tempDir = fs.mkdtempSync(path.join(tmpdir(), `atom-${sliceType}-`));
-  const atomFile = path.join(tempDir, "app.atom");
-  const slicesFile = path.join(tempDir, `${sliceType}.slices.json`);
+  let sliceOutputDir = fs.mkdtempSync(
+    path.join(tmpdir(), `atom-${sliceType}-`)
+  );
+  if (options && options.output) {
+    sliceOutputDir =
+      fs.existsSync(options.output) &&
+      fs.lstatSync(options.output).isDirectory()
+        ? path.basename(options.output)
+        : path.dirname(options.output);
+  }
+  const atomFile = path.join(sliceOutputDir, "app.atom");
+  const slicesFile = path.join(sliceOutputDir, `${sliceType}.slices.json`);
   const args = [
     sliceType,
     "-l",
@@ -204,7 +235,7 @@ export const createSlice = (purlOrLanguage, filePath, sliceType = "usages") => {
     );
   }
   return {
-    tempDir,
+    tempDir: sliceOutputDir,
     slicesFile,
     atomFile
   };
@@ -231,17 +262,19 @@ export const initFromSbom = (components) => {
   const purlLocationMap = {};
   const purlImportsMap = {};
   for (const comp of components) {
-    if (!comp || !comp.evidence || !comp.evidence.occurrences) {
+    if (!comp || !comp.evidence) {
       continue;
     }
-    purlLocationMap[comp.purl] = new Set(
-      comp.evidence.occurrences.map((v) => v.location)
-    );
     (comp.properties || [])
       .filter((v) => v.name === "ImportedModules")
       .forEach((v) => {
         purlImportsMap[comp.purl] = (v.value || "").split(",");
       });
+    if (comp.evidence.occurrences) {
+      purlLocationMap[comp.purl] = new Set(
+        comp.evidence.occurrences.map((v) => v.location)
+      );
+    }
   }
   return {
     purlLocationMap,
@@ -260,8 +293,10 @@ export const analyzeProject = async (dbObjMap, options) => {
   const language = options.language;
   let usageSlice = undefined;
   let dataFlowSlice = undefined;
+  let reachablesSlice = undefined;
   let usagesSlicesFile = undefined;
   let dataFlowSlicesFile = undefined;
+  let reachablesSlicesFile = undefined;
   let dataFlowFrames = {};
   let servicesMap = {};
   let retMap = {};
@@ -278,7 +313,7 @@ export const analyzeProject = async (dbObjMap, options) => {
     usagesSlicesFile = options.usagesSlicesFile;
   } else {
     // Generate our own slices
-    retMap = createSlice(language, dirPath, "usages");
+    retMap = createSlice(language, dirPath, "usages", options);
     if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
       usageSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
       usagesSlicesFile = retMap.slicesFile;
@@ -310,7 +345,7 @@ export const analyzeProject = async (dbObjMap, options) => {
         fs.readFileSync(options.dataFlowSlicesFile, "utf-8")
       );
     } else {
-      retMap = createSlice(language, dirPath, "data-flow");
+      retMap = createSlice(language, dirPath, "data-flow", options);
       if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
         dataFlowSlicesFile = retMap.slicesFile;
         dataFlowSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
@@ -330,10 +365,36 @@ export const analyzeProject = async (dbObjMap, options) => {
       purlImportsMap
     );
   }
+  if (options.withReachables) {
+    if (
+      options.reachablesSlicesFile &&
+      fs.existsSync(options.reachablesSlicesFile)
+    ) {
+      reachablesSlicesFile = options.reachablesSlicesFile;
+      reachablesSlice = JSON.parse(
+        fs.readFileSync(options.reachablesSlicesFile, "utf-8")
+      );
+    } else {
+      retMap = createSlice(language, dirPath, "reachables", options);
+      if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
+        reachablesSlicesFile = retMap.slicesFile;
+        reachablesSlice = JSON.parse(
+          fs.readFileSync(retMap.slicesFile, "utf-8")
+        );
+        console.log(
+          `To speed up this step, cache ${reachablesSlicesFile} and invoke evinse with the --reachables-slices-file argument.`
+        );
+      }
+    }
+  }
+  if (reachablesSlice && Object.keys(reachablesSlice).length) {
+    dataFlowFrames = await collectReachableFrames(language, reachablesSlice);
+  }
   return {
     atomFile: retMap.atomFile,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames,
@@ -365,7 +426,8 @@ export const parseObjectSlices = async (
     if (
       !slice.fileName ||
       !slice.fileName.trim().length ||
-      slice.fileName === "<empty>"
+      slice.fileName === "<empty>" ||
+      slice.fileName === "<unknown>"
     ) {
       continue;
     }
@@ -379,6 +441,7 @@ export const parseObjectSlices = async (
     );
     detectServicesFromUsages(language, slice, servicesMap);
   }
+  detectServicesFromUDT(language, usageSlice.userDefinedTypes, servicesMap);
   return {
     purlLocationMap,
     servicesMap,
@@ -428,10 +491,13 @@ export const parseSliceUsages = async (
         atype[0] !== false &&
         !isFilterableType(language, userDefinedTypesMap, atype[1])
       ) {
-        if (!atype[1].includes("(")) {
+        if (!atype[1].includes("(") && !atype[1].includes(".py")) {
           typesToLookup.add(atype[1]);
           // Javascript calls can be resolved to a precise line number only from the call nodes
-          if (language == "javascript" && ausageLine) {
+          if (
+            ["javascript", "js", "ts", "typescript"].includes(language) &&
+            ausageLine
+          ) {
             if (atype[1].includes(":")) {
               typesToLookup.add(atype[1].split("::")[0].replace(/:/g, "/"));
             }
@@ -456,7 +522,10 @@ export const parseSliceUsages = async (
       if (
         !isFilterableType(language, userDefinedTypesMap, acall?.resolvedMethod)
       ) {
-        if (!acall?.resolvedMethod.includes("(")) {
+        if (
+          !acall?.resolvedMethod.includes("(") &&
+          !acall?.resolvedMethod.includes(".py")
+        ) {
           typesToLookup.add(acall?.resolvedMethod);
           // Javascript calls can be resolved to a precise line number only from the call nodes
           if (acall.lineNumber) {
@@ -484,7 +553,7 @@ export const parseSliceUsages = async (
       }
       for (const aparamType of acall?.paramTypes || []) {
         if (!isFilterableType(language, userDefinedTypesMap, aparamType)) {
-          if (!aparamType.includes("(")) {
+          if (!aparamType.includes("(") && !aparamType.includes(".py")) {
             typesToLookup.add(aparamType);
             if (acall.lineNumber) {
               if (aparamType.includes(":")) {
@@ -533,16 +602,17 @@ export const parseSliceUsages = async (
       }
     } else {
       // Check the namespaces db
-      const nsHits =
-        typePurlsCache[atype] ||
-        (await dbObjMap.Namespaces.findAll({
+      let nsHits = typePurlsCache[atype];
+      if (["java", "jar"].includes(language)) {
+        nsHits = await dbObjMap.Namespaces.findAll({
           attributes: ["purl"],
           where: {
             data: {
               [Op.like]: `%${atype}%`
             }
           }
-        }));
+        });
+      }
       if (nsHits && nsHits.length) {
         for (const ns of nsHits) {
           if (!purlLocationMap[ns.purl]) {
@@ -565,16 +635,21 @@ export const isFilterableType = (
 ) => {
   if (
     !typeFullName ||
-    ["ANY", "UNKNOWN", "VOID"].includes(typeFullName.toUpperCase())
+    ["ANY", "UNKNOWN", "VOID", "IMPORT"].includes(typeFullName.toUpperCase())
   ) {
     return true;
   }
-  if (
-    typeFullName.startsWith("<operator") ||
-    typeFullName.startsWith("<unresolved") ||
-    typeFullName.startsWith("<unknownFullName")
-  ) {
-    return true;
+  for (const ab of [
+    "<operator",
+    "<unresolved",
+    "<unknownFullName",
+    "__builtin",
+    "LAMBDA",
+    "../"
+  ]) {
+    if (typeFullName.startsWith(ab)) {
+      return true;
+    }
   }
   if (language && ["java", "jar"].includes(language)) {
     if (
@@ -590,7 +665,7 @@ export const isFilterableType = (
       return true;
     }
   }
-  if (language === "javascript") {
+  if (["javascript", "js", "ts", "typescript"].includes(language)) {
     if (
       typeFullName.includes(".js") ||
       typeFullName.includes("=>") ||
@@ -598,13 +673,20 @@ export const isFilterableType = (
       typeFullName.startsWith("{ ") ||
       typeFullName.startsWith("JSON") ||
       typeFullName.startsWith("void:") ||
-      typeFullName.startsWith("LAMBDA") ||
-      typeFullName.startsWith("../") ||
       typeFullName.startsWith("node:")
     ) {
       return true;
     }
   }
+  if (["python", "py"].includes(language)) {
+    if (
+      typeFullName.startsWith("tmp") ||
+      typeFullName.startsWith("self.") ||
+      typeFullName.startsWith("_")
+    ) {
+      return true;
+    }
+  }
   if (userDefinedTypesMap[typeFullName]) {
     return true;
   }
@@ -668,6 +750,61 @@ export const detectServicesFromUsages = (language, slice, servicesMap = {}) => {
   }
 };
 
+/**
+ * Method to detect services from user defined types in the usage slice
+ *
+ * @param {string} language Application language
+ * @param {array} userDefinedTypes User defined types
+ * @param {object} servicesMap Existing service map
+ */
+export const detectServicesFromUDT = (
+  language,
+  userDefinedTypes,
+  servicesMap
+) => {
+  if (
+    ["python", "py"].includes(language) &&
+    userDefinedTypes &&
+    userDefinedTypes.length
+  ) {
+    for (const audt of userDefinedTypes) {
+      if (
+        audt.name.includes("route") ||
+        audt.name.includes("path") ||
+        audt.name.includes("url")
+      ) {
+        const fields = audt.fields || [];
+        if (
+          fields.length &&
+          fields[0] &&
+          fields[0].name &&
+          fields[0].name.length > 1
+        ) {
+          const endpoints = extractEndpoints(language, fields[0].name);
+          let serviceName = "service";
+          if (audt.fileName) {
+            serviceName = `${basename(
+              audt.fileName.replace(".py", "")
+            )}-service`;
+          }
+          if (!servicesMap[serviceName]) {
+            servicesMap[serviceName] = {
+              endpoints: new Set(),
+              authenticated: false,
+              xTrustBoundary: undefined
+            };
+          }
+          if (endpoints) {
+            for (const endpoint of endpoints) {
+              servicesMap[serviceName].endpoints.add(endpoint);
+            }
+          }
+        }
+      }
+    }
+  }
+};
+
 export const constructServiceName = (language, slice) => {
   let serviceName = "service";
   if (slice?.fullName) {
@@ -706,7 +843,10 @@ export const extractEndpoints = (language, code) => {
           );
       }
       break;
+    case "js":
+    case "ts":
     case "javascript":
+    case "typescript":
       if (code.includes("app.") || code.includes("route")) {
         const matches = code.match(/['"](.*?)['"]/gi) || [];
         endpoints = matches
@@ -722,24 +862,18 @@ export const extractEndpoints = (language, code) => {
           );
       }
       break;
+    case "py":
+    case "python":
+      endpoints = (code.match(/['"](.*?)['"]/gi) || [])
+        .map((v) => v.replace(/["']/g, "").replace("\n", ""))
+        .filter((v) => v.length > 2);
+      break;
     default:
       break;
   }
   return endpoints;
 };
 
-/**
- * Function to determine if slicing is required for the given language's dependencies.
- * For performance reasons, we make java operate only with namespaces
- *
- * @param {string} purl
- * @returns
- */
-export const isSlicingRequired = (purl) => {
-  const language = purlToLanguage(purl);
-  return ["python"].includes(language);
-};
-
 /**
  * Method to create the SBOM with evidence file called evinse file.
  *
@@ -752,6 +886,7 @@ export const createEvinseFile = (sliceArtefacts, options) => {
     tempDir,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames
@@ -830,6 +965,14 @@ export const createEvinseFile = (sliceArtefacts, options) => {
         text: fs.readFileSync(dataFlowSlicesFile, "utf8")
       });
     }
+    if (reachablesSlicesFile && fs.existsSync(reachablesSlicesFile)) {
+      bomJson.annotations.push({
+        subjects: [bomJson.serialNumber],
+        annotator: { component: bomJson.metadata.tools.components[0] },
+        timestamp: new Date().toISOString(),
+        text: fs.readFileSync(reachablesSlicesFile, "utf8")
+      });
+    }
   }
   // Increment the version
   bomJson.version = (bomJson.version || 1) + 1;
@@ -889,7 +1032,10 @@ export const collectDataFlowFrames = async (
         continue;
       }
       let typeFullName = theNode.typeFullName;
-      if (language === "javascript" && typeFullName == "ANY") {
+      if (
+        ["javascript", "js", "ts", "typescript"].includes(language) &&
+        typeFullName == "ANY"
+      ) {
         if (
           theNode.code &&
           (theNode.code.startsWith("new ") ||
@@ -915,16 +1061,17 @@ export const collectDataFlowFrames = async (
           }
         } else {
           // Check the namespaces db
-          const nsHits =
-            typePurlsCache[typeFullName] ||
-            (await dbObjMap.Namespaces.findAll({
+          let nsHits = typePurlsCache[typeFullName];
+          if (["java", "jar"].includes(language)) {
+            nsHits = await dbObjMap.Namespaces.findAll({
               attributes: ["purl"],
               where: {
                 data: {
                   [Op.like]: `%${typeFullName}%`
                 }
               }
-            }));
+            });
+          }
           if (nsHits && nsHits.length) {
             for (const ns of nsHits) {
               referredPurls.add(ns.purl);
@@ -973,6 +1120,49 @@ export const collectDataFlowFrames = async (
   return dfFrames;
 };
 
+/**
+ * Method to convert reachable slice into usable callstack frames
+ * Implemented based on the logic proposed here - https://github.com/AppThreat/atom/blob/main/specification/docs/slices.md#data-flow-slice
+ *
+ * @param {string} language Application language
+ * @param {object} reachablesSlice Reachables slice object from atom
+ */
+export const collectReachableFrames = async (language, reachablesSlice) => {
+  const reachableNodes = reachablesSlice?.reachables || [];
+  // purl key and an array of frames array
+  // CycloneDX 1.5 currently accepts only 1 frame as evidence
+  // so this method is more future-proof
+  const dfFrames = {};
+  for (const anode of reachableNodes) {
+    let aframe = [];
+    let referredPurls = new Set(anode.purls || []);
+    for (const fnode of anode.flows) {
+      if (!fnode.parentFileName || fnode.parentFileName === "<unknown>") {
+        continue;
+      }
+      aframe.push({
+        package: fnode.parentPackageName,
+        module: fnode.parentClassName || "",
+        function: fnode.parentMethodName || "",
+        line: fnode.lineNumber || undefined,
+        column: fnode.columnNumber || undefined,
+        fullFilename: fnode.parentFileName || ""
+      });
+    }
+    referredPurls = Array.from(referredPurls);
+    if (referredPurls.length) {
+      for (const apurl of referredPurls) {
+        if (!dfFrames[apurl]) {
+          dfFrames[apurl] = [];
+        }
+        // Store this frame as an evidence for this purl
+        dfFrames[apurl].push(aframe);
+      }
+    }
+  }
+  return dfFrames;
+};
+
 /**
  * Method to pick a callstack frame as an evidence. This method is required since CycloneDX 1.5 accepts only a single frame as evidence.
  *
@@ -1000,7 +1190,7 @@ export const getClassTypeFromSignature = (language, typeFullName) => {
     const tmpA = typeFullName.split(".");
     tmpA.pop();
     typeFullName = tmpA.join(".");
-  } else if (language === "javascript") {
+  } else if (["javascript", "js", "ts", "typescript"].includes(language)) {
     typeFullName = typeFullName.replace("new: ", "").replace("await ", "");
     if (typeFullName.includes(":")) {
       const tmpA = typeFullName.split("::")[0].replace(/:/g, "/").split("/");
@@ -1009,6 +1199,15 @@ export const getClassTypeFromSignature = (language, typeFullName) => {
       }
       typeFullName = tmpA.join("/");
     }
+  } else if (["python", "py"].includes(language)) {
+    typeFullName = typeFullName
+      .replace(".py:<module>", "")
+      .replace(/\//g, ".")
+      .replace(".<metaClassCallHandler>", "")
+      .replace(".<fakeNew>", "")
+      .replace(".<body>", "")
+      .replace(".__iter__", "")
+      .replace(".__init__", "");
   }
   if (
     typeFullName.startsWith("<unresolved") ||