@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/inventory/evidenceUtils.js

@@ -1,4 +1,3 @@
-import { readFileSync } from "node:fs";
 import { platform } from "node:os";
 import { extname } from "node:path";
 
@@ -6,12 +5,17 @@ import { build } from "@cdxgen/cdx-purl";
 
 import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
-import { addDosaiReachabilityProperties } from "./dosai.js";
+import {
+  addDosaiReachabilityProperties,
+  readDosaiMethodsReport,
+} from "./dosai.js";
 import {
   addDosaiSetValue,
   buildDosaiPurlAliasMap,
+  dosaiCallSiteLocation,
   dosaiSourceLocation,
   dosaiSourceLocationFromNode,
+  pickDosaiComponentByLocation,
   resolveDosaiComponentPurl,
 } from "./dosaiParsers.js";
 import { createGtfoBinsPropertiesFromRow } from "./gtfobins.js";
@@ -238,27 +242,47 @@ function addDotnetIdentityMethod(apkg, value) {
  * Enrich .NET package components with occurrence evidence and imported module/method
  * information from a dosai dependency slices file.
  *
- * Builds a mapping of DLL filenames to purls using the `internal:PackageFiles` property of each
- * package, then reads the slices file to add occurrence locations, imported modules,
- * called methods, and assembly version information where available.
+ * Each dosai record names its package by purl, resolved in the project of the
+ * record's file, and is attached to the component of that version (see
+ * {@link resolveDosaiComponentPurl}). The DLL files of each package
+ * (`internal:PackageFiles`) only fill in for a record whose purl names no
+ * component, and only when the DLL belongs to one component or the record's
+ * location says which: two versions of a package ship the same DLL names.
+ *
+ * The fallback is needed because dosai names a `System.*` assembly it cannot
+ * map to a package with a framework purl (`pkg:nuget/System.Runtime`), as it
+ * does whenever the tree has lock files but no restore output: the DLL of
+ * System.Data.SQLite.Core is still the package's. A DLL that only a package's
+ * build-time folders carry is not listed in `internal:PackageFiles`, so it
+ * credits no package with the project's framework calls (issue 4441).
  *
  * @param {Object[]} pkgList Array of .NET package component objects to enrich
  * @param {string} slicesFile Path to the dosai dependency slices JSON file
+ * @param {Object} [options] Options
+ * @param {string} [options.srcPath] Directory dosai analyzed, which its relative locations
+ *        and the components' relative manifest paths are relative to
  * @returns {Object[]} The enriched package list (same array, mutated in place)
  */
-export function addEvidenceForDotnet(pkgList, slicesFile) {
+export function addEvidenceForDotnet(pkgList, slicesFile, options = {}) {
   // We need two datastructures.
   // dll to purl mapping from the pkgList
   // purl to occurrences list using the slicesFile
   if (!slicesFile || !safeExistsSync(slicesFile)) {
     return pkgList;
   }
-  const pkgFilePurlMap = {};
+  const pkgFilePurls = {};
   const purlLocationMap = {};
   const purlModulesMap = {};
   const purlMethodsMap = {};
   const purlReachabilityMap = {};
-  const purlAliasMap = buildDosaiPurlAliasMap(pkgList);
+  const purlAliasMap = buildDosaiPurlAliasMap(pkgList, options);
+  const componentPurls = new Set(pkgList.map((apkg) => apkg.purl));
+  // A dosai purl that names a component of this list, resolved in the
+  // record's project.
+  const componentPurlFor = (purl, location) => {
+    const resolved = resolveDosaiComponentPurl(purl, purlAliasMap, location);
+    return componentPurls.has(resolved) ? resolved : undefined;
+  };
   for (const apkg of pkgList) {
     if (apkg.properties && Array.isArray(apkg.properties)) {
       apkg.properties
@@ -268,17 +292,35 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
             const tmpA = aprop.value.split(", ");
             if (tmpA?.length) {
               tmpA.forEach((dllFile) => {
-                pkgFilePurlMap[dllFile] = apkg.purl;
+                pkgFilePurls[dllFile] ??= new Set();
+                pkgFilePurls[dllFile].add(apkg.purl);
               });
             }
           }
         });
     }
   }
-  let slicesData;
-  try {
-    slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
-  } catch (_err) {
+  // The component a DLL belongs to: its only package, else the candidate of the
+  // record's project.
+  const purlForDll = (dllFile, location) =>
+    pkgFilePurls[dllFile]
+      ? pickDosaiComponentByLocation(
+          pkgFilePurls[dllFile],
+          purlAliasMap,
+          location,
+        )
+      : undefined;
+  // Only MethodCalls into a package .dll produce evidence below, so a report
+  // too large to parse whole keeps just those.
+  const slicesData = readDosaiMethodsReport(slicesFile, {
+    keepMethodCall: (methodCall) =>
+      Boolean(
+        methodCall?.Purl ||
+          (methodCall?.Module?.endsWith(".dll") &&
+            pkgFilePurls[methodCall.Module]),
+      ),
+  });
+  if (!slicesData) {
     return pkgList;
   }
   if (slicesData && Object.keys(slicesData)) {
@@ -287,43 +329,36 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
     );
     if (slicesData.Dependencies) {
       for (const adep of slicesData.Dependencies) {
-        if (adep.Purl) {
-          const modPurl = resolveDosaiComponentPurl(adep.Purl, purlAliasMap);
-          if (modPurl) {
-            addDosaiSetValue(
-              purlLocationMap,
-              modPurl,
-              dosaiSourceLocation(adep),
-            );
-            addDosaiSetValue(
-              purlModulesMap,
-              modPurl,
-              adep.Name || adep.Namespace,
-            );
-          }
+        const location = dosaiSourceLocation(adep);
+        const resolvedPurl = adep.Purl
+          ? componentPurlFor(adep.Purl, adep.Path)
+          : undefined;
+        if (resolvedPurl) {
+          addDosaiSetValue(purlLocationMap, resolvedPurl, location);
+          addDosaiSetValue(
+            purlModulesMap,
+            resolvedPurl,
+            adep.Name || adep.Namespace,
+          );
+          continue;
         }
         // Case 1: Dependencies slice has the .dll file
-        if (adep.Module?.endsWith(".dll") && pkgFilePurlMap[adep.Module]) {
-          const modPurl = pkgFilePurlMap[adep.Module];
-          if (!purlLocationMap[modPurl]) {
-            purlLocationMap[modPurl] = new Set();
-          }
-          purlLocationMap[modPurl].add(`${adep.Path}#${adep.LineNumber}`);
-        } else if (
-          adep?.Name &&
-          (adep?.Namespace?.startsWith("System") ||
-            adep?.Namespace?.startsWith("Microsoft"))
-        ) {
-          // Case 2: System packages where the .dll information is missing
-          // In this case, the dll file name is the name followed by dll.
-          const moduleDll = `${adep.Name}.dll`;
-          if (pkgFilePurlMap[moduleDll]) {
-            const modPurl = pkgFilePurlMap[moduleDll];
-            if (!purlLocationMap[modPurl]) {
-              purlLocationMap[modPurl] = new Set();
-            }
-            purlLocationMap[modPurl].add(`${adep.Path}#${adep.LineNumber}`);
-          }
+        // Case 2: System packages where the .dll information is missing. In
+        // this case, the dll file name is the name followed by dll.
+        const moduleDll = adep.Module?.endsWith(".dll")
+          ? adep.Module
+          : adep?.Name &&
+              (adep?.Namespace?.startsWith("System") ||
+                adep?.Namespace?.startsWith("Microsoft"))
+            ? `${adep.Name}.dll`
+            : undefined;
+        const modPurl = moduleDll && purlForDll(moduleDll, adep.Path);
+        if (modPurl) {
+          addDosaiSetValue(
+            purlLocationMap,
+            modPurl,
+            `${adep.Path}#${adep.LineNumber}`,
+          );
         }
       }
     }
@@ -334,95 +369,113 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
       const graphNodes = Object.fromEntries(
         (slicesData.CallGraph?.Nodes || []).map((node) => [node.Id, node]),
       );
-      for (const reachability of slicesData.PackageReachability) {
-        const modPurl = resolveDosaiComponentPurl(
-          reachability.Purl,
-          purlAliasMap,
-        );
-        if (!modPurl) {
-          continue;
-        }
-        // Confidence/evidence facts for this purl, applied to the component below so an
-        // unbuilt tree stays distinguishable from an only-imported package.
-        purlReachabilityMap[modPurl] ??= reachability;
+      // The first fact for a component wins, so a fact that names the
+      // component's exact purl goes before one that reached it by name.
+      const facts = [...slicesData.PackageReachability].sort(
+        (a, b) =>
+          Number(!componentPurls.has(a.Purl)) -
+          Number(!componentPurls.has(b.Purl)),
+      );
+      for (const reachability of facts) {
+        // A fact can span projects (a versionless purl from unrestored
+        // projects), so each location is resolved in its own project, and the
+        // fact applies to every component one of its locations reached.
+        const factPurls = new Set();
+        // Modules and methods of call-graph items that name no source file
+        // (frames inside the package), held for the fact's own version.
+        const unplaced = [];
+        const resolveAt = (location) =>
+          resolveDosaiComponentPurl(reachability.Purl, purlAliasMap, location);
         let hasExplicitSourceLocations = false;
         for (const sourceLocation of reachability.SourceLocations || []) {
           const location = dosaiSourceLocation(sourceLocation);
+          const modPurl = resolveAt(location);
           addDosaiSetValue(purlLocationMap, modPurl, location);
+          if (modPurl) {
+            factPurls.add(modPurl);
+          }
           hasExplicitSourceLocations ||= Boolean(location);
         }
         for (const edgeId of reachability.EdgeIds || []) {
           const edge = graphEdges[edgeId];
+          const location = dosaiSourceLocation(edge);
+          const modPurl = resolveAt(location ?? edge?.Path);
+          const method = edge?.CalledMethodName || edge?.TargetName;
+          if (!modPurl) {
+            unplaced.push([purlMethodsMap, method]);
+            continue;
+          }
+          factPurls.add(modPurl);
           if (!hasExplicitSourceLocations) {
-            addDosaiSetValue(
-              purlLocationMap,
-              modPurl,
-              dosaiSourceLocation(edge),
-            );
+            addDosaiSetValue(purlLocationMap, modPurl, location);
           }
-          addDosaiSetValue(
-            purlMethodsMap,
-            modPurl,
-            edge?.CalledMethodName || edge?.TargetName,
-          );
+          addDosaiSetValue(purlMethodsMap, modPurl, method);
         }
         for (const nodeId of reachability.NodeIds || []) {
           const node = graphNodes[nodeId];
+          const location = dosaiSourceLocationFromNode(node);
+          const modPurl = resolveAt(location);
+          const module = node?.ClassName || node?.Module;
+          const method = node?.Name || node?.Identity?.MethodName;
+          if (!modPurl) {
+            unplaced.push([purlModulesMap, module], [purlMethodsMap, method]);
+            continue;
+          }
+          factPurls.add(modPurl);
           if (!hasExplicitSourceLocations) {
-            addDosaiSetValue(
-              purlLocationMap,
-              modPurl,
-              dosaiSourceLocationFromNode(node),
-            );
+            addDosaiSetValue(purlLocationMap, modPurl, location);
           }
-          addDosaiSetValue(
-            purlModulesMap,
-            modPurl,
-            node?.ClassName || node?.Module,
-          );
-          addDosaiSetValue(
-            purlMethodsMap,
-            modPurl,
-            node?.Name || node?.Identity?.MethodName,
-          );
+          addDosaiSetValue(purlModulesMap, modPurl, module);
+          addDosaiSetValue(purlMethodsMap, modPurl, method);
         }
-      }
-    }
-    if (slicesData.MethodCalls) {
-      for (const amethodCall of slicesData.MethodCalls) {
-        if (
-          amethodCall.Module?.endsWith(".dll") &&
-          pkgFilePurlMap[amethodCall.Module]
-        ) {
-          const modPurl = pkgFilePurlMap[amethodCall.Module];
-          if (!purlLocationMap[modPurl]) {
-            purlLocationMap[modPurl] = new Set();
-          }
-          if (!purlModulesMap[modPurl]) {
-            purlModulesMap[modPurl] = new Set();
+        if (!factPurls.size) {
+          const modPurl = resolveAt(undefined);
+          if (modPurl) {
+            factPurls.add(modPurl);
           }
-          if (!purlMethodsMap[modPurl]) {
-            purlMethodsMap[modPurl] = new Set();
+        }
+        if (factPurls.size === 1) {
+          const [modPurl] = factPurls;
+          for (const [map, value] of unplaced) {
+            addDosaiSetValue(map, modPurl, value);
           }
-          purlLocationMap[modPurl].add(
-            `${amethodCall.Path}#${amethodCall.LineNumber}`,
-          );
-          purlModulesMap[modPurl].add(amethodCall.ClassName);
-          purlMethodsMap[modPurl].add(amethodCall.CalledMethod);
+        }
+        // Confidence/evidence facts for this purl, applied to the component below so an
+        // unbuilt tree stays distinguishable from an only-imported package.
+        for (const modPurl of factPurls) {
+          purlReachabilityMap[modPurl] ??= reachability;
         }
       }
     }
-    if (slicesData.AssemblyInformation) {
-      for (const apkg of pkgList) {
-        if (!apkg.version) {
-          for (const assemblyInfo of slicesData.AssemblyInformation) {
-            if (apkg.name === assemblyInfo.Name) {
-              apkg.version = assemblyInfo.Version;
-            }
-          }
+    if (slicesData.MethodCalls) {
+      for (const amethodCall of slicesData.MethodCalls) {
+        // dosai's purl is resolved in the caller's project; the DLL name is the
+        // same for every version of the package, and fills in for a purl that
+        // names no component, such as the framework purl dosai gives a
+        // System.* assembly it cannot map to a package.
+        const modPurl =
+          (amethodCall.Purl
+            ? componentPurlFor(amethodCall.Purl, amethodCall.Path)
+            : undefined) ||
+          (amethodCall.Module?.endsWith(".dll")
+            ? purlForDll(amethodCall.Module, amethodCall.Path)
+            : undefined);
+        if (!modPurl) {
+          continue;
         }
+        addDosaiSetValue(
+          purlLocationMap,
+          modPurl,
+          dosaiCallSiteLocation(amethodCall, options.srcPath),
+        );
+        addDosaiSetValue(purlModulesMap, modPurl, amethodCall.ClassName);
+        addDosaiSetValue(purlMethodsMap, modPurl, amethodCall.CalledMethod);
       }
     }
+    // AssemblyInformation used to fill in a missing version by assembly name.
+    // An assembly version is not the package version (Newtonsoft.Json 13.0.3
+    // ships assembly 13.0.0.0), the purl and bom-ref kept no version, and the
+    // last same-named assembly won, so the component described two packages.
   }
   if (Object.keys(purlLocationMap).length) {
     for (const apkg of pkgList) {