@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/cppEvidence.js

@@ -19,17 +19,205 @@ import {
 import { build } from "@cdxgen/cdx-purl";
 
 import { DEBUG_MODE } from "../core/activity.js";
-import { safeExistsSync } from "../core/fs.js";
+import { getAllFiles, safeExistsSync } from "../core/fs.js";
+import { thoughtLog } from "../core/logger.js";
 import { CPP_STD_MODULES } from "../core/state.js";
 import { findAppModules } from "../inventory/atomUtils.js";
+import { cmakeBuildDirCandidates } from "../inventory/cmakeBuildDirs.js";
 import { getOSPackageForFile } from "../inventory/osPackageResolver.js";
 import { locateGenericPackage } from "../inventory/purl.js";
+import {
+  createIncludeAttributor,
+  parseCIncludeSlices,
+} from "./cppIncludeResolver.js";
 import {
   parseCmakeLikeFile,
   parseCUsageSlice,
   parseXmakeRequiresLock,
 } from "./parsers-misc.js";
 
+/**
+ * Headers of the C standard library (C89 to C23) and of POSIX and the common
+ * system interfaces. They come with the C library or the operating system: a
+ * component only when an OS package provides them, never a generic one.
+ */
+const C_SYSTEM_HEADERS = new Set([
+  "assert.h",
+  "complex.h",
+  "ctype.h",
+  "errno.h",
+  "fenv.h",
+  "float.h",
+  "inttypes.h",
+  "iso646.h",
+  "limits.h",
+  "locale.h",
+  "math.h",
+  "setjmp.h",
+  "signal.h",
+  "stdalign.h",
+  "stdarg.h",
+  "stdatomic.h",
+  "stdbit.h",
+  "stdbool.h",
+  "stdckdint.h",
+  "stddef.h",
+  "stdint.h",
+  "stdio.h",
+  "stdlib.h",
+  "stdnoreturn.h",
+  "string.h",
+  "tgmath.h",
+  "threads.h",
+  "time.h",
+  "uchar.h",
+  "wchar.h",
+  "wctype.h",
+  "aio.h",
+  "alloca.h",
+  "arpa/inet.h",
+  "cpio.h",
+  "dirent.h",
+  "dlfcn.h",
+  "endian.h",
+  "err.h",
+  "execinfo.h",
+  "fcntl.h",
+  "fmtmsg.h",
+  "fnmatch.h",
+  "ftw.h",
+  "getopt.h",
+  "glob.h",
+  "grp.h",
+  "iconv.h",
+  "ifaddrs.h",
+  "langinfo.h",
+  "libgen.h",
+  "libintl.h",
+  "malloc.h",
+  "memory.h",
+  "monetary.h",
+  "mqueue.h",
+  "ndbm.h",
+  "net/if.h",
+  "netdb.h",
+  "netinet/in.h",
+  "netinet/tcp.h",
+  "nl_types.h",
+  "poll.h",
+  "pthread.h",
+  "pwd.h",
+  "regex.h",
+  "sched.h",
+  "search.h",
+  "semaphore.h",
+  "spawn.h",
+  "strings.h",
+  "stropts.h",
+  "sys/ioctl.h",
+  "sys/ipc.h",
+  "sys/mman.h",
+  "sys/msg.h",
+  "sys/param.h",
+  "sys/resource.h",
+  "sys/select.h",
+  "sys/sem.h",
+  "sys/shm.h",
+  "sys/socket.h",
+  "sys/stat.h",
+  "sys/statvfs.h",
+  "sys/syscall.h",
+  "sys/sysctl.h",
+  "sys/time.h",
+  "sys/times.h",
+  "sys/types.h",
+  "sys/uio.h",
+  "sys/un.h",
+  "sys/utsname.h",
+  "sys/wait.h",
+  "syslog.h",
+  "tar.h",
+  "termios.h",
+  "ucontext.h",
+  "ulimit.h",
+  "unistd.h",
+  "utime.h",
+  "utmpx.h",
+  "wordexp.h",
+]);
+
+/**
+ * Add the symbols a file uses to a component's `internal:ImportedSymbols`,
+ * keeping every other property and the symbols already recorded.
+ *
+ * @param {Object} apkg Component to enrich
+ * @param {string[]} symbols Symbol names
+ */
+function mergeImportedSymbols(apkg, symbols) {
+  apkg.properties = apkg.properties || [];
+  const prop = apkg.properties.find(
+    (p) => p.name === "internal:ImportedSymbols",
+  );
+  if (prop) {
+    prop.value = Array.from(new Set([...prop.value.split("|"), ...symbols]))
+      .sort()
+      .join("|");
+  } else {
+    apkg.properties.push({
+      name: "internal:ImportedSymbols",
+      value: symbols.join("|"),
+    });
+  }
+}
+
+/**
+ * The file name extensions of the C and C++ sources, headers and module
+ * units atom's C frontend parses. The search ignores case, so this also
+ * covers `.C`, `.H` and the upper-case Windows spellings.
+ */
+const CPP_SOURCE_EXTENSIONS = [
+  "c",
+  "cc",
+  "cpp",
+  "cxx",
+  "c++",
+  "h",
+  "hh",
+  "hpp",
+  "hxx",
+  "h++",
+  "h.in",
+  "i",
+  "inl",
+  "ipp",
+  "tcc",
+  "tmh",
+  "ccm",
+  "cppm",
+  "cxxm",
+  "c++m",
+  "ixx",
+  "mxx",
+];
+
+/**
+ * Whether a directory holds C or C++ sources for atom to parse.
+ *
+ * The search skips the directories the manifest searches skip, such as
+ * node_modules and dot directories, and honours the exclude patterns, so the
+ * sources of a native npm addon do not count.
+ *
+ * @param {string} src directory
+ * @param {Object} options CLI options
+ * @returns {boolean} true when the directory holds at least one C or C++ file
+ */
+export function hasCppSources(src, options = {}) {
+  return (
+    getAllFiles(src, `**/*.{${CPP_SOURCE_EXTENSIONS.join(",")}}`, options)
+      .length > 0
+  );
+}
+
 /**
  * Method to find c/c++ modules by collecting usages with atom
  *
@@ -37,8 +225,18 @@ import {
  * @param {object} options Command line options
  * @param {array} osPkgsList Array of OS pacakges represented as components
  * @param {array} epkgList Existing packages list
+ * @param {function(string): boolean} [isFirstPartyHeader] Whether an
+ *   included header is the project's own (found under the project root or
+ *   one of its own include directories, outside any dependency's sources);
+ *   such a header is not a component
  */
-export function getCppModules(src, options, osPkgsList, epkgList) {
+export function getCppModules(
+  src,
+  options,
+  osPkgsList,
+  epkgList,
+  isFirstPartyHeader = undefined,
+) {
   // Generic is the type to use where the package registry could not be located
   const pkgType = "generic";
   const pkgList = [];
@@ -48,9 +246,21 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
   let parentComponent;
   const dependsOn = new Set();
 
+  // Components from CMake and Conan carry no group: the key spells a missing
+  // group the same way on insert and on lookup.
+  const epkgKey = (group, name) => `${group || ""}/${name}`;
   (epkgList || []).forEach((p) => {
-    epkgMap[`${p.group}/${p.name}`] = p;
+    epkgMap[epkgKey(p.group, p.name)] = p;
   });
+  // Attributes a header from the file its include resolved to, when atom
+  // names that file (atom 4 and later)
+  const attributeInclude = createIncludeAttributor({
+    src,
+    components: epkgList || [],
+    buildDirs: cmakeBuildDirCandidates(src, options),
+    isFirstPartyHeader,
+  });
+  const vcpkgPorts = attributeInclude.vcpkgPorts();
   // Let's look for any vcpkg.json file to tell us about the directory we're scanning
   // users can use this file to give us a clue even if they do not use vcpkg library manager
   if (safeExistsSync(join(src, "vcpkg.json"))) {
@@ -106,21 +316,23 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
           // Is this a dependency we haven't seen before including the all lower and upper case version?
           if (
             avcpkgName &&
-            !epkgMap[`/${avcpkgName}`] &&
-            !epkgMap[`/${avcpkgName.toLowerCase()}`] &&
-            !epkgMap[`/${avcpkgName.toUpperCase()}`]
+            !epkgMap[epkgKey("", avcpkgName)] &&
+            !epkgMap[epkgKey("", avcpkgName.toLowerCase())] &&
+            !epkgMap[epkgKey("", avcpkgName.toUpperCase())]
           ) {
+            // the version vcpkg installed, when the build tree has the port
+            const installed = vcpkgPorts.get(avcpkgName.toLowerCase());
             const pkgPurl = build({
               type: pkgType,
               namespace: "" || null,
               name: avcpkgName,
-              version: "" || null,
+              version: installed?.version || null,
             });
             const apkg = {
               group: "",
               name: avcpkgName,
               type: pkgType,
-              version: "",
+              version: installed?.version || "",
               purl: pkgPurl,
               scope,
               "bom-ref": decodeURIComponent(pkgPurl),
@@ -129,6 +341,9 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
                 ...(vcpkgFeatures
                   ? [{ name: "cdx:vcpkg:features", value: vcpkgFeatures }]
                   : []),
+                ...(installed
+                  ? [{ name: "cdx:vcpkg:triplet", value: installed.triplet }]
+                  : []),
               ],
               evidence: {
                 identity: {
@@ -195,7 +410,7 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
     if (DEBUG_MODE) {
       console.log("Re-using existing slices file", options.usagesSlicesFile);
     }
-  } else {
+  } else if (hasCppSources(src, options)) {
     sliceData = findAppModules(
       src,
       options.deep ? "c" : "h",
@@ -203,11 +418,125 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
       options.usagesSlicesFile,
       options,
     );
+  } else {
+    thoughtLog(
+      `There are no C or C++ sources at '${src}', so atom usages has nothing to read.`,
+    );
+    sliceData = [];
   }
   const usageData = parseCUsageSlice(sliceData);
-  for (let afile of Object.keys(usageData)) {
+  const includeInfo = parseCIncludeSlices(sliceData);
+  const osComponents = new Map();
+  const addedRefs = new Set();
+  // The component a resolved include belongs to, and whether it is already
+  // listed (a dependency or vendored directory, a declared vcpkg port, a Conan
+  // package the lock file named)
+  const includeComponents = new Map();
+  const listedNamed = (name, purlType) =>
+    [...(epkgList || []), ...pkgList].find(
+      (p) =>
+        p.name?.toLowerCase() === name.toLowerCase() &&
+        (!purlType || `${p.purl || ""}`.startsWith(`pkg:${purlType}/`)),
+    );
+  const componentForInclude = (target) => {
+    if (target.kind === "component") {
+      return { component: target.component, listed: true };
+    }
+    let coords;
+    if (target.kind === "os") {
+      coords = {
+        name: target.pkgInfo.name,
+        version: target.pkgInfo.version || "",
+        purl: target.pkgInfo.purl,
+        technique: "filename",
+      };
+    } else if (target.kind === "vcpkg") {
+      const declared = listedNamed(target.port, pkgType);
+      if (declared) {
+        // already in the list: declared in vcpkg.json, or given by the caller
+        return { component: declared, listed: true };
+      }
+      coords = {
+        name: target.port,
+        version: target.version,
+        purl: build({
+          type: pkgType,
+          name: target.port,
+          version: target.version || null,
+        }),
+        properties: [{ name: "cdx:vcpkg:triplet", value: target.triplet }],
+        technique: "manifest-analysis",
+      };
+    } else {
+      const locked = listedNamed(target.name, "conan");
+      if (locked) {
+        return { component: locked, listed: true };
+      }
+      coords = {
+        name: target.name,
+        version: target.version,
+        purl: build({
+          type: "conan",
+          name: target.name,
+          version: target.version || null,
+        }),
+        technique: "manifest-analysis",
+      };
+    }
+    const ref = decodeURIComponent(
+      coords.purl || `${coords.name}@${coords.version}`,
+    );
+    let component = includeComponents.get(ref);
+    if (!component) {
+      component = {
+        name: coords.name,
+        version: coords.version,
+        type: "library",
+        purl: coords.purl,
+        "bom-ref": ref,
+        properties: coords.properties || [],
+        evidence: {
+          identity: {
+            field: "purl",
+            confidence: 0.8,
+            methods: [
+              {
+                technique: coords.technique,
+                confidence: 0.8,
+                value: "Resolved include file",
+              },
+            ],
+          },
+        },
+      };
+      includeComponents.set(ref, component);
+    }
+    return { component, listed: false };
+  };
+  const symbolsOf = (key, header) =>
+    Array.from(
+      new Set([
+        ...(usageData[key] || []),
+        ...(includeInfo.get(key)?.symbols || []),
+      ]),
+    )
+      .filter(
+        (v) =>
+          !v.startsWith("<") &&
+          !v.startsWith("__") &&
+          v !== "main" &&
+          !v.includes("anonymous_") &&
+          !v.includes(header),
+      )
+      .map((v) => v.split(":")[0])
+      .sort();
+  // Only includes name headers; the other keys are the project's own source
+  // files, with the symbols they use
+  for (const sliceKey of Object.keys(usageData).filter((k) =>
+    includeInfo.has(k),
+  )) {
     // Normalize windows separator
-    afile = afile.replace("..\\", "").replace(/\\/g, "/");
+    const afile = sliceKey.replace("..\\", "").replace(/\\/g, "/");
     const fileName = basename(afile);
     if (!fileName?.length) {
       continue;
@@ -235,13 +564,58 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
     if (CPP_STD_MODULES.includes(name)) {
       continue;
     }
-    let apkg = getOSPackageForFile(afile, osPkgsList) ||
-      epkgMap[`${group}/${name}`] || {
+    // The file the include resolved to names its package exactly
+    const resolvedFiles = includeInfo.get(sliceKey)?.paths;
+    const target = resolvedFiles?.size
+      ? attributeInclude(resolvedFiles)
+      : undefined;
+    if (target?.kind === "first-party") {
+      continue;
+    }
+    if (target) {
+      const { component, listed } = componentForInclude(target);
+      const usymbols = symbolsOf(sliceKey, afile);
+      if (usymbols.length) {
+        mergeImportedSymbols(component, usymbols);
+      }
+      if (!listed && !addedRefs.has(component["bom-ref"])) {
+        pkgList.push(component);
+        addedRefs.add(component["bom-ref"]);
+      }
+      dependsOn.add(component["bom-ref"]);
+      continue;
+    }
+    if (isFirstPartyHeader?.(afile)) {
+      continue;
+    }
+    // The OS package list is shared by every header: a component taken from it
+    // is a copy, so the internal:PkgProvides the next header is matched against
+    // stays intact. A component the caller already listed is enriched in place
+    // and not listed again.
+    const osPkg = getOSPackageForFile(afile, osPkgsList);
+    if (!osPkg && C_SYSTEM_HEADERS.has(afile)) {
+      continue;
+    }
+    const listedPkg = osPkg ? undefined : epkgMap[epkgKey(group, name)];
+    let apkg;
+    if (osPkg) {
+      const osRef = osPkg["bom-ref"] || osPkg.purl || osPkg.name;
+      apkg = osComponents.get(osRef);
+      if (!apkg) {
+        apkg = {
+          ...osPkg,
+          properties: (osPkg.properties || []).map((prop) => ({ ...prop })),
+        };
+        osComponents.set(osRef, apkg);
+      }
+    } else {
+      apkg = listedPkg || {
         name,
         group,
         version: "",
         type: pkgType,
       };
+    }
     // If this is a relative file, there is a good chance we can reuse the project group
     if (!afile.startsWith(_sep) && !group.length) {
       group = options.projectGroup || "";
@@ -279,48 +653,22 @@ export function getCppModules(src, options, osPkgsList, epkgList) {
       };
       apkg["bom-ref"] = decodeURIComponent(apkg["purl"]);
     }
-    if (usageData[afile]) {
-      const usymbols = Array.from(usageData[afile])
-        .filter(
-          (v) =>
-            !v.startsWith("<") &&
-            !v.startsWith("__") &&
-            v !== "main" &&
-            !v.includes("anonymous_") &&
-            !v.includes(afile),
-        )
-        .map((v) => v.split(":")[0])
-        .sort();
-      if (!apkg["properties"] && usymbols.length) {
-        apkg["properties"] = [
-          { name: "internal:ImportedSymbols", value: usymbols.join("|") },
-        ];
-      } else {
-        apkg["properties"] = [];
-      }
-      const newProps = [];
-      let symbolsPropertyFound = false;
-      for (const prop of apkg["properties"]) {
-        if (prop.name === "internal:ImportedSymbols") {
-          symbolsPropertyFound = true;
-          let existingSymbols = prop.value.split("|");
-          existingSymbols = existingSymbols.concat(usymbols);
-          prop.value = Array.from(new Set(existingSymbols)).sort().join("|");
-        }
-        newProps.push(prop);
-      }
-      if (!symbolsPropertyFound && usymbols.length) {
-        apkg["properties"].push({
-          name: "internal:ImportedSymbols",
-          value: usymbols.join("|"),
-        });
-      }
-      apkg["properties"] = newProps;
+    const usymbols = symbolsOf(sliceKey, afile);
+    if (usymbols.length) {
+      mergeImportedSymbols(apkg, usymbols);
     }
     // At this point, we have a package but we don't know what it's called
     // So let's try to locate this generic package using some heuristics
     apkg = locateGenericPackage(apkg);
-    if (!pkgAddedMap[name]) {
+    if (listedPkg) {
+      dependsOn.add(apkg["bom-ref"]);
+    } else if (osPkg) {
+      if (!addedRefs.has(apkg["bom-ref"])) {
+        pkgList.push(apkg);
+        dependsOn.add(apkg["bom-ref"]);
+        addedRefs.add(apkg["bom-ref"]);
+      }
+    } else if (!pkgAddedMap[name]) {
       pkgList.push(apkg);
       dependsOn.add(apkg["bom-ref"]);
       pkgAddedMap[name] = true;