@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/ecosystems/vendoredCode.js

@@ -7,7 +7,7 @@
  */
 
 import { readFileSync, statSync } from "node:fs";
-import { basename, dirname, relative, resolve, sep } from "node:path";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
 
 import { getAllFiles } from "../core/fs.js";
 import { isInsideDir } from "../inventory/cmakeBuildDirs.js";
@@ -69,6 +69,222 @@ function licenseOf(file) {
   }
 }
 
+/** A version header or CMakeLists.txt larger than this is not read. */
+const MAX_VERSION_FILE_BYTES = 256 * 1024;
+
+function readSmall(file) {
+  try {
+    const stat = statSync(file);
+    if (!stat.isFile() || stat.size > MAX_VERSION_FILE_BYTES) {
+      return undefined;
+    }
+    return readFileSync(file, "utf-8");
+  } catch (_err) {
+    return undefined;
+  }
+}
+
+function defineValue(text, name) {
+  for (const line of text.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith("#")) {
+      continue;
+    }
+    const parts = trimmed.replace(/^#\s*/, "").split(/\s+/);
+    if (parts[0] === "define" && parts[1] === name && parts.length >= 3) {
+      return parts.slice(2).join(" ").replace(/^"|"$/g, "");
+    }
+  }
+  return undefined;
+}
+
+function numericTriple(text, names) {
+  const values = names.map((name) => defineValue(text, name));
+  return values.every((value) => /^\d+$/.test(value || ""))
+    ? values.join(".")
+    : undefined;
+}
+
+function packedBrotliVersion(value) {
+  if (!/^0x[0-9a-fA-F]{1,8}$/.test(value || "")) {
+    return undefined;
+  }
+  const packed = Number.parseInt(value, 16);
+  return `${packed >>> 24}.${(packed >>> 12) & 0xfff}.${packed & 0xfff}`;
+}
+
+/**
+ * Libraries that are often copied into C and C++ trees, the header each keeps
+ * its version in, and the CPE NVD names it by. The header is the library's
+ * own, so finding it at that path is what identifies the copy.
+ */
+const VENDORED_VERSION_HEADERS = [
+  {
+    file: "c/common/version.h",
+    // 1.1 and later define MAJOR/MINOR/PATCH; 1.0.x only the packed
+    // BROTLI_VERSION 0xMMMnnnppp (1.0.9 is 0x1000009).
+    read: (text) =>
+      numericTriple(text, [
+        "BROTLI_VERSION_MAJOR",
+        "BROTLI_VERSION_MINOR",
+        "BROTLI_VERSION_PATCH",
+      ]) || packedBrotliVersion(defineValue(text, "BROTLI_VERSION")),
+    cpe: "google:brotli",
+    // google/brotli 0ed4268 (CVE-2026-111497) added this error code.
+    fixMarker: {
+      file: "c/include/brotli/decode.h",
+      text: "BLOCK_SWITCH",
+      property: "cdx:brotli:c:blockSwitchFix",
+    },
+  },
+  {
+    file: "zlib.h",
+    read: (text) => defineValue(text, "ZLIB_VERSION"),
+    cpe: "zlib:zlib",
+  },
+  {
+    file: "lib/zstd.h",
+    read: (text) =>
+      numericTriple(text, [
+        "ZSTD_VERSION_MAJOR",
+        "ZSTD_VERSION_MINOR",
+        "ZSTD_VERSION_RELEASE",
+      ]),
+    cpe: "facebook:zstandard",
+  },
+  {
+    file: "lib/lz4.h",
+    read: (text) =>
+      numericTriple(text, [
+        "LZ4_VERSION_MAJOR",
+        "LZ4_VERSION_MINOR",
+        "LZ4_VERSION_RELEASE",
+      ]),
+    cpe: "lz4_project:lz4",
+  },
+  {
+    file: "png.h",
+    read: (text) => defineValue(text, "PNG_LIBPNG_VER_STRING"),
+    cpe: "libpng:libpng",
+  },
+  {
+    file: "sqlite3.h",
+    read: (text) => defineValue(text, "SQLITE_VERSION"),
+    cpe: "sqlite:sqlite",
+  },
+  {
+    file: "lib/expat.h",
+    read: (text) =>
+      numericTriple(text, [
+        "XML_MAJOR_VERSION",
+        "XML_MINOR_VERSION",
+        "XML_MICRO_VERSION",
+      ]),
+    cpe: "libexpat_project:libexpat",
+  },
+];
+
+const PLAIN_VERSION_RE = /^\d+(\.\d+){1,3}$/;
+
+// `project(brotli VERSION 1.2.0 LANGUAGES C)`, on one line.
+function cmakeProjectVersion(dir) {
+  const text = readSmall(join(dir, "CMakeLists.txt"));
+  if (!text) {
+    return undefined;
+  }
+  for (const line of text.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed.toLowerCase().startsWith("project(")) {
+      continue;
+    }
+    const args = trimmed.slice("project(".length).split(/[\s)]+/);
+    const index = args.findIndex((arg) => arg.toUpperCase() === "VERSION");
+    if (index !== -1 && PLAIN_VERSION_RE.test(args[index + 1] || "")) {
+      return args[index + 1];
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Read the version of a vendored directory from the library's own version
+ * header, else from its CMake project() call.
+ *
+ * @param {string} dir Absolute path of the vendored directory
+ * @returns {{version?: string, source?: string, cpe?: string, properties: Object[]}|undefined}
+ *   undefined when nothing is known, and without a version when only the fix
+ *   marker could be read
+ */
+export function vendoredVersionOf(dir) {
+  for (const entry of VENDORED_VERSION_HEADERS) {
+    const text = readSmall(join(dir, entry.file));
+    if (text === undefined) {
+      continue;
+    }
+    // The fix marker is read whether or not a version could be: a tree whose
+    // version header changed shape still shows whether its decoder is fixed.
+    const properties = [];
+    if (entry.fixMarker) {
+      const markerText = readSmall(join(dir, entry.fixMarker.file));
+      if (markerText !== undefined) {
+        properties.push({
+          name: entry.fixMarker.property,
+          value: markerText.includes(entry.fixMarker.text) ? "true" : "false",
+        });
+      }
+    }
+    const version = entry.read(text);
+    if (version && PLAIN_VERSION_RE.test(version)) {
+      return {
+        version,
+        source: entry.file,
+        cpe: `cpe:2.3:a:${entry.cpe}:${version}:*:*:*:*:*:*:*`,
+        properties,
+      };
+    }
+    if (properties.length) {
+      return { properties };
+    }
+  }
+  const version = cmakeProjectVersion(dir);
+  return version
+    ? { version, source: "CMakeLists.txt", properties: [] }
+    : undefined;
+}
+
+/**
+ * Directory names under which a project keeps other projects' code.
+ */
+const VENDORED_DIR_NAMES = new Set([
+  "third_party",
+  "third-party",
+  "thirdparty",
+  "vendor",
+  "vendored",
+  "external",
+]);
+
+/**
+ * Whether a file lies under a directory conventionally holding another
+ * project's code (`third_party/`, `vendor/`, `external/`) below the scan root.
+ * Build files there belong to the copied project: building them runs that
+ * project's build, and their dependencies are its build tooling.
+ *
+ * @param {string} root Project scan root
+ * @param {string} file File path
+ * @returns {boolean}
+ */
+export function isInVendoredDir(root, file) {
+  const rel = relative(resolve(root), resolve(file));
+  if (!rel || rel.startsWith("..")) {
+    return false;
+  }
+  return rel
+    .split(/[\\/]+/)
+    .slice(0, -1)
+    .some((segment) => VENDORED_DIR_NAMES.has(segment.toLowerCase()));
+}
+
 function licenseChoice(id) {
   return id.includes(" ") ? { expression: id } : { license: { id } };
 }
@@ -120,17 +336,32 @@ export function findVendoredCode(root, options = {}, excludeDirs = []) {
     }
     const path = relDir(file);
     const name = basename(dir);
-    const purl = tryBuildPurl({ type: "generic", name, subpath: path });
+    const versionInfo = vendoredVersionOf(dir);
+    const purl = tryBuildPurl({
+      type: "generic",
+      name,
+      version: versionInfo?.version || null,
+      subpath: path,
+    });
     const licenseFile = relative(absRoot, file).split(sep).join("/");
     const component = {
       type: "library",
       name,
-      version: "",
+      version: versionInfo?.version || "",
       licenses: [licenseChoice(id)],
       properties: [
         { name: "cdx:vendored", value: "true" },
         { name: "cdx:vendored:path", value: path },
         { name: "cdx:vendored:licenseFile", value: licenseFile },
+        ...(versionInfo?.version
+          ? [
+              {
+                name: "cdx:vendored:versionSource",
+                value: `${path}/${versionInfo.source}`,
+              },
+            ]
+          : []),
+        ...(versionInfo?.properties || []),
       ],
       evidence: {
         identity: {
@@ -143,6 +374,9 @@ export function findVendoredCode(root, options = {}, excludeDirs = []) {
         licenses: [licenseChoice(id)],
       },
     };
+    if (versionInfo?.cpe) {
+      component.cpe = versionInfo.cpe;
+    }
     if (purl) {
       component.purl = purl;
       component["bom-ref"] = decodeURIComponent(purl);

package/lib/ecosystems/venvProbe.js

@@ -0,0 +1,360 @@
+// Bounded, read-only probe for the Python packages of virtual environments and
+// pyenv installs on a live host, for `-t os` / obom.
+//
+// osquery's python_packages table reads the interpreter's own site-packages
+// directories, so a package installed into a virtual environment (an app's
+// /opt/app/venv, a Poetry or pipenv environment under a home directory) or into
+// a pyenv-built Python is missing from an OBOM. Brotli and brotlicffi are
+// usually installed exactly there, pulled in by requests, aiohttp or httpx.
+//
+// Environments are found in fixed places, one wildcard level at a time: a
+// directory holding pyvenv.cfg is a virtual environment, and a pyenv
+// `versions/*` directory is an interpreter install. Only `*.dist-info/METADATA`
+// files are read; nothing is executed.
+
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import process from "node:process";
+
+import { isDryRun, recordActivity } from "../core/activity.js";
+import { safeExistsSync, safeReadFileSync } from "../core/fs.js";
+import { thoughtLog } from "../core/logger.js";
+import { applyPurl, tryBuildPurl } from "../inventory/purl.js";
+import { parseBdistMetadata } from "./ecosystems.js";
+
+const MAX_ENVIRONMENTS = 64;
+const MAX_DIRECTORIES = 512;
+const MAX_PACKAGES = 8192;
+const MAX_METADATA_BYTES = 4 * 1024 * 1024;
+
+// Relative to the root.
+const ROOT_PATTERNS = [
+  "opt/*",
+  "opt/*/*",
+  "srv/*",
+  "srv/*/*",
+  "app",
+  "app/*",
+  "usr/local/*",
+];
+// Relative to a home directory.
+const HOME_PATTERNS = [
+  "*",
+  "*/*",
+  ".virtualenvs/*",
+  ".local/share/virtualenvs/*",
+  ".cache/pypoetry/virtualenvs/*",
+  "Library/Caches/pypoetry/virtualenvs/*",
+  ".pyenv/versions/*",
+];
+const HOME_PARENTS = { linux: ["home"], darwin: ["Users"] };
+// Folders of a macOS home that privacy protection (TCC) guards. Looking inside
+// one makes macOS ask the user to grant the terminal access, so a wildcard
+// never enters them; Library is skipped too, for the Mail, Messages and cloud
+// storage folders under it. Patterns that name a folder below Library, such as
+// Poetry's cache, still reach it.
+const MACOS_PROTECTED_HOME_FOLDERS = new Set([
+  "Desktop",
+  "Documents",
+  "Downloads",
+  "Library",
+  "Movies",
+  "Music",
+  "Pictures",
+]);
+
+function listDirectory(directory) {
+  try {
+    return readdirSync(directory).sort();
+  } catch {
+    return [];
+  }
+}
+
+function isDirectory(path) {
+  try {
+    return statSync(path).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function isFile(path) {
+  try {
+    return statSync(path).isFile();
+  } catch {
+    return false;
+  }
+}
+
+// `skipFirst` names folders a wildcard in the pattern's first segment never
+// enters.
+function expand(base, pattern, skipFirst) {
+  let current = [base];
+  for (const [depth, segment] of pattern.split("/").entries()) {
+    const next = [];
+    for (const directory of current) {
+      if (segment !== "*") {
+        next.push(join(directory, segment));
+        continue;
+      }
+      for (const name of listDirectory(directory)) {
+        // Hidden directories are only entered when a pattern names them.
+        if (name.startsWith(".") || (depth === 0 && skipFirst?.has(name))) {
+          continue;
+        }
+        next.push(join(directory, name));
+        if (next.length >= MAX_DIRECTORIES) {
+          break;
+        }
+      }
+    }
+    current = next.filter(isDirectory).slice(0, MAX_DIRECTORIES);
+  }
+  return current;
+}
+
+/**
+ * Find the virtual environments and pyenv interpreters under the fixed
+ * places of a root.
+ *
+ * @param {string} rootPath root directory
+ * @param {string} platform `linux` or `darwin`
+ * @returns {{path: string, kind: string}[]} environments, at most MAX_ENVIRONMENTS
+ */
+export function findPythonEnvironments(rootPath, platform) {
+  const bases = [{ base: rootPath, patterns: ROOT_PATTERNS }];
+  const homes = platform === "linux" ? [join(rootPath, "root")] : [];
+  for (const parent of HOME_PARENTS[platform] || []) {
+    for (const name of listDirectory(join(rootPath, parent))) {
+      homes.push(join(rootPath, parent, name));
+    }
+  }
+  const skipFirst =
+    platform === "darwin" ? MACOS_PROTECTED_HOME_FOLDERS : undefined;
+  for (const home of homes.filter(isDirectory)) {
+    bases.push({ base: home, patterns: HOME_PATTERNS, skipFirst });
+  }
+  const environments = [];
+  const seen = new Set();
+  for (const { base, patterns, skipFirst } of bases) {
+    for (const pattern of patterns) {
+      for (const candidate of expand(base, pattern, skipFirst)) {
+        if (seen.has(candidate)) {
+          continue;
+        }
+        let kind;
+        if (isFile(join(candidate, "pyvenv.cfg"))) {
+          kind = "virtualenv";
+        } else if (pattern.endsWith(".pyenv/versions/*")) {
+          kind = "pyenv";
+        }
+        // A directory with a nested .venv is the project, not the environment.
+        if (!kind && isFile(join(candidate, ".venv", "pyvenv.cfg"))) {
+          const nested = join(candidate, ".venv");
+          if (!seen.has(nested)) {
+            seen.add(nested);
+            environments.push({ path: nested, kind: "virtualenv" });
+          }
+          continue;
+        }
+        if (!kind) {
+          continue;
+        }
+        seen.add(candidate);
+        environments.push({ path: candidate, kind });
+        if (environments.length >= MAX_ENVIRONMENTS) {
+          return environments;
+        }
+      }
+    }
+  }
+  return environments;
+}
+
+function sitePackagesOf(environment) {
+  const lib = join(environment, "lib");
+  const directories = [];
+  for (const name of listDirectory(lib)) {
+    if (!name.startsWith("python")) {
+      continue;
+    }
+    const sitePackages = join(lib, name, "site-packages");
+    if (isDirectory(sitePackages)) {
+      directories.push(sitePackages);
+    }
+  }
+  return directories;
+}
+
+function readMetadata(file) {
+  try {
+    const stats = statSync(file);
+    if (!stats.isFile() || stats.size > MAX_METADATA_BYTES) {
+      return undefined;
+    }
+  } catch {
+    return undefined;
+  }
+  const text = safeReadFileSync(file, "utf-8");
+  return typeof text === "string" ? text : undefined;
+}
+
+/**
+ * Inventory the packages installed in the virtual environments and pyenv
+ * interpreters of a host. One component per distinct name and version, with
+ * an occurrence for each dist-info directory it was found in, so no host path
+ * reaches a purl.
+ *
+ * @param {Object} [options] options
+ * @param {string} [options.rootPath="/"] root directory; tests point it at a built tree
+ * @param {string} [options.platform] `linux` or `darwin`
+ * @returns {Object[]} pypi components
+ */
+export function probeHostPythonEnvironments(options = {}) {
+  const rootPath = options.rootPath || "/";
+  const platform = options.platform || process.platform;
+  if (!HOME_PARENTS[platform]) {
+    return [];
+  }
+  if (isDryRun) {
+    recordActivity({
+      capability: "python-env-probe",
+      kind: "read",
+      reason:
+        "Dry run mode blocks the Python environment probe, which reads package metadata outside the project.",
+      status: "blocked",
+      target: rootPath,
+    });
+    return [];
+  }
+  if (!safeExistsSync(rootPath)) {
+    return [];
+  }
+  const environments = findPythonEnvironments(rootPath, platform);
+  const byRef = new Map();
+  let packages = 0;
+  for (const environment of environments) {
+    for (const sitePackages of sitePackagesOf(environment.path)) {
+      for (const name of listDirectory(sitePackages)) {
+        if (!name.endsWith(".dist-info") || packages >= MAX_PACKAGES) {
+          continue;
+        }
+        const distInfo = join(sitePackages, name);
+        const metadataFile = join(distInfo, "METADATA");
+        const raw = readMetadata(metadataFile);
+        if (raw === undefined) {
+          continue;
+        }
+        const [pkg] = parseBdistMetadata(metadataFile, raw);
+        if (!pkg?.name || !pkg.version) {
+          continue;
+        }
+        packages += 1;
+        const purl = tryBuildPurl({
+          type: "pypi",
+          name: pkg.name,
+          version: pkg.version,
+        });
+        if (!purl) {
+          continue;
+        }
+        const ref = decodeURIComponent(purl);
+        let component = byRef.get(ref);
+        if (!component) {
+          component = applyPurl(
+            {
+              type: "library",
+              name: pkg.name,
+              version: pkg.version,
+              ...(pkg.description ? { description: pkg.description } : {}),
+              properties: [
+                { name: "cdx:python:environment", value: environment.kind },
+              ],
+              evidence: { occurrences: [] },
+            },
+            purl,
+          );
+          byRef.set(ref, component);
+        } else if (
+          !component.properties.some(
+            (prop) =>
+              prop.name === "cdx:python:environment" &&
+              prop.value === environment.kind,
+          )
+        ) {
+          component.properties.push({
+            name: "cdx:python:environment",
+            value: environment.kind,
+          });
+        }
+        component.evidence.occurrences.push({ location: distInfo });
+      }
+    }
+  }
+  recordActivity({
+    capability: "python-env-probe",
+    kind: "read",
+    reason: `Read ${packages} package(s) from ${environments.length} Python environment(s) in the fixed places.`,
+    status: "completed",
+    target: rootPath,
+  });
+  if (byRef.size) {
+    thoughtLog(
+      `I found ${byRef.size} Python package(s) in ${environments.length} virtual environment(s) or pyenv install(s).`,
+    );
+  }
+  return Array.from(byRef.values()).sort((a, b) =>
+    a["bom-ref"] < b["bom-ref"] ? -1 : a["bom-ref"] > b["bom-ref"] ? 1 : 0,
+  );
+}
+
+/**
+ * Add the packages of the host's Python environments to an OBOM's component
+ * list. A package osquery already listed from the interpreter's own
+ * site-packages keeps its component; the environment's dist-info becomes one
+ * more occurrence of it, and its cdx:python:environment property says where
+ * else it is installed.
+ *
+ * @param {Object[]} components OBOM components
+ * @param {Object[]} envPackages result of probeHostPythonEnvironments
+ * @returns {Object[]} the components, with the environment packages merged in
+ */
+export function mergeHostPythonEnvironmentPackages(components, envPackages) {
+  if (!envPackages?.length) {
+    return components;
+  }
+  const byPurl = new Map();
+  for (const component of components) {
+    if (typeof component?.purl === "string") {
+      byPurl.set(decodeURIComponent(component.purl), component);
+    }
+  }
+  const added = [];
+  for (const pkg of envPackages) {
+    const existing = byPurl.get(decodeURIComponent(pkg.purl));
+    if (!existing) {
+      added.push(pkg);
+      continue;
+    }
+    existing.evidence = existing.evidence || {};
+    const occurrences = existing.evidence.occurrences || [];
+    for (const occurrence of pkg.evidence?.occurrences || []) {
+      if (!occurrences.some((o) => o.location === occurrence.location)) {
+        occurrences.push(occurrence);
+      }
+    }
+    existing.evidence.occurrences = occurrences;
+    existing.properties = existing.properties || [];
+    for (const prop of pkg.properties || []) {
+      if (
+        !existing.properties.some(
+          (p) => p.name === prop.name && p.value === prop.value,
+        )
+      ) {
+        existing.properties.push(prop);
+      }
+    }
+  }
+  return added.length ? components.concat(added) : components;
+}