@cdxgen/cdxgen 13.0.1 → 13.1.0

package/lib/ecosystems/pythonutils.js

@@ -331,6 +331,53 @@ export function get_python_command_from_env(env) {
   return fallbackCmd;
 }
 
+/**
+ * Is this import satisfied by a module that lives in the project itself?
+ *
+ * `atom parsedeps` reports every non-stdlib import it sees, including imports of the
+ * project's own packages. Attributing those to a distribution invents dependencies:
+ * `data/pypi-pkg-aliases.json` maps the module `app` to `zope2`, `test` to `pytabix`,
+ * `models` to `asposestorage` and `src` to `auto-mix-prep`, so any project with a
+ * local package of that name acquires an obscure component it never depended on -
+ * along with every namespace and tag that component carries downstream. Filtering by
+ * name would need a denylist of every plausible first-party name; asking the
+ * filesystem whether the module is ours answers it exactly.
+ *
+ * Checks the project root and a `src/` layout for a module file, a package directory,
+ * and a PEP 420 namespace package directory.
+ *
+ * @param {string} src Path to the project being analysed
+ * @param {string} name Imported top-level module name
+ * @returns {boolean} `true` if the project supplies this module itself
+ */
+export function isFirstPartyModule(src, name) {
+  if (!src || !name || name.includes("/") || name.includes("\\")) {
+    return false;
+  }
+  for (const root of [src, join(src, "src")]) {
+    if (
+      safeExistsSync(join(root, `${name}.py`)) ||
+      safeExistsSync(join(root, name, "__init__.py")) ||
+      safeExistsSync(join(root, name, "__main__.py"))
+    ) {
+      return true;
+    }
+    // PEP 420 namespace package: a directory of the right name holding python
+    // sources but no __init__.py.
+    const candidate = join(root, name);
+    if (safeExistsSync(candidate)) {
+      try {
+        if (readdirSync(candidate).some((f) => f.endsWith(".py"))) {
+          return true;
+        }
+      } catch {
+        // not a readable directory - treat as not first-party
+      }
+    }
+  }
+  return false;
+}
+
 /**
  * Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
  *
@@ -362,7 +409,8 @@ export async function getPyModules(src, epkgList, options) {
     (x) =>
       !pyDefaultModules.has(x.name.toLowerCase()) &&
       !x.name.startsWith("_") &&
-      !x.name.startsWith("."),
+      !x.name.startsWith(".") &&
+      !isFirstPartyModule(src, x.name),
   );
   let pkgList = modList.map((p) => {
     const apkg = {

package/lib/ecosystems/rubyutils.js

@@ -19,6 +19,7 @@ import {
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
+  recordPolicyDegradationFromError,
 } from "../inventory/fetchBatch.js";
 
 // FIXME. This has to get removed, once we improve the module detection one-liner.
@@ -780,6 +781,7 @@ export async function getRubyGemsMetadata(pkgList) {
           continue;
         }
       } catch (err) {
+        recordPolicyDegradationFromError(err);
         if (DEBUG_MODE) {
           console.error(name, err);
         }
@@ -832,6 +834,7 @@ async function enrichGemFromVersionEndpoint(
     }
     applyGemMetadata(p, body);
   } catch (err) {
+    recordPolicyDegradationFromError(err);
     if (DEBUG_MODE) {
       console.error(p, err);
     }
@@ -1241,6 +1244,16 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
       pkg.externalReferences.push({ type: refType, url: value, comment: key });
     }
   }
+  // `s.homepage = "..."` is the project website. Keep it as an external
+  // reference; the transient `homepage` key is a plain string that never
+  // reaches the BOM because component conversion expects the `{ url }` shape.
+  if (pkg.homepage) {
+    pkg.externalReferences = pkg.externalReferences || [];
+    if (!pkg.externalReferences.some((ref) => ref.url === pkg.homepage)) {
+      pkg.externalReferences.push({ type: "website", url: pkg.homepage });
+    }
+    delete pkg.homepage;
+  }
   if (pkg.name) {
     const purlString = toGemPurl(pkg.name, pkg.version, gemPlatform);
     pkg.purl = purlString;

package/lib/evinser/evinser.js

@@ -42,12 +42,15 @@ import {
 import { collectMvnDependencies } from "../inventory/deps.js";
 import { mergeServices } from "../inventory/depsUtils.js";
 import {
+  collectDosaiAiComponents,
   collectDosaiDataFlowFrames,
   collectDosaiPurlEvidence,
+  collectDosaiServiceComponents,
   collectDosaiServicesFromMethods,
   createDosaiDataFlowSlice,
   createDosaiMethodsSlice,
   isDosaiDotnetLanguage,
+  normalizeDosaiServiceMap,
   persistDosaiSemanticsReport,
 } from "../inventory/dosai.js";
 import { parseOccurrenceEvidenceLocation } from "../inventory/evidenceUtils.js";
@@ -662,6 +665,7 @@ export async function analyzeProject(dbObjMap, options) {
   const bomJson = JSON.parse(fs.readFileSync(bomFile, "utf8"));
   const components = bomJson.components || [];
   let cryptoComponents = [];
+  let aiComponents = [];
   let cryptoGeneratePurls = {};
   let openapiSpecFile;
   // Load any existing purl-location information from the sbom.
@@ -750,6 +754,13 @@ export async function analyzeProject(dbObjMap, options) {
           purlLocationMap[purl].add(location);
         }
       }
+      // Rich Services[] inventory first (bom-refs, trust zones, data); endpoint
+      // derivation fills any services the providers did not emit.
+      // AI components must be carried out of analyzeProject explicitly. Mutating the local
+      // `components` array here had no effect: the return below omits it and createEvinseFile
+      // re-reads the BOM from disk, so the whole AI-BOM contribution was silently discarded.
+      collectDosaiAiComponents(usageSlice, aiComponents);
+      servicesMap = collectDosaiServiceComponents(usageSlice, servicesMap);
       servicesMap = collectDosaiServicesFromMethods(usageSlice, servicesMap);
       userDefinedTypesMap = {};
     }
@@ -802,6 +813,7 @@ export async function analyzeProject(dbObjMap, options) {
       tempDirOwned: retMap?.tempDirOwned,
       userDefinedTypesMap,
       cryptoComponents,
+      aiComponents,
       cryptoGeneratePurls,
     };
   }
@@ -832,6 +844,7 @@ export async function analyzeProject(dbObjMap, options) {
     const retMap = collectReachableFrames(language, reachablesSlice);
     dataFlowFrames = retMap.dataFlowFrames;
     cryptoComponents = retMap.cryptoComponents;
+    aiComponents = retMap.aiComponents || [];
     cryptoGeneratePurls = retMap.cryptoGeneratePurls;
   }
   // Reuse existing usages slices
@@ -960,6 +973,7 @@ export async function analyzeProject(dbObjMap, options) {
     tempDirOwned: retMap?.tempDirOwned,
     userDefinedTypesMap,
     cryptoComponents,
+    aiComponents,
     cryptoGeneratePurls,
     openapiSpecFile,
   };
@@ -1804,7 +1818,7 @@ export function extractEndpoints(language, code) {
  * @param {Object} options Command line options
  * @returns
  */
-export function createEvinseFile(sliceArtefacts, options) {
+export async function createEvinseFile(sliceArtefacts, options) {
   const {
     tempDir,
     tempDirOwned,
@@ -1817,6 +1831,7 @@ export function createEvinseFile(sliceArtefacts, options) {
     componentPropertiesMap = {},
     metadataProperties = [],
     cryptoComponents,
+    aiComponents,
     cryptoGeneratePurls,
   } = sliceArtefacts;
   const bomFile = options.input;
@@ -1873,16 +1888,12 @@ export function createEvinseFile(sliceArtefacts, options) {
     }
   } // for
   if (servicesMap && Object.keys(servicesMap).length) {
-    const services = [];
-    for (const serviceName of Object.keys(servicesMap)) {
-      services.push({
-        name: serviceName,
-        endpoints: Array.from(servicesMap[serviceName].endpoints),
-        authenticated: servicesMap[serviceName].authenticated,
-        "x-trust-boundary": servicesMap[serviceName].xTrustBoundary,
-        properties: servicesMap[serviceName].properties,
-      });
-    }
+    // normalizeDosaiServiceMap is the single mapping from the internal service
+    // definitions to CycloneDX services[]. The OpenAPI/Java/Python detectors set
+    // the camelCase `xTrustBoundary` while dosai's provider inventory (schema
+    // 4.0.0) carries stable bom-refs, trust zones, data classifications, and
+    // evidence occurrences; both shapes flow through the same normalizer.
+    const services = normalizeDosaiServiceMap(servicesMap);
     // Add to existing services while preserving CycloneDX uniqueItems validity
     bomJson.services = mergeServices(bomJson.services || [], services);
     servicesPresent = true;
@@ -1891,6 +1902,10 @@ export function createEvinseFile(sliceArtefacts, options) {
   if (cryptoComponents?.length) {
     bomJson.components = bomJson.components.concat(cryptoComponents);
   }
+  // Add the AI components (machine-learning-model, dataset) discovered by dosai
+  if (aiComponents?.length) {
+    bomJson.components = bomJson.components.concat(aiComponents);
+  }
   // Fix the dependencies section with provides information
   if (
     cryptoGeneratePurls &&
@@ -1943,7 +1958,10 @@ export function createEvinseFile(sliceArtefacts, options) {
   // Redo post-processing with evinse data. `executeOsQuery` has to be injected
   // because postgen no longer imports it directly (stages is below managers);
   // without it the crypto formulation path receives `undefined` and throws.
-  const bomNSData = postProcess({ bomJson }, { ...options, executeOsQuery });
+  const bomNSData = await postProcess(
+    { bomJson },
+    { ...options, executeOsQuery },
+  );
   safeWriteSync(
     evinseOutFile,
     JSON.stringify(bomNSData.bomJson, null, options.jsonPretty ? 2 : null),

package/lib/helpers/remote/dependency-track.js

@@ -1,4 +1,5 @@
 import { Buffer } from "node:buffer";
+import { randomBytes } from "node:crypto";
 
 import { hasDangerousUnicode } from "../../core/activity.js";
 
@@ -44,26 +45,23 @@ export function getDependencyTrackBomUrl(serverUrl) {
 }
 
 /**
- * Build the payload for Dependency-Track BOM submission.
+ * Build the form fields for a Dependency-Track BOM submission.
+ *
+ * Field names follow the `multipart/form-data` variant of `/api/v1/bom`, which differs
+ * from the JSON variant: the JSON body calls the flag `isLatestProjectVersion` and takes
+ * `projectTags` as an array of objects, while the form takes `isLatest` and a
+ * comma-separated tag list.
  *
  * @param {Object} args CLI/server arguments
- * @param {Object} bomContents BOM Json
- * @returns {Object | undefined} payload object if project coordinates are valid
+ * @returns {Object | undefined} field map if the project coordinates are valid
  */
-export function buildDependencyTrackBomPayload(args, bomContents) {
-  let encodedBomContents = Buffer.from(JSON.stringify(bomContents)).toString(
-    "base64",
-  );
-  if (encodedBomContents.startsWith("77u/")) {
-    encodedBomContents = encodedBomContents.substring(4);
-  }
+export function buildDependencyTrackBomPayload(args) {
   const autoCreate =
     typeof args.autoCreate === "boolean"
       ? args.autoCreate
       : args.autoCreate !== "false";
   const bomPayload = {
     autoCreate: String(autoCreate),
-    bom: encodedBomContents,
   };
   if (
     typeof args.projectId !== "undefined" ||
@@ -103,15 +101,84 @@ export function buildDependencyTrackBomPayload(args, bomContents) {
     args.isLatest === "true" ||
     args.isLatest === "false"
   ) {
-    bomPayload.isLatest =
+    bomPayload.isLatest = String(
       typeof args.isLatest === "boolean"
         ? args.isLatest
-        : args.isLatest === "true";
+        : args.isLatest === "true",
+    );
   }
   if (typeof args.projectTag !== "undefined") {
+    // The server splits this field on commas, so a tag cannot itself contain one.
     bomPayload.projectTags = (
       Array.isArray(args.projectTag) ? args.projectTag : [args.projectTag]
-    ).map((tag) => ({ name: tag }));
+    ).join(",");
   }
   return bomPayload;
 }
+
+/**
+ * Encode text fields and a single file part as a `multipart/form-data` body.
+ *
+ * @param {Object} fields Text fields keyed by field name
+ * @param {Object} file File part
+ * @param {string} file.name Field name of the file part
+ * @param {string} file.filename File name reported to the server
+ * @param {string} file.contentType Content type of the file part
+ * @param {Buffer} file.content File content
+ * @returns {{ body: Buffer, contentType: string }} encoded body and its content type
+ */
+export function encodeMultipartFormData(fields, file) {
+  // A random boundary keeps the delimiter from colliding with the BOM content.
+  const boundary = `--------------------------cdxgen${randomBytes(16).toString("hex")}`;
+  const chunks = [];
+  const pushPart = (header, content) => {
+    chunks.push(
+      Buffer.from(`--${boundary}\r\n${header}\r\n\r\n`),
+      Buffer.isBuffer(content) ? content : Buffer.from(String(content)),
+      Buffer.from("\r\n"),
+    );
+  };
+  for (const [name, value] of Object.entries(fields)) {
+    pushPart(`Content-Disposition: form-data; name="${name}"`, value);
+  }
+  pushPart(
+    `Content-Disposition: form-data; name="${file.name}"; filename="${file.filename}"\r\nContent-Type: ${file.contentType}`,
+    file.content,
+  );
+  chunks.push(Buffer.from(`--${boundary}--\r\n`));
+  return {
+    body: Buffer.concat(chunks),
+    contentType: `multipart/form-data; boundary=${boundary}`,
+  };
+}
+
+/**
+ * Build the `multipart/form-data` request for Dependency-Track BOM submission.
+ *
+ * The JSON variant of the endpoint carries the BOM as a base64 string, which Jackson
+ * refuses to read beyond 20,000,000 characters - roughly a 15 MB BOM, which container
+ * image scans exceed routinely. The form variant carries the BOM as a file part and has
+ * no such limit, and is what the server recommends for large uploads.
+ *
+ * @param {Object} args CLI/server arguments
+ * @param {Object | string} bomContents BOM Json
+ * @returns {{ body: Buffer, contentType: string } | undefined} request body and content type if the arguments are valid
+ */
+export function buildDependencyTrackBomRequest(args, bomContents) {
+  if (!bomContents) {
+    return undefined;
+  }
+  const fields = buildDependencyTrackBomPayload(args);
+  if (!fields) {
+    return undefined;
+  }
+  const bomString =
+    typeof bomContents === "string" ? bomContents : JSON.stringify(bomContents);
+  return encodeMultipartFormData(fields, {
+    name: "bom",
+    filename: "bom.json",
+    contentType: "application/vnd.cyclonedx+json",
+    // The server rejects a BOM that still carries a byte order mark.
+    content: Buffer.from(bomString.replace(/^\ufeff/, "")),
+  });
+}

package/lib/inventory/auditCategories.js

@@ -23,6 +23,15 @@ export const GOLEM_AUDIT_CATEGORIES = Object.freeze([
   "golem-compliance",
 ]);
 
+/**
+ * Frozen list of audit categories that never activate on their own. These
+ * rule packs serve a specific feature (`--introspect` for build-fidelity) and
+ * stay inactive unless the caller explicitly requests the category, so a
+ * default `--bom-audit` run - which evaluates every loaded rule - cannot pick
+ * them up.
+ */
+export const OPT_IN_BOM_AUDIT_CATEGORIES = Object.freeze(["build-fidelity"]);
+
 /** Frozen list of AI-bill-of-materials audit categories. */
 export const AI_BOM_AUDIT_CATEGORIES = Object.freeze([
   "ai-governance",

package/lib/inventory/depsUtils.js

@@ -222,6 +222,28 @@ function serviceIdentityKey(service) {
   return `${service?.group || ""}:${service?.name || ""}:${service?.version || ""}`.toLowerCase();
 }
 
+/**
+ * Union two services[].data[] lists, keyed by flow + classification + name so that repeated
+ * classifications of the same payload do not accumulate duplicates.
+ *
+ * @param {Array} existingData Data entries already on the service
+ * @param {Array} newData Data entries being merged in
+ * @returns {Array} Merged data entries
+ */
+function mergeServiceData(existingData = [], newData = []) {
+  const seen = new Map();
+  for (const entry of [...existingData, ...newData]) {
+    if (!entry?.flow || !entry?.classification) {
+      continue;
+    }
+    const key = `${entry.flow}|${entry.classification}|${entry.name || ""}`;
+    if (!seen.has(key)) {
+      seen.set(key, entry);
+    }
+  }
+  return Array.from(seen.values());
+}
+
 function mergeServiceProperties(existingProps = [], newProps = []) {
   const merged = [...existingProps];
   for (const newProp of newProps) {
@@ -249,6 +271,38 @@ function normalizeServiceEndpoints(endpoints) {
   return [];
 }
 
+function evidenceOccurrenceKey(occurrence) {
+  const location = occurrence?.location || {};
+  return `${location.path || ""}#${location.line ?? ""}#${location.column ?? ""}`;
+}
+
+/**
+ * Union two service evidence objects by their occurrences so a collision keeps
+ * every call site instead of whichever side arrived first.
+ *
+ * @param {Object} [existingEvidence] Evidence already on the service
+ * @param {Object} [newEvidence] Evidence being merged in
+ * @returns {Object} Evidence covering both inputs
+ */
+function mergeServiceEvidence(existingEvidence, newEvidence) {
+  if (!existingEvidence) {
+    return newEvidence;
+  }
+  const occurrences = [...(existingEvidence.occurrences || [])];
+  const seen = new Set(occurrences.map(evidenceOccurrenceKey));
+  for (const occurrence of newEvidence?.occurrences || []) {
+    const key = evidenceOccurrenceKey(occurrence);
+    if (!seen.has(key)) {
+      seen.add(key);
+      occurrences.push(occurrence);
+    }
+  }
+  if (!occurrences.length) {
+    return existingEvidence;
+  }
+  return { ...existingEvidence, occurrences };
+}
+
 /**
  * Merge CycloneDX services using bom-ref or group/name/version identity.
  *
@@ -313,6 +367,24 @@ export function mergeServices(services, newServices) {
       existing.properties || [],
       service.properties || [],
     );
+    // Carry the richer fields through a collision too. Without these, a dosai service that collided
+    // with one derived from endpoints lost precisely what schema 4.0.0 adds: its bom-ref, its data
+    // classifications, and its evidence.
+    existing["bom-ref"] = existing["bom-ref"] || service["bom-ref"];
+    if (Array.isArray(service.data) && service.data.length) {
+      existing.data = mergeServiceData(existing.data || [], service.data);
+    }
+    if (service.evidence) {
+      existing.evidence = mergeServiceEvidence(
+        existing.evidence,
+        service.evidence,
+      );
+    }
+    if (Array.isArray(service.tags) && service.tags.length) {
+      existing.tags = Array.from(
+        new Set([...(existing.tags || []), ...service.tags]),
+      );
+    }
     if (Array.isArray(service.services) && service.services.length) {
       existing.services = mergeServices(
         existing.services || [],