@cdxgen/cdxgen 13.0.1 → 13.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +26 -13
  2. package/bin/audit.js +6 -0
  3. package/bin/cdxgen.js +125 -4
  4. package/bin/evinse.js +1 -1
  5. package/data/component-tags.json +8 -1
  6. package/data/frameworks-list.json +86 -1
  7. package/data/remediations.json +1144 -0
  8. package/data/rules/asar-archives.yaml +1 -1
  9. package/data/rules/build-fidelity.yaml +782 -0
  10. package/data/rules/package-integrity.yaml +13 -13
  11. package/lib/audit/index.js +48 -4
  12. package/lib/audit/scoring.js +2 -8
  13. package/lib/cli/bomAssembly.js +20 -1
  14. package/lib/cli/cliOptions.js +94 -7
  15. package/lib/cli/index.js +76 -48
  16. package/lib/cli/jsBom.js +32 -3
  17. package/lib/cli/jvmBom.js +1494 -1315
  18. package/lib/cli/managedBom.js +97 -28
  19. package/lib/cli/nativeBom.js +189 -10
  20. package/lib/core/buildLedger.js +799 -0
  21. package/lib/core/deferredExit.js +141 -0
  22. package/lib/core/env.js +69 -1
  23. package/lib/core/fs.js +60 -0
  24. package/lib/core/logger.js +12 -99
  25. package/lib/core/severity.js +16 -0
  26. package/lib/core/syncFileWriter.js +106 -0
  27. package/lib/ecosystems/ecosystems.js +9 -0
  28. package/lib/ecosystems/gradleutils.js +355 -15
  29. package/lib/ecosystems/parsers-js.js +26 -0
  30. package/lib/ecosystems/parsers-jvm.js +9 -0
  31. package/lib/ecosystems/parsers-misc.js +16 -2
  32. package/lib/ecosystems/parsers-python.js +171 -35
  33. package/lib/ecosystems/parsers-rust.js +12 -2
  34. package/lib/ecosystems/pypiClassifiers.js +326 -0
  35. package/lib/ecosystems/pythonutils.js +49 -1
  36. package/lib/ecosystems/rubyutils.js +13 -0
  37. package/lib/evinser/evinser.js +30 -12
  38. package/lib/helpers/remote/dependency-track.js +81 -14
  39. package/lib/inventory/auditCategories.js +9 -0
  40. package/lib/inventory/depsUtils.js +72 -0
  41. package/lib/inventory/dosai.js +431 -22
  42. package/lib/inventory/envcontext.js +206 -18
  43. package/lib/inventory/fetchBatch.js +57 -0
  44. package/lib/inventory/jvmToolEnv.js +593 -0
  45. package/lib/inventory/osinfo.js +1 -0
  46. package/lib/inventory/toolRequirements.js +903 -0
  47. package/lib/managers/binary.js +24 -0
  48. package/lib/managers/docker.js +26 -5
  49. package/lib/server/server.js +1 -1
  50. package/lib/stages/postgen/annotationBuilder.js +97 -0
  51. package/lib/stages/postgen/auditBom.js +76 -79
  52. package/lib/stages/postgen/introspection/annotate.js +320 -0
  53. package/lib/stages/postgen/introspection/commandFacts.js +293 -0
  54. package/lib/stages/postgen/introspection/emit.js +192 -0
  55. package/lib/stages/postgen/introspection/formulationEvidence.js +427 -0
  56. package/lib/stages/postgen/introspection/markers.js +227 -0
  57. package/lib/stages/postgen/introspection/reflect.js +1214 -0
  58. package/lib/stages/postgen/introspection/report.js +1208 -0
  59. package/lib/stages/postgen/introspection/score.js +1067 -0
  60. package/lib/stages/postgen/introspection/shapeCommand.js +304 -0
  61. package/lib/stages/postgen/parentEdge.js +156 -0
  62. package/lib/stages/postgen/postgen.js +115 -2
  63. package/lib/stages/postgen/ruleEngine.js +17 -0
  64. package/lib/stages/pregen/envAudit.js +42 -2
  65. package/lib/stages/pregen/pregen.js +1171 -6
  66. package/lib/validator/bomValidator.js +11 -2
  67. package/package.json +14 -14
  68. package/types/lib/audit/index.d.ts.map +1 -1
  69. package/types/lib/audit/scoring.d.ts +2 -8
  70. package/types/lib/audit/scoring.d.ts.map +1 -1
  71. package/types/lib/cli/bomAssembly.d.ts.map +1 -1
  72. package/types/lib/cli/cliOptions.d.ts +2 -1
  73. package/types/lib/cli/cliOptions.d.ts.map +1 -1
  74. package/types/lib/cli/index.d.ts +8 -0
  75. package/types/lib/cli/index.d.ts.map +1 -1
  76. package/types/lib/cli/jsBom.d.ts.map +1 -1
  77. package/types/lib/cli/jvmBom.d.ts.map +1 -1
  78. package/types/lib/cli/managedBom.d.ts.map +1 -1
  79. package/types/lib/cli/nativeBom.d.ts.map +1 -1
  80. package/types/lib/core/buildLedger.d.ts +272 -0
  81. package/types/lib/core/buildLedger.d.ts.map +1 -0
  82. package/types/lib/core/deferredExit.d.ts +84 -0
  83. package/types/lib/core/deferredExit.d.ts.map +1 -0
  84. package/types/lib/core/env.d.ts +22 -1
  85. package/types/lib/core/env.d.ts.map +1 -1
  86. package/types/lib/core/fs.d.ts +21 -0
  87. package/types/lib/core/fs.d.ts.map +1 -1
  88. package/types/lib/core/logger.d.ts +2 -11
  89. package/types/lib/core/logger.d.ts.map +1 -1
  90. package/types/lib/core/severity.d.ts +11 -0
  91. package/types/lib/core/severity.d.ts.map +1 -0
  92. package/types/lib/core/syncFileWriter.d.ts +33 -0
  93. package/types/lib/core/syncFileWriter.d.ts.map +1 -0
  94. package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
  95. package/types/lib/ecosystems/gradleutils.d.ts +33 -0
  96. package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
  97. package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
  98. package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
  99. package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
  100. package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
  101. package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
  102. package/types/lib/ecosystems/pypiClassifiers.d.ts +80 -0
  103. package/types/lib/ecosystems/pypiClassifiers.d.ts.map +1 -0
  104. package/types/lib/ecosystems/pythonutils.d.ts +20 -0
  105. package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
  106. package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
  107. package/types/lib/evinser/evinser.d.ts +5 -1
  108. package/types/lib/evinser/evinser.d.ts.map +1 -1
  109. package/types/lib/helpers/remote/dependency-track.d.ts +45 -4
  110. package/types/lib/helpers/remote/dependency-track.d.ts.map +1 -1
  111. package/types/lib/inventory/auditCategories.d.ts +8 -0
  112. package/types/lib/inventory/auditCategories.d.ts.map +1 -1
  113. package/types/lib/inventory/depsUtils.d.ts.map +1 -1
  114. package/types/lib/inventory/dosai.d.ts +20 -0
  115. package/types/lib/inventory/dosai.d.ts.map +1 -1
  116. package/types/lib/inventory/envcontext.d.ts.map +1 -1
  117. package/types/lib/inventory/fetchBatch.d.ts +9 -0
  118. package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
  119. package/types/lib/inventory/jvmToolEnv.d.ts +189 -0
  120. package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -0
  121. package/types/lib/inventory/osinfo.d.ts.map +1 -1
  122. package/types/lib/inventory/toolRequirements.d.ts +216 -0
  123. package/types/lib/inventory/toolRequirements.d.ts.map +1 -0
  124. package/types/lib/managers/binary.d.ts.map +1 -1
  125. package/types/lib/managers/docker.d.ts.map +1 -1
  126. package/types/lib/stages/postgen/annotationBuilder.d.ts +53 -0
  127. package/types/lib/stages/postgen/annotationBuilder.d.ts.map +1 -0
  128. package/types/lib/stages/postgen/auditBom.d.ts.map +1 -1
  129. package/types/lib/stages/postgen/introspection/annotate.d.ts +81 -0
  130. package/types/lib/stages/postgen/introspection/annotate.d.ts.map +1 -0
  131. package/types/lib/stages/postgen/introspection/commandFacts.d.ts +32 -0
  132. package/types/lib/stages/postgen/introspection/commandFacts.d.ts.map +1 -0
  133. package/types/lib/stages/postgen/introspection/emit.d.ts +71 -0
  134. package/types/lib/stages/postgen/introspection/emit.d.ts.map +1 -0
  135. package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts +154 -0
  136. package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts.map +1 -0
  137. package/types/lib/stages/postgen/introspection/markers.d.ts +57 -0
  138. package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -0
  139. package/types/lib/stages/postgen/introspection/reflect.d.ts +77 -0
  140. package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -0
  141. package/types/lib/stages/postgen/introspection/report.d.ts +85 -0
  142. package/types/lib/stages/postgen/introspection/report.d.ts.map +1 -0
  143. package/types/lib/stages/postgen/introspection/score.d.ts +149 -0
  144. package/types/lib/stages/postgen/introspection/score.d.ts.map +1 -0
  145. package/types/lib/stages/postgen/introspection/shapeCommand.d.ts +79 -0
  146. package/types/lib/stages/postgen/introspection/shapeCommand.d.ts.map +1 -0
  147. package/types/lib/stages/postgen/parentEdge.d.ts +35 -0
  148. package/types/lib/stages/postgen/parentEdge.d.ts.map +1 -0
  149. package/types/lib/stages/postgen/postgen.d.ts +2 -2
  150. package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
  151. package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
  152. package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
  153. package/types/lib/stages/pregen/pregen.d.ts +131 -0
  154. package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
  155. package/types/lib/validator/bomValidator.d.ts.map +1 -1
@@ -8,6 +8,7 @@ import toml from "@iarna/toml";
8
8
  import { parse as _load } from "yaml";
9
9
 
10
10
  import { DEBUG_MODE } from "../core/activity.js";
11
+ import { recordDegradation } from "../core/buildLedger.js";
11
12
  import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
12
13
  import { PYTHON_STD_MODULES } from "../core/state.js";
13
14
  import { tryBuildPurl } from "../inventory/purl.js";
@@ -27,6 +28,41 @@ import {
27
28
  normalizePyLockRegistry,
28
29
  } from "./pylockutils.js";
29
30
 
31
+ /**
32
+ * TOML lock files a specific Python manager owns, mapped to the manager and
33
+ * the remediation id that ranks its repair. The failing file itself is the
34
+ * detection: the lock command that regenerates it belongs to the manager
35
+ * that wrote it.
36
+ *
37
+ * @type {Record<string, {manager: string, remediationId: string}>}
38
+ */
39
+ const PY_LOCKFILE_DEGRADATIONS = {
40
+ "uv.lock": {
41
+ manager: "uv",
42
+ remediationId: "python.lockfile-unparseable.uv",
43
+ },
44
+ "poetry.lock": {
45
+ manager: "poetry",
46
+ remediationId: "python.lockfile-unparseable.poetry",
47
+ },
48
+ "pdm.lock": {
49
+ manager: "pdm",
50
+ remediationId: "python.lockfile-unparseable.pdm",
51
+ },
52
+ };
53
+
54
+ /**
55
+ * The repair for a TOML lock file that names no single manager, such as
56
+ * PEP 751's `pylock.toml`. Its lock command carries the manager variable, so
57
+ * the shaping fills in whichever manager the project turns out to use.
58
+ *
59
+ * @type {{manager: string, remediationId: string}}
60
+ */
61
+ const GENERIC_PY_LOCKFILE_DEGRADATION = {
62
+ manager: "python",
63
+ remediationId: "python.lockfile-unparseable",
64
+ };
65
+
30
66
  /**
31
67
  * Method to parse python requires_dist attribute found in pypi setup.py
32
68
  *
@@ -444,7 +480,12 @@ export function parsePyProjectTomlFile(tomlFile) {
444
480
  for (const agroup of Object.keys(tomlData["dependency-groups"])) {
445
481
  tomlData["dependency-groups"][agroup].forEach((p) => {
446
482
  if (typeof p === "string" || p instanceof String) {
447
- const pname = p.split(/(==|<=|~=|>=)/)[0].split(" ")[0];
483
+ const pname = normalizePythonDependencyKey(
484
+ p.split(/(==|<=|~=|>=)/)[0].split(" ")[0],
485
+ );
486
+ if (!pname) {
487
+ return;
488
+ }
448
489
  if (!groupDepsKeys[pname]) {
449
490
  groupDepsKeys[pname] = [];
450
491
  }
@@ -466,8 +507,13 @@ export function parsePyProjectTomlFile(tomlFile) {
466
507
  }
467
508
  if (tomlData?.tool?.poetry?.dependencies) {
468
509
  for (const adep of Object.keys(tomlData?.tool?.poetry?.dependencies)) {
510
+ // poetry keys keep the author's casing (Click = "^8.1"), while every
511
+ // consumer of directDepsKeys looks entries up with the normalised
512
+ // (lowercase, hyphenated) package name, so normalise at the source.
513
+ const poetryDepKey = normalizePythonDependencyKey(adep);
469
514
  if (
470
- ![
515
+ !poetryDepKey ||
516
+ [
471
517
  "python",
472
518
  "py",
473
519
  "pytest",
@@ -475,43 +521,50 @@ export function parsePyProjectTomlFile(tomlFile) {
475
521
  "ruff",
476
522
  "setuptools",
477
523
  "bandit",
478
- ].includes(adep)
524
+ ].includes(poetryDepKey)
479
525
  ) {
480
- directDepsKeys[adep] = true;
481
- const poetryDependency = tomlData.tool.poetry.dependencies[adep];
482
- if (poetryDependency?.git) {
483
- recordPythonDependencySource(
484
- dependencySourceMap,
485
- adep,
486
- "git",
487
- poetryDependency.git,
488
- );
489
- } else if (poetryDependency?.url) {
490
- recordPythonDependencySource(
491
- dependencySourceMap,
492
- adep,
493
- "url",
494
- poetryDependency.url,
495
- );
496
- } else if (poetryDependency?.path) {
497
- recordPythonDependencySource(
498
- dependencySourceMap,
499
- adep,
500
- "path",
501
- poetryDependency.path,
502
- );
503
- }
526
+ continue;
527
+ }
528
+ directDepsKeys[poetryDepKey] = true;
529
+ const poetryDependency = tomlData.tool.poetry.dependencies[adep];
530
+ if (poetryDependency?.git) {
531
+ recordPythonDependencySource(
532
+ dependencySourceMap,
533
+ adep,
534
+ "git",
535
+ poetryDependency.git,
536
+ );
537
+ } else if (poetryDependency?.url) {
538
+ recordPythonDependencySource(
539
+ dependencySourceMap,
540
+ adep,
541
+ "url",
542
+ poetryDependency.url,
543
+ );
544
+ } else if (poetryDependency?.path) {
545
+ recordPythonDependencySource(
546
+ dependencySourceMap,
547
+ adep,
548
+ "path",
549
+ poetryDependency.path,
550
+ );
504
551
  }
505
552
  } // for
553
+ // Group keys are matched against the normalised package names from the
554
+ // lock file, so they are normalised here too.
506
555
  if (tomlData?.tool?.poetry?.group) {
507
556
  for (const agroup of Object.keys(tomlData.tool.poetry.group)) {
508
557
  for (const adep of Object.keys(
509
558
  tomlData.tool.poetry.group[agroup]?.dependencies,
510
559
  )) {
511
- if (!groupDepsKeys[adep]) {
512
- groupDepsKeys[adep] = [];
560
+ const poetryGroupDepKey = normalizePythonDependencyKey(adep);
561
+ if (!poetryGroupDepKey) {
562
+ continue;
513
563
  }
514
- groupDepsKeys[adep].push(agroup);
564
+ if (!groupDepsKeys[poetryGroupDepKey]) {
565
+ groupDepsKeys[poetryGroupDepKey] = [];
566
+ }
567
+ groupDepsKeys[poetryGroupDepKey].push(agroup);
515
568
  const poetryDependency =
516
569
  tomlData.tool.poetry.group[agroup]?.dependencies?.[adep];
517
570
  if (poetryDependency?.git) {
@@ -780,7 +833,27 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
780
833
  dependencySourceMap = pyProjMap.dependencySourceMap || {};
781
834
  parentComponent = pyProjMap.parentComponent;
782
835
  workspacePaths = pyProjMap.workspacePaths;
783
- if (workspacePaths?.length) {
836
+ // `parsePyProjectTomlFile` reports a parse failure and returns an empty map, so a
837
+ // pyproject.toml this parser cannot read leaves `parentComponent` undefined while
838
+ // the lock file still declares workspaces. Building the workspace tree then throws
839
+ // on the first dereference and takes the whole SBOM with it. Degrade to a flattened
840
+ // tree instead - the caller supplies its own parent component, so the components
841
+ // themselves are still reported.
842
+ if (workspacePaths?.length && !parentComponent) {
843
+ console.log(
844
+ `Unable to read a parent component from ${pyProjectFile}, so the workspace structure declared in ${basename(lockFile)} cannot be reconstructed.`,
845
+ );
846
+ console.log(
847
+ "The dependency tree in the generated SBOM will be flattened and therefore incorrect.",
848
+ );
849
+ recordDegradation("python.pyproject-unparseable", {
850
+ ecosystem: "python",
851
+ impact: "transitive-deps",
852
+ path: pyProjectFile,
853
+ detail: `${basename(pyProjectFile)} could not be parsed, so the workspace structure declared in ${basename(lockFile)} was not reconstructed.`,
854
+ });
855
+ }
856
+ if (workspacePaths?.length && parentComponent) {
784
857
  if (!hasWorkspaces) {
785
858
  hasWorkspaces = true;
786
859
  }
@@ -840,7 +913,12 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
840
913
  ] = awpyproj;
841
914
  }
842
915
  }
843
- const wparentComponentRef = wcompMap.parentComponent["bom-ref"];
916
+ // The block above already treats an unreadable workspace pyproject.toml as
917
+ // tolerable, so this must not dereference the component it skipped. A
918
+ // workspace member whose manifest cannot be parsed contributes no parent
919
+ // ref, and recording `undefined` as the parent of its dependencies would
920
+ // corrupt the tree for the members that did parse.
921
+ const wparentComponentRef = wcompMap?.parentComponent?.["bom-ref"];
844
922
  if (wcompMap?.dependencySourceMap) {
845
923
  Object.assign(
846
924
  workspaceDependencySourceMap,
@@ -848,7 +926,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
848
926
  );
849
927
  }
850
928
  // Track the parents of workspace direct dependencies
851
- if (wcompMap?.directDepsKeys) {
929
+ if (wcompMap?.directDepsKeys && wparentComponentRef) {
852
930
  for (const wdd of Object.keys(wcompMap?.directDepsKeys)) {
853
931
  if (!pkgParentMap[wdd]) {
854
932
  pkgParentMap[wdd] = [];
@@ -865,6 +943,20 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
865
943
  lockTomlObj = toml.parse(lockData);
866
944
  } catch (err) {
867
945
  if (lockFile) {
946
+ // The failing file names the manager whose lock command regenerates
947
+ // it. A standards-named lock file (PEP 751 `pylock.toml`) belongs to
948
+ // no single manager, so it falls back to the generic repair, whose
949
+ // lock command the shaping resolves from whichever manager the
950
+ // project turns out to use.
951
+ const degradation =
952
+ PY_LOCKFILE_DEGRADATIONS[basename(lockFile)] ||
953
+ GENERIC_PY_LOCKFILE_DEGRADATION;
954
+ recordDegradation(degradation.remediationId, {
955
+ ecosystem: "python",
956
+ impact: "transitive-deps",
957
+ path: lockFile,
958
+ detail: `The ${degradation.manager} lock file could not be parsed, so no locked versions were captured from it.`,
959
+ });
868
960
  console.log(`Error while parsing the lock file ${lockFile}.`, err);
869
961
  } else {
870
962
  console.log("Error while parsing the lock data as toml", err);
@@ -1218,6 +1310,38 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
1218
1310
  }
1219
1311
  }
1220
1312
  }
1313
+ // Seed the parent component's first-level edges from the manifest. Lock
1314
+ // files describe the resolved closure but do not say which of the locked
1315
+ // packages the root project directly depends on; parsePyProjectTomlFile
1316
+ // derived that set into directDepsKeys from [project.dependencies] /
1317
+ // [tool.poetry.dependencies]. Only packages that are locked resolve to a
1318
+ // bom-ref here, so the seeded edge never dangles and dev-group or
1319
+ // unresolved dependencies stay out of the first level. Seeding
1320
+ // depsMap (instead of pushing into dependenciesList) lets the
1321
+ // materialisation below apply its usual ref resolution and sorting.
1322
+ if (parentComponent?.["bom-ref"] && Object.keys(directDepsKeys).length) {
1323
+ const rootRef = parentComponent["bom-ref"];
1324
+ const rootDeps = new Set();
1325
+ for (const depKey of Object.keys(directDepsKeys)) {
1326
+ const depRef =
1327
+ existingPkgMap[depKey] ||
1328
+ existingPkgMap[depKey.replaceAll("-", "_")] ||
1329
+ existingPkgMap[`py${depKey}`];
1330
+ if (depRef && depRef !== rootRef) {
1331
+ rootDeps.add(depRef);
1332
+ }
1333
+ }
1334
+ if (rootDeps.size) {
1335
+ const existingRootDeps = depsMap[rootRef];
1336
+ if (existingRootDeps) {
1337
+ for (const adep of rootDeps) {
1338
+ existingRootDeps.add(adep);
1339
+ }
1340
+ } else {
1341
+ depsMap[rootRef] = rootDeps;
1342
+ }
1343
+ }
1344
+ }
1221
1345
  for (const key of Object.keys(depsMap)) {
1222
1346
  const dependsOnList = new Set();
1223
1347
  const parentPkg = pkgBomRefMap[key];
@@ -1921,8 +2045,20 @@ export function parsePixiTomlFile(pixiToml) {
1921
2045
  pkg.name = projectData.name;
1922
2046
  pkg.version = projectData.version;
1923
2047
  // pkg.authors = tomlData['project']['authors'];
1924
- pkg.homepage = projectData.homepage;
1925
- pkg.repository = projectData.repository;
2048
+ // The parent component only gets `cleanParentComponent`, which converts the
2049
+ // transient `{ url }` shape — raw strings would be dropped silently, so
2050
+ // emit external references directly instead.
2051
+ if (projectData.homepage) {
2052
+ pkg.externalReferences = pkg.externalReferences || [];
2053
+ pkg.externalReferences.push({
2054
+ type: "website",
2055
+ url: projectData.homepage,
2056
+ });
2057
+ }
2058
+ if (projectData.repository) {
2059
+ pkg.externalReferences = pkg.externalReferences || [];
2060
+ pkg.externalReferences.push({ type: "vcs", url: projectData.repository });
2061
+ }
1926
2062
  return pkg;
1927
2063
  }
1928
2064
 
@@ -62,7 +62,15 @@ function parseCargoDependencyFromPackageNode(packageNode) {
62
62
  const isExtendFromWorkspace = (attribute) => {
63
63
  return attribute?.workspace;
64
64
  };
65
- if (!pkgName || !pkgVersion) {
65
+ // A missing name, or a name/version that only exists as an unresolved
66
+ // `*.workspace = true` reference, cannot identify a package. An absent
67
+ // version is fine though: cargo itself records unversioned local packages
68
+ // such as unpublished workspace members as version 0.0.0 (issue #4327).
69
+ if (
70
+ !pkgName ||
71
+ isExtendFromWorkspace(pkgName) ||
72
+ isExtendFromWorkspace(pkgVersion)
73
+ ) {
66
74
  throw new Error("name or version is not defined");
67
75
  }
68
76
 
@@ -76,8 +84,10 @@ function parseCargoDependencyFromPackageNode(packageNode) {
76
84
  pkg.group = group;
77
85
  pkg.name = basename(pkgName.toString());
78
86
  }
79
- if (!isExtendFromWorkspace(pkgVersion) && pkgVersion) {
87
+ if (pkgVersion) {
80
88
  pkg.version = pkgVersion;
89
+ } else {
90
+ pkg.version = "0.0.0";
81
91
  }
82
92
  if (!isExtendFromWorkspace(pkgAuthors) && pkgAuthors) {
83
93
  if (Array.isArray(pkgAuthors)) {
@@ -0,0 +1,326 @@
1
+ // PyPI Trove classifier interpretation and package -> module-name resolution.
2
+ //
3
+ // PyPI responses carry two structured signals that were previously read only for
4
+ // their license lines and then discarded:
5
+ //
6
+ // * `Framework :: X` — the author's declaration that the distribution belongs to
7
+ // framework X. It is more reliable than any name-prefix heuristic: it catches
8
+ // `flask-talisman`, `dj-database-url` and `starlette-context` without anyone
9
+ // maintaining a list, and it never fires on a package that merely has a
10
+ // framework's name embedded in its own.
11
+ // * `Topic :: ...` — a curated taxonomy of what the distribution does, which maps
12
+ // cleanly onto the description-tag vocabulary in `data/component-tags.json`.
13
+ //
14
+ // Tags derived here flow into `component.tags` and, for the `framework` tag, into
15
+ // `determinePackageType()` in `lib/cli/bomAssembly.js`, which already promotes any
16
+ // package carrying a `framework` tag to `type: "framework"`.
17
+
18
+ import { PYPI_MODULE_PACKAGE_MAPPING } from "../core/state.js";
19
+
20
+ /**
21
+ * `Framework ::` values that do not mean "an application is built on this".
22
+ *
23
+ * Two groups, both of which would otherwise put a `framework` tag — and therefore
24
+ * `type: "framework"` — on a large share of an ordinary dependency tree, drowning
25
+ * the runtime frameworks the tag exists to surface:
26
+ *
27
+ * * Development, build, test and documentation tooling. A pytest plugin
28
+ * declaring `Framework :: Pytest` is making a true statement, but it belongs
29
+ * to the dev dependencies, not to what the application runs on.
30
+ * * Concurrency runtimes. `Framework :: AsyncIO` is declared by a very large
31
+ * number of libraries to advertise that they work under asyncio; observed on
32
+ * `fastapi`, and `Framework :: AnyIO` likewise on `starlette`. It describes an
33
+ * execution model, not a framework the application is written against.
34
+ *
35
+ * The declared name is still emitted as a tag in every case, and still recorded in
36
+ * `cdx:pypi:frameworks`; only the generic `framework` tag is withheld.
37
+ */
38
+ export const PYPI_NON_APPLICATION_FRAMEWORKS = new Set([
39
+ "anyio",
40
+ "asyncio",
41
+ "buildout",
42
+ "cookiecutter",
43
+ "flake8",
44
+ "hatch",
45
+ "hypothesis",
46
+ "mkdocs",
47
+ "pytest",
48
+ "robotframework",
49
+ "setuptools",
50
+ "sphinx",
51
+ "tox",
52
+ "trio",
53
+ "wheel",
54
+ ]);
55
+
56
+ /**
57
+ * `Topic ::` prefix -> tags, longest prefix wins.
58
+ *
59
+ * Every value on the right-hand side is an existing entry in
60
+ * `data/component-tags.json` so that a tag derived from a classifier and the same
61
+ * tag derived from a description are indistinguishable to consumers.
62
+ */
63
+ const TOPIC_TAGS = [
64
+ [
65
+ "Topic :: Software Development :: Libraries :: Application Frameworks",
66
+ ["framework"],
67
+ ],
68
+ ["Topic :: Software Development :: Object Brokering", ["rpc"]],
69
+ ["Topic :: Software Development :: Testing", ["test"]],
70
+ ["Topic :: Internet :: WWW/HTTP :: WSGI", ["web", "http", "wsgi"]],
71
+ ["Topic :: Internet :: WWW/HTTP :: Session", ["web", "http", "auth"]],
72
+ ["Topic :: Internet :: WWW/HTTP :: Dynamic Content", ["web", "http"]],
73
+ ["Topic :: Internet :: WWW/HTTP", ["web", "http"]],
74
+ ["Topic :: Internet :: XML-RPC", ["rpc", "xml"]],
75
+ ["Topic :: Internet :: Proxy Servers", ["web", "traffic"]],
76
+ ["Topic :: Database :: Front-Ends", ["database"]],
77
+ ["Topic :: Database", ["database"]],
78
+ ["Topic :: Security :: Cryptography", ["crypto", "security"]],
79
+ ["Topic :: Security", ["security"]],
80
+ ["Topic :: System :: Logging", ["logging"]],
81
+ ["Topic :: System :: Distributed Computing", ["rpc"]],
82
+ ["Topic :: Communications :: Email", ["mail"]],
83
+ ["Topic :: Text Processing :: Markup :: HTML", ["html", "template"]],
84
+ ["Topic :: Text Processing :: Markup :: XML", ["xml"]],
85
+ ["Topic :: Scientific/Engineering :: Artificial Intelligence", ["ai", "ml"]],
86
+ [
87
+ "Topic :: Scientific/Engineering :: Image Recognition",
88
+ ["ml", "object-detect"],
89
+ ],
90
+ ["Topic :: Office/Business :: Financial", ["finance"]],
91
+ ["Topic :: Terminals", ["cli"]],
92
+ ["Topic :: Utilities", ["tools"]],
93
+ ];
94
+
95
+ /**
96
+ * Normalize a distribution name per PEP 503.
97
+ *
98
+ * @param {string} name Distribution name
99
+ * @returns {string} Normalized name, or an empty string for unusable input
100
+ */
101
+ export function normalizePypiName(name) {
102
+ if (!name || typeof name !== "string") {
103
+ return "";
104
+ }
105
+ let out = "";
106
+ let lastWasSeparator = false;
107
+ for (const ch of name.trim().toLowerCase()) {
108
+ if (ch === "-" || ch === "_" || ch === ".") {
109
+ if (!lastWasSeparator && out.length) {
110
+ out += "-";
111
+ }
112
+ lastWasSeparator = true;
113
+ continue;
114
+ }
115
+ out += ch;
116
+ lastWasSeparator = false;
117
+ }
118
+ return out.endsWith("-") ? out.slice(0, -1) : out;
119
+ }
120
+
121
+ /**
122
+ * `data/pypi-pkg-aliases.json` maps module name -> distribution name. Inverting it
123
+ * gives the mapping consumers actually need: distribution -> the module names it
124
+ * installs. Built once on first use.
125
+ *
126
+ * @type {Map<string, string[]> | undefined}
127
+ */
128
+ let invertedAliases;
129
+
130
+ function packageToModules() {
131
+ if (invertedAliases) {
132
+ return invertedAliases;
133
+ }
134
+ invertedAliases = new Map();
135
+ for (const [moduleName, pkgName] of Object.entries(
136
+ PYPI_MODULE_PACKAGE_MAPPING,
137
+ )) {
138
+ const key = normalizePypiName(pkgName);
139
+ if (!key || !moduleName) {
140
+ continue;
141
+ }
142
+ const existing = invertedAliases.get(key);
143
+ if (existing) {
144
+ if (!existing.includes(moduleName)) {
145
+ existing.push(moduleName);
146
+ }
147
+ } else {
148
+ invertedAliases.set(key, [moduleName]);
149
+ }
150
+ }
151
+ return invertedAliases;
152
+ }
153
+
154
+ /**
155
+ * Best-known import module names for a PyPI distribution.
156
+ *
157
+ * Two sources, in order: the curated alias map (which is where the cases that
158
+ * cannot be derived from the name live — `pyyaml` installs `yaml`, `absl-py`
159
+ * installs `absl`, `beautifulsoup4` installs `bs4`), and the name-derived form,
160
+ * which is correct for the large majority of distributions. Callers that need to
161
+ * distinguish the two should read the alias map themselves; for tagging purposes
162
+ * both are usable and a wrong extra candidate simply matches nothing.
163
+ *
164
+ * This is deliberately not a guess at *every* module a distribution ships — only
165
+ * at its top-level import names.
166
+ *
167
+ * @param {string} name Distribution name, e.g. "PyYAML"
168
+ * @returns {string[]} Candidate top-level module names, deduplicated
169
+ */
170
+ export function pypiModulesForPackage(name) {
171
+ const normalized = normalizePypiName(name);
172
+ if (!normalized) {
173
+ return [];
174
+ }
175
+ const modules = [...(packageToModules().get(normalized) || [])];
176
+ // The name-derived form. PEP 503 collapses `.` and `_` to `-`; imports use `_`.
177
+ const derived = normalized.replaceAll("-", "_");
178
+ if (derived && !modules.includes(derived)) {
179
+ modules.push(derived);
180
+ }
181
+ return modules;
182
+ }
183
+
184
+ /**
185
+ * Interpret a PyPI classifier list.
186
+ *
187
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
188
+ * @returns {{frameworks: string[], tags: string[]}} Declared frameworks (lowercased,
189
+ * e.g. `["django"]`) and the tag vocabulary they and the `Topic ::` lines imply
190
+ */
191
+ export function parsePypiClassifiers(classifiers) {
192
+ const frameworks = [];
193
+ const tags = new Set();
194
+ if (!Array.isArray(classifiers)) {
195
+ return { frameworks, tags: [] };
196
+ }
197
+ for (const classifier of classifiers) {
198
+ if (!classifier || typeof classifier !== "string") {
199
+ continue;
200
+ }
201
+ if (classifier.startsWith("Framework :: ")) {
202
+ // "Framework :: Django :: 5.0" -> "django". The version segment is dropped:
203
+ // it is the framework version the package supports, not a distinct signal.
204
+ const declared = classifier.split(" :: ")[1]?.trim();
205
+ if (!declared) {
206
+ continue;
207
+ }
208
+ const name = declared.toLowerCase().replaceAll(" ", "-");
209
+ if (!frameworks.includes(name)) {
210
+ frameworks.push(name);
211
+ }
212
+ tags.add(name);
213
+ if (!PYPI_NON_APPLICATION_FRAMEWORKS.has(name)) {
214
+ tags.add("framework");
215
+ }
216
+ continue;
217
+ }
218
+ if (!classifier.startsWith("Topic :: ")) {
219
+ continue;
220
+ }
221
+ for (const [prefix, topicTags] of TOPIC_TAGS) {
222
+ if (classifier === prefix || classifier.startsWith(`${prefix} ::`)) {
223
+ for (const tag of topicTags) {
224
+ tags.add(tag);
225
+ }
226
+ break;
227
+ }
228
+ }
229
+ }
230
+ return { frameworks, tags: Array.from(tags) };
231
+ }
232
+
233
+ function addProperty(pkg, name, value) {
234
+ if (!value) {
235
+ return;
236
+ }
237
+ if (!pkg.properties) {
238
+ pkg.properties = [];
239
+ }
240
+ const existing = pkg.properties.find((prop) => prop.name === name);
241
+ if (existing) {
242
+ existing.value = value;
243
+ return;
244
+ }
245
+ pkg.properties.push({ name, value });
246
+ }
247
+
248
+ /**
249
+ * Attach classifier-derived tags and properties to a package.
250
+ *
251
+ * Tags are merged into any the package already carries rather than replacing
252
+ * them — `lib/managers/binary.js` sets `tags: ["source"]` as an identity marker,
253
+ * and losing it would change how that package is reconciled later.
254
+ *
255
+ * @param {Object} pkg Package object, mutated in place
256
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
257
+ * @returns {Object} The same package object
258
+ */
259
+ export function applyPypiClassifierMetadata(pkg, classifiers) {
260
+ if (!pkg || !Array.isArray(classifiers) || !classifiers.length) {
261
+ return pkg;
262
+ }
263
+ const { frameworks, tags } = parsePypiClassifiers(classifiers);
264
+ if (tags.length) {
265
+ const merged = new Set(Array.isArray(pkg.tags) ? pkg.tags : []);
266
+ for (const tag of tags) {
267
+ merged.add(tag);
268
+ }
269
+ pkg.tags = Array.from(merged);
270
+ }
271
+ if (frameworks.length) {
272
+ addProperty(pkg, "cdx:pypi:frameworks", frameworks.join(","));
273
+ }
274
+ // The full list is retained so that consumers wanting the whole taxonomy are
275
+ // not limited to the tag vocabulary mapped above. Classifiers come from a
276
+ // closed PyPI-maintained set, so there is no free-text or secret exposure.
277
+ addProperty(
278
+ pkg,
279
+ "cdx:pypi:classifiers",
280
+ classifiers.filter((c) => typeof c === "string" && c.length).join("\n"),
281
+ );
282
+ return pkg;
283
+ }
284
+
285
+ /**
286
+ * Attach the distribution's top-level import module names as `internal:Namespaces`.
287
+ *
288
+ * This is the same property jar, gem, nuget and composer components already carry,
289
+ * so downstream consumers that map namespaces back to components need no
290
+ * pypi-specific path. Unlike the jar case the value is name-derived and
291
+ * alias-curated rather than read out of the artifact, which is why
292
+ * `cdx:pypi:modulesFrom` records the provenance.
293
+ *
294
+ * @param {Object} pkg Package object, mutated in place
295
+ * @returns {Object} The same package object
296
+ */
297
+ export function applyPypiModuleNames(pkg) {
298
+ if (!pkg?.name) {
299
+ return pkg;
300
+ }
301
+ // Extras (`requests[security]`) are not part of the distribution name.
302
+ const name = pkg.name.includes("[") ? pkg.name.split("[")[0] : pkg.name;
303
+ if (name.includes("https")) {
304
+ return pkg;
305
+ }
306
+ const normalized = normalizePypiName(name);
307
+ const modules = pypiModulesForPackage(name);
308
+ if (!modules.length) {
309
+ return pkg;
310
+ }
311
+ // Most alias-map entries map a distribution to the module its own name already
312
+ // implies, so the map having an entry says nothing. The provenance worth
313
+ // recording is whether it contributed a module the name could not have given -
314
+ // that is the difference between `flask` and `pyyaml` -> `yaml`.
315
+ const derived = normalized.replaceAll("-", "_");
316
+ const curatedAddedSomething = (packageToModules().get(normalized) || []).some(
317
+ (moduleName) => moduleName !== derived,
318
+ );
319
+ addProperty(pkg, "internal:Namespaces", modules.join("\n"));
320
+ addProperty(
321
+ pkg,
322
+ "cdx:pypi:modulesFrom",
323
+ curatedAddedSomething ? "alias-map,distribution-name" : "distribution-name",
324
+ );
325
+ return pkg;
326
+ }