@cdxgen/cdxgen 13.0.1 → 13.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -13
- package/bin/audit.js +6 -0
- package/bin/cdxgen.js +125 -4
- package/bin/evinse.js +1 -1
- package/data/component-tags.json +8 -1
- package/data/frameworks-list.json +86 -1
- package/data/remediations.json +1144 -0
- package/data/rules/asar-archives.yaml +1 -1
- package/data/rules/build-fidelity.yaml +782 -0
- package/data/rules/package-integrity.yaml +13 -13
- package/lib/audit/index.js +48 -4
- package/lib/audit/scoring.js +2 -8
- package/lib/cli/bomAssembly.js +20 -1
- package/lib/cli/cliOptions.js +94 -7
- package/lib/cli/index.js +76 -48
- package/lib/cli/jsBom.js +32 -3
- package/lib/cli/jvmBom.js +1494 -1315
- package/lib/cli/managedBom.js +97 -28
- package/lib/cli/nativeBom.js +189 -10
- package/lib/core/buildLedger.js +799 -0
- package/lib/core/deferredExit.js +141 -0
- package/lib/core/env.js +69 -1
- package/lib/core/fs.js +60 -0
- package/lib/core/logger.js +12 -99
- package/lib/core/severity.js +16 -0
- package/lib/core/syncFileWriter.js +106 -0
- package/lib/ecosystems/ecosystems.js +9 -0
- package/lib/ecosystems/gradleutils.js +355 -15
- package/lib/ecosystems/parsers-js.js +26 -0
- package/lib/ecosystems/parsers-jvm.js +9 -0
- package/lib/ecosystems/parsers-misc.js +16 -2
- package/lib/ecosystems/parsers-python.js +171 -35
- package/lib/ecosystems/parsers-rust.js +12 -2
- package/lib/ecosystems/pypiClassifiers.js +326 -0
- package/lib/ecosystems/pythonutils.js +49 -1
- package/lib/ecosystems/rubyutils.js +13 -0
- package/lib/evinser/evinser.js +30 -12
- package/lib/helpers/remote/dependency-track.js +81 -14
- package/lib/inventory/auditCategories.js +9 -0
- package/lib/inventory/depsUtils.js +72 -0
- package/lib/inventory/dosai.js +431 -22
- package/lib/inventory/envcontext.js +206 -18
- package/lib/inventory/fetchBatch.js +57 -0
- package/lib/inventory/jvmToolEnv.js +593 -0
- package/lib/inventory/osinfo.js +1 -0
- package/lib/inventory/toolRequirements.js +903 -0
- package/lib/managers/binary.js +24 -0
- package/lib/managers/docker.js +26 -5
- package/lib/server/server.js +1 -1
- package/lib/stages/postgen/annotationBuilder.js +97 -0
- package/lib/stages/postgen/auditBom.js +76 -79
- package/lib/stages/postgen/introspection/annotate.js +320 -0
- package/lib/stages/postgen/introspection/commandFacts.js +293 -0
- package/lib/stages/postgen/introspection/emit.js +192 -0
- package/lib/stages/postgen/introspection/formulationEvidence.js +427 -0
- package/lib/stages/postgen/introspection/markers.js +227 -0
- package/lib/stages/postgen/introspection/reflect.js +1214 -0
- package/lib/stages/postgen/introspection/report.js +1208 -0
- package/lib/stages/postgen/introspection/score.js +1067 -0
- package/lib/stages/postgen/introspection/shapeCommand.js +304 -0
- package/lib/stages/postgen/parentEdge.js +156 -0
- package/lib/stages/postgen/postgen.js +115 -2
- package/lib/stages/postgen/ruleEngine.js +17 -0
- package/lib/stages/pregen/envAudit.js +42 -2
- package/lib/stages/pregen/pregen.js +1171 -6
- package/lib/validator/bomValidator.js +11 -2
- package/package.json +14 -14
- package/types/lib/audit/index.d.ts.map +1 -1
- package/types/lib/audit/scoring.d.ts +2 -8
- package/types/lib/audit/scoring.d.ts.map +1 -1
- package/types/lib/cli/bomAssembly.d.ts.map +1 -1
- package/types/lib/cli/cliOptions.d.ts +2 -1
- package/types/lib/cli/cliOptions.d.ts.map +1 -1
- package/types/lib/cli/index.d.ts +8 -0
- package/types/lib/cli/index.d.ts.map +1 -1
- package/types/lib/cli/jsBom.d.ts.map +1 -1
- package/types/lib/cli/jvmBom.d.ts.map +1 -1
- package/types/lib/cli/managedBom.d.ts.map +1 -1
- package/types/lib/cli/nativeBom.d.ts.map +1 -1
- package/types/lib/core/buildLedger.d.ts +272 -0
- package/types/lib/core/buildLedger.d.ts.map +1 -0
- package/types/lib/core/deferredExit.d.ts +84 -0
- package/types/lib/core/deferredExit.d.ts.map +1 -0
- package/types/lib/core/env.d.ts +22 -1
- package/types/lib/core/env.d.ts.map +1 -1
- package/types/lib/core/fs.d.ts +21 -0
- package/types/lib/core/fs.d.ts.map +1 -1
- package/types/lib/core/logger.d.ts +2 -11
- package/types/lib/core/logger.d.ts.map +1 -1
- package/types/lib/core/severity.d.ts +11 -0
- package/types/lib/core/severity.d.ts.map +1 -0
- package/types/lib/core/syncFileWriter.d.ts +33 -0
- package/types/lib/core/syncFileWriter.d.ts.map +1 -0
- package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
- package/types/lib/ecosystems/gradleutils.d.ts +33 -0
- package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
- package/types/lib/ecosystems/pypiClassifiers.d.ts +80 -0
- package/types/lib/ecosystems/pypiClassifiers.d.ts.map +1 -0
- package/types/lib/ecosystems/pythonutils.d.ts +20 -0
- package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
- package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
- package/types/lib/evinser/evinser.d.ts +5 -1
- package/types/lib/evinser/evinser.d.ts.map +1 -1
- package/types/lib/helpers/remote/dependency-track.d.ts +45 -4
- package/types/lib/helpers/remote/dependency-track.d.ts.map +1 -1
- package/types/lib/inventory/auditCategories.d.ts +8 -0
- package/types/lib/inventory/auditCategories.d.ts.map +1 -1
- package/types/lib/inventory/depsUtils.d.ts.map +1 -1
- package/types/lib/inventory/dosai.d.ts +20 -0
- package/types/lib/inventory/dosai.d.ts.map +1 -1
- package/types/lib/inventory/envcontext.d.ts.map +1 -1
- package/types/lib/inventory/fetchBatch.d.ts +9 -0
- package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
- package/types/lib/inventory/jvmToolEnv.d.ts +189 -0
- package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -0
- package/types/lib/inventory/osinfo.d.ts.map +1 -1
- package/types/lib/inventory/toolRequirements.d.ts +216 -0
- package/types/lib/inventory/toolRequirements.d.ts.map +1 -0
- package/types/lib/managers/binary.d.ts.map +1 -1
- package/types/lib/managers/docker.d.ts.map +1 -1
- package/types/lib/stages/postgen/annotationBuilder.d.ts +53 -0
- package/types/lib/stages/postgen/annotationBuilder.d.ts.map +1 -0
- package/types/lib/stages/postgen/auditBom.d.ts.map +1 -1
- package/types/lib/stages/postgen/introspection/annotate.d.ts +81 -0
- package/types/lib/stages/postgen/introspection/annotate.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts +32 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts +71 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts +154 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts +57 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts +77 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/report.d.ts +85 -0
- package/types/lib/stages/postgen/introspection/report.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/score.d.ts +149 -0
- package/types/lib/stages/postgen/introspection/score.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts +79 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts.map +1 -0
- package/types/lib/stages/postgen/parentEdge.d.ts +35 -0
- package/types/lib/stages/postgen/parentEdge.d.ts.map +1 -0
- package/types/lib/stages/postgen/postgen.d.ts +2 -2
- package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
- package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
- package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
- package/types/lib/stages/pregen/pregen.d.ts +131 -0
- package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
- package/types/lib/validator/bomValidator.d.ts.map +1 -1
|
@@ -8,6 +8,7 @@ import toml from "@iarna/toml";
|
|
|
8
8
|
import { parse as _load } from "yaml";
|
|
9
9
|
|
|
10
10
|
import { DEBUG_MODE } from "../core/activity.js";
|
|
11
|
+
import { recordDegradation } from "../core/buildLedger.js";
|
|
11
12
|
import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
|
|
12
13
|
import { PYTHON_STD_MODULES } from "../core/state.js";
|
|
13
14
|
import { tryBuildPurl } from "../inventory/purl.js";
|
|
@@ -27,6 +28,41 @@ import {
|
|
|
27
28
|
normalizePyLockRegistry,
|
|
28
29
|
} from "./pylockutils.js";
|
|
29
30
|
|
|
31
|
+
/**
|
|
32
|
+
* TOML lock files a specific Python manager owns, mapped to the manager and
|
|
33
|
+
* the remediation id that ranks its repair. The failing file itself is the
|
|
34
|
+
* detection: the lock command that regenerates it belongs to the manager
|
|
35
|
+
* that wrote it.
|
|
36
|
+
*
|
|
37
|
+
* @type {Record<string, {manager: string, remediationId: string}>}
|
|
38
|
+
*/
|
|
39
|
+
const PY_LOCKFILE_DEGRADATIONS = {
|
|
40
|
+
"uv.lock": {
|
|
41
|
+
manager: "uv",
|
|
42
|
+
remediationId: "python.lockfile-unparseable.uv",
|
|
43
|
+
},
|
|
44
|
+
"poetry.lock": {
|
|
45
|
+
manager: "poetry",
|
|
46
|
+
remediationId: "python.lockfile-unparseable.poetry",
|
|
47
|
+
},
|
|
48
|
+
"pdm.lock": {
|
|
49
|
+
manager: "pdm",
|
|
50
|
+
remediationId: "python.lockfile-unparseable.pdm",
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The repair for a TOML lock file that names no single manager, such as
|
|
56
|
+
* PEP 751's `pylock.toml`. Its lock command carries the manager variable, so
|
|
57
|
+
* the shaping fills in whichever manager the project turns out to use.
|
|
58
|
+
*
|
|
59
|
+
* @type {{manager: string, remediationId: string}}
|
|
60
|
+
*/
|
|
61
|
+
const GENERIC_PY_LOCKFILE_DEGRADATION = {
|
|
62
|
+
manager: "python",
|
|
63
|
+
remediationId: "python.lockfile-unparseable",
|
|
64
|
+
};
|
|
65
|
+
|
|
30
66
|
/**
|
|
31
67
|
* Method to parse python requires_dist attribute found in pypi setup.py
|
|
32
68
|
*
|
|
@@ -444,7 +480,12 @@ export function parsePyProjectTomlFile(tomlFile) {
|
|
|
444
480
|
for (const agroup of Object.keys(tomlData["dependency-groups"])) {
|
|
445
481
|
tomlData["dependency-groups"][agroup].forEach((p) => {
|
|
446
482
|
if (typeof p === "string" || p instanceof String) {
|
|
447
|
-
const pname =
|
|
483
|
+
const pname = normalizePythonDependencyKey(
|
|
484
|
+
p.split(/(==|<=|~=|>=)/)[0].split(" ")[0],
|
|
485
|
+
);
|
|
486
|
+
if (!pname) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
448
489
|
if (!groupDepsKeys[pname]) {
|
|
449
490
|
groupDepsKeys[pname] = [];
|
|
450
491
|
}
|
|
@@ -466,8 +507,13 @@ export function parsePyProjectTomlFile(tomlFile) {
|
|
|
466
507
|
}
|
|
467
508
|
if (tomlData?.tool?.poetry?.dependencies) {
|
|
468
509
|
for (const adep of Object.keys(tomlData?.tool?.poetry?.dependencies)) {
|
|
510
|
+
// poetry keys keep the author's casing (Click = "^8.1"), while every
|
|
511
|
+
// consumer of directDepsKeys looks entries up with the normalised
|
|
512
|
+
// (lowercase, hyphenated) package name, so normalise at the source.
|
|
513
|
+
const poetryDepKey = normalizePythonDependencyKey(adep);
|
|
469
514
|
if (
|
|
470
|
-
!
|
|
515
|
+
!poetryDepKey ||
|
|
516
|
+
[
|
|
471
517
|
"python",
|
|
472
518
|
"py",
|
|
473
519
|
"pytest",
|
|
@@ -475,43 +521,50 @@ export function parsePyProjectTomlFile(tomlFile) {
|
|
|
475
521
|
"ruff",
|
|
476
522
|
"setuptools",
|
|
477
523
|
"bandit",
|
|
478
|
-
].includes(
|
|
524
|
+
].includes(poetryDepKey)
|
|
479
525
|
) {
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
directDepsKeys[poetryDepKey] = true;
|
|
529
|
+
const poetryDependency = tomlData.tool.poetry.dependencies[adep];
|
|
530
|
+
if (poetryDependency?.git) {
|
|
531
|
+
recordPythonDependencySource(
|
|
532
|
+
dependencySourceMap,
|
|
533
|
+
adep,
|
|
534
|
+
"git",
|
|
535
|
+
poetryDependency.git,
|
|
536
|
+
);
|
|
537
|
+
} else if (poetryDependency?.url) {
|
|
538
|
+
recordPythonDependencySource(
|
|
539
|
+
dependencySourceMap,
|
|
540
|
+
adep,
|
|
541
|
+
"url",
|
|
542
|
+
poetryDependency.url,
|
|
543
|
+
);
|
|
544
|
+
} else if (poetryDependency?.path) {
|
|
545
|
+
recordPythonDependencySource(
|
|
546
|
+
dependencySourceMap,
|
|
547
|
+
adep,
|
|
548
|
+
"path",
|
|
549
|
+
poetryDependency.path,
|
|
550
|
+
);
|
|
504
551
|
}
|
|
505
552
|
} // for
|
|
553
|
+
// Group keys are matched against the normalised package names from the
|
|
554
|
+
// lock file, so they are normalised here too.
|
|
506
555
|
if (tomlData?.tool?.poetry?.group) {
|
|
507
556
|
for (const agroup of Object.keys(tomlData.tool.poetry.group)) {
|
|
508
557
|
for (const adep of Object.keys(
|
|
509
558
|
tomlData.tool.poetry.group[agroup]?.dependencies,
|
|
510
559
|
)) {
|
|
511
|
-
|
|
512
|
-
|
|
560
|
+
const poetryGroupDepKey = normalizePythonDependencyKey(adep);
|
|
561
|
+
if (!poetryGroupDepKey) {
|
|
562
|
+
continue;
|
|
513
563
|
}
|
|
514
|
-
groupDepsKeys[
|
|
564
|
+
if (!groupDepsKeys[poetryGroupDepKey]) {
|
|
565
|
+
groupDepsKeys[poetryGroupDepKey] = [];
|
|
566
|
+
}
|
|
567
|
+
groupDepsKeys[poetryGroupDepKey].push(agroup);
|
|
515
568
|
const poetryDependency =
|
|
516
569
|
tomlData.tool.poetry.group[agroup]?.dependencies?.[adep];
|
|
517
570
|
if (poetryDependency?.git) {
|
|
@@ -780,7 +833,27 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
|
|
|
780
833
|
dependencySourceMap = pyProjMap.dependencySourceMap || {};
|
|
781
834
|
parentComponent = pyProjMap.parentComponent;
|
|
782
835
|
workspacePaths = pyProjMap.workspacePaths;
|
|
783
|
-
|
|
836
|
+
// `parsePyProjectTomlFile` reports a parse failure and returns an empty map, so a
|
|
837
|
+
// pyproject.toml this parser cannot read leaves `parentComponent` undefined while
|
|
838
|
+
// the lock file still declares workspaces. Building the workspace tree then throws
|
|
839
|
+
// on the first dereference and takes the whole SBOM with it. Degrade to a flattened
|
|
840
|
+
// tree instead - the caller supplies its own parent component, so the components
|
|
841
|
+
// themselves are still reported.
|
|
842
|
+
if (workspacePaths?.length && !parentComponent) {
|
|
843
|
+
console.log(
|
|
844
|
+
`Unable to read a parent component from ${pyProjectFile}, so the workspace structure declared in ${basename(lockFile)} cannot be reconstructed.`,
|
|
845
|
+
);
|
|
846
|
+
console.log(
|
|
847
|
+
"The dependency tree in the generated SBOM will be flattened and therefore incorrect.",
|
|
848
|
+
);
|
|
849
|
+
recordDegradation("python.pyproject-unparseable", {
|
|
850
|
+
ecosystem: "python",
|
|
851
|
+
impact: "transitive-deps",
|
|
852
|
+
path: pyProjectFile,
|
|
853
|
+
detail: `${basename(pyProjectFile)} could not be parsed, so the workspace structure declared in ${basename(lockFile)} was not reconstructed.`,
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
if (workspacePaths?.length && parentComponent) {
|
|
784
857
|
if (!hasWorkspaces) {
|
|
785
858
|
hasWorkspaces = true;
|
|
786
859
|
}
|
|
@@ -840,7 +913,12 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
|
|
|
840
913
|
] = awpyproj;
|
|
841
914
|
}
|
|
842
915
|
}
|
|
843
|
-
|
|
916
|
+
// The block above already treats an unreadable workspace pyproject.toml as
|
|
917
|
+
// tolerable, so this must not dereference the component it skipped. A
|
|
918
|
+
// workspace member whose manifest cannot be parsed contributes no parent
|
|
919
|
+
// ref, and recording `undefined` as the parent of its dependencies would
|
|
920
|
+
// corrupt the tree for the members that did parse.
|
|
921
|
+
const wparentComponentRef = wcompMap?.parentComponent?.["bom-ref"];
|
|
844
922
|
if (wcompMap?.dependencySourceMap) {
|
|
845
923
|
Object.assign(
|
|
846
924
|
workspaceDependencySourceMap,
|
|
@@ -848,7 +926,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
|
|
|
848
926
|
);
|
|
849
927
|
}
|
|
850
928
|
// Track the parents of workspace direct dependencies
|
|
851
|
-
if (wcompMap?.directDepsKeys) {
|
|
929
|
+
if (wcompMap?.directDepsKeys && wparentComponentRef) {
|
|
852
930
|
for (const wdd of Object.keys(wcompMap?.directDepsKeys)) {
|
|
853
931
|
if (!pkgParentMap[wdd]) {
|
|
854
932
|
pkgParentMap[wdd] = [];
|
|
@@ -865,6 +943,20 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
|
|
|
865
943
|
lockTomlObj = toml.parse(lockData);
|
|
866
944
|
} catch (err) {
|
|
867
945
|
if (lockFile) {
|
|
946
|
+
// The failing file names the manager whose lock command regenerates
|
|
947
|
+
// it. A standards-named lock file (PEP 751 `pylock.toml`) belongs to
|
|
948
|
+
// no single manager, so it falls back to the generic repair, whose
|
|
949
|
+
// lock command the shaping resolves from whichever manager the
|
|
950
|
+
// project turns out to use.
|
|
951
|
+
const degradation =
|
|
952
|
+
PY_LOCKFILE_DEGRADATIONS[basename(lockFile)] ||
|
|
953
|
+
GENERIC_PY_LOCKFILE_DEGRADATION;
|
|
954
|
+
recordDegradation(degradation.remediationId, {
|
|
955
|
+
ecosystem: "python",
|
|
956
|
+
impact: "transitive-deps",
|
|
957
|
+
path: lockFile,
|
|
958
|
+
detail: `The ${degradation.manager} lock file could not be parsed, so no locked versions were captured from it.`,
|
|
959
|
+
});
|
|
868
960
|
console.log(`Error while parsing the lock file ${lockFile}.`, err);
|
|
869
961
|
} else {
|
|
870
962
|
console.log("Error while parsing the lock data as toml", err);
|
|
@@ -1218,6 +1310,38 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
|
|
|
1218
1310
|
}
|
|
1219
1311
|
}
|
|
1220
1312
|
}
|
|
1313
|
+
// Seed the parent component's first-level edges from the manifest. Lock
|
|
1314
|
+
// files describe the resolved closure but do not say which of the locked
|
|
1315
|
+
// packages the root project directly depends on; parsePyProjectTomlFile
|
|
1316
|
+
// derived that set into directDepsKeys from [project.dependencies] /
|
|
1317
|
+
// [tool.poetry.dependencies]. Only packages that are locked resolve to a
|
|
1318
|
+
// bom-ref here, so the seeded edge never dangles and dev-group or
|
|
1319
|
+
// unresolved dependencies stay out of the first level. Seeding
|
|
1320
|
+
// depsMap (instead of pushing into dependenciesList) lets the
|
|
1321
|
+
// materialisation below apply its usual ref resolution and sorting.
|
|
1322
|
+
if (parentComponent?.["bom-ref"] && Object.keys(directDepsKeys).length) {
|
|
1323
|
+
const rootRef = parentComponent["bom-ref"];
|
|
1324
|
+
const rootDeps = new Set();
|
|
1325
|
+
for (const depKey of Object.keys(directDepsKeys)) {
|
|
1326
|
+
const depRef =
|
|
1327
|
+
existingPkgMap[depKey] ||
|
|
1328
|
+
existingPkgMap[depKey.replaceAll("-", "_")] ||
|
|
1329
|
+
existingPkgMap[`py${depKey}`];
|
|
1330
|
+
if (depRef && depRef !== rootRef) {
|
|
1331
|
+
rootDeps.add(depRef);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
if (rootDeps.size) {
|
|
1335
|
+
const existingRootDeps = depsMap[rootRef];
|
|
1336
|
+
if (existingRootDeps) {
|
|
1337
|
+
for (const adep of rootDeps) {
|
|
1338
|
+
existingRootDeps.add(adep);
|
|
1339
|
+
}
|
|
1340
|
+
} else {
|
|
1341
|
+
depsMap[rootRef] = rootDeps;
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1221
1345
|
for (const key of Object.keys(depsMap)) {
|
|
1222
1346
|
const dependsOnList = new Set();
|
|
1223
1347
|
const parentPkg = pkgBomRefMap[key];
|
|
@@ -1921,8 +2045,20 @@ export function parsePixiTomlFile(pixiToml) {
|
|
|
1921
2045
|
pkg.name = projectData.name;
|
|
1922
2046
|
pkg.version = projectData.version;
|
|
1923
2047
|
// pkg.authors = tomlData['project']['authors'];
|
|
1924
|
-
|
|
1925
|
-
|
|
2048
|
+
// The parent component only gets `cleanParentComponent`, which converts the
|
|
2049
|
+
// transient `{ url }` shape — raw strings would be dropped silently, so
|
|
2050
|
+
// emit external references directly instead.
|
|
2051
|
+
if (projectData.homepage) {
|
|
2052
|
+
pkg.externalReferences = pkg.externalReferences || [];
|
|
2053
|
+
pkg.externalReferences.push({
|
|
2054
|
+
type: "website",
|
|
2055
|
+
url: projectData.homepage,
|
|
2056
|
+
});
|
|
2057
|
+
}
|
|
2058
|
+
if (projectData.repository) {
|
|
2059
|
+
pkg.externalReferences = pkg.externalReferences || [];
|
|
2060
|
+
pkg.externalReferences.push({ type: "vcs", url: projectData.repository });
|
|
2061
|
+
}
|
|
1926
2062
|
return pkg;
|
|
1927
2063
|
}
|
|
1928
2064
|
|
|
@@ -62,7 +62,15 @@ function parseCargoDependencyFromPackageNode(packageNode) {
|
|
|
62
62
|
const isExtendFromWorkspace = (attribute) => {
|
|
63
63
|
return attribute?.workspace;
|
|
64
64
|
};
|
|
65
|
-
|
|
65
|
+
// A missing name, or a name/version that only exists as an unresolved
|
|
66
|
+
// `*.workspace = true` reference, cannot identify a package. An absent
|
|
67
|
+
// version is fine though: cargo itself records unversioned local packages
|
|
68
|
+
// such as unpublished workspace members as version 0.0.0 (issue #4327).
|
|
69
|
+
if (
|
|
70
|
+
!pkgName ||
|
|
71
|
+
isExtendFromWorkspace(pkgName) ||
|
|
72
|
+
isExtendFromWorkspace(pkgVersion)
|
|
73
|
+
) {
|
|
66
74
|
throw new Error("name or version is not defined");
|
|
67
75
|
}
|
|
68
76
|
|
|
@@ -76,8 +84,10 @@ function parseCargoDependencyFromPackageNode(packageNode) {
|
|
|
76
84
|
pkg.group = group;
|
|
77
85
|
pkg.name = basename(pkgName.toString());
|
|
78
86
|
}
|
|
79
|
-
if (
|
|
87
|
+
if (pkgVersion) {
|
|
80
88
|
pkg.version = pkgVersion;
|
|
89
|
+
} else {
|
|
90
|
+
pkg.version = "0.0.0";
|
|
81
91
|
}
|
|
82
92
|
if (!isExtendFromWorkspace(pkgAuthors) && pkgAuthors) {
|
|
83
93
|
if (Array.isArray(pkgAuthors)) {
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// PyPI Trove classifier interpretation and package -> module-name resolution.
|
|
2
|
+
//
|
|
3
|
+
// PyPI responses carry two structured signals that were previously read only for
|
|
4
|
+
// their license lines and then discarded:
|
|
5
|
+
//
|
|
6
|
+
// * `Framework :: X` — the author's declaration that the distribution belongs to
|
|
7
|
+
// framework X. It is more reliable than any name-prefix heuristic: it catches
|
|
8
|
+
// `flask-talisman`, `dj-database-url` and `starlette-context` without anyone
|
|
9
|
+
// maintaining a list, and it never fires on a package that merely has a
|
|
10
|
+
// framework's name embedded in its own.
|
|
11
|
+
// * `Topic :: ...` — a curated taxonomy of what the distribution does, which maps
|
|
12
|
+
// cleanly onto the description-tag vocabulary in `data/component-tags.json`.
|
|
13
|
+
//
|
|
14
|
+
// Tags derived here flow into `component.tags` and, for the `framework` tag, into
|
|
15
|
+
// `determinePackageType()` in `lib/cli/bomAssembly.js`, which already promotes any
|
|
16
|
+
// package carrying a `framework` tag to `type: "framework"`.
|
|
17
|
+
|
|
18
|
+
import { PYPI_MODULE_PACKAGE_MAPPING } from "../core/state.js";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `Framework ::` values that do not mean "an application is built on this".
|
|
22
|
+
*
|
|
23
|
+
* Two groups, both of which would otherwise put a `framework` tag — and therefore
|
|
24
|
+
* `type: "framework"` — on a large share of an ordinary dependency tree, drowning
|
|
25
|
+
* the runtime frameworks the tag exists to surface:
|
|
26
|
+
*
|
|
27
|
+
* * Development, build, test and documentation tooling. A pytest plugin
|
|
28
|
+
* declaring `Framework :: Pytest` is making a true statement, but it belongs
|
|
29
|
+
* to the dev dependencies, not to what the application runs on.
|
|
30
|
+
* * Concurrency runtimes. `Framework :: AsyncIO` is declared by a very large
|
|
31
|
+
* number of libraries to advertise that they work under asyncio; observed on
|
|
32
|
+
* `fastapi`, and `Framework :: AnyIO` likewise on `starlette`. It describes an
|
|
33
|
+
* execution model, not a framework the application is written against.
|
|
34
|
+
*
|
|
35
|
+
* The declared name is still emitted as a tag in every case, and still recorded in
|
|
36
|
+
* `cdx:pypi:frameworks`; only the generic `framework` tag is withheld.
|
|
37
|
+
*/
|
|
38
|
+
export const PYPI_NON_APPLICATION_FRAMEWORKS = new Set([
|
|
39
|
+
"anyio",
|
|
40
|
+
"asyncio",
|
|
41
|
+
"buildout",
|
|
42
|
+
"cookiecutter",
|
|
43
|
+
"flake8",
|
|
44
|
+
"hatch",
|
|
45
|
+
"hypothesis",
|
|
46
|
+
"mkdocs",
|
|
47
|
+
"pytest",
|
|
48
|
+
"robotframework",
|
|
49
|
+
"setuptools",
|
|
50
|
+
"sphinx",
|
|
51
|
+
"tox",
|
|
52
|
+
"trio",
|
|
53
|
+
"wheel",
|
|
54
|
+
]);
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* `Topic ::` prefix -> tags, longest prefix wins.
|
|
58
|
+
*
|
|
59
|
+
* Every value on the right-hand side is an existing entry in
|
|
60
|
+
* `data/component-tags.json` so that a tag derived from a classifier and the same
|
|
61
|
+
* tag derived from a description are indistinguishable to consumers.
|
|
62
|
+
*/
|
|
63
|
+
const TOPIC_TAGS = [
|
|
64
|
+
[
|
|
65
|
+
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
|
66
|
+
["framework"],
|
|
67
|
+
],
|
|
68
|
+
["Topic :: Software Development :: Object Brokering", ["rpc"]],
|
|
69
|
+
["Topic :: Software Development :: Testing", ["test"]],
|
|
70
|
+
["Topic :: Internet :: WWW/HTTP :: WSGI", ["web", "http", "wsgi"]],
|
|
71
|
+
["Topic :: Internet :: WWW/HTTP :: Session", ["web", "http", "auth"]],
|
|
72
|
+
["Topic :: Internet :: WWW/HTTP :: Dynamic Content", ["web", "http"]],
|
|
73
|
+
["Topic :: Internet :: WWW/HTTP", ["web", "http"]],
|
|
74
|
+
["Topic :: Internet :: XML-RPC", ["rpc", "xml"]],
|
|
75
|
+
["Topic :: Internet :: Proxy Servers", ["web", "traffic"]],
|
|
76
|
+
["Topic :: Database :: Front-Ends", ["database"]],
|
|
77
|
+
["Topic :: Database", ["database"]],
|
|
78
|
+
["Topic :: Security :: Cryptography", ["crypto", "security"]],
|
|
79
|
+
["Topic :: Security", ["security"]],
|
|
80
|
+
["Topic :: System :: Logging", ["logging"]],
|
|
81
|
+
["Topic :: System :: Distributed Computing", ["rpc"]],
|
|
82
|
+
["Topic :: Communications :: Email", ["mail"]],
|
|
83
|
+
["Topic :: Text Processing :: Markup :: HTML", ["html", "template"]],
|
|
84
|
+
["Topic :: Text Processing :: Markup :: XML", ["xml"]],
|
|
85
|
+
["Topic :: Scientific/Engineering :: Artificial Intelligence", ["ai", "ml"]],
|
|
86
|
+
[
|
|
87
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
88
|
+
["ml", "object-detect"],
|
|
89
|
+
],
|
|
90
|
+
["Topic :: Office/Business :: Financial", ["finance"]],
|
|
91
|
+
["Topic :: Terminals", ["cli"]],
|
|
92
|
+
["Topic :: Utilities", ["tools"]],
|
|
93
|
+
];
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Normalize a distribution name per PEP 503.
|
|
97
|
+
*
|
|
98
|
+
* @param {string} name Distribution name
|
|
99
|
+
* @returns {string} Normalized name, or an empty string for unusable input
|
|
100
|
+
*/
|
|
101
|
+
export function normalizePypiName(name) {
|
|
102
|
+
if (!name || typeof name !== "string") {
|
|
103
|
+
return "";
|
|
104
|
+
}
|
|
105
|
+
let out = "";
|
|
106
|
+
let lastWasSeparator = false;
|
|
107
|
+
for (const ch of name.trim().toLowerCase()) {
|
|
108
|
+
if (ch === "-" || ch === "_" || ch === ".") {
|
|
109
|
+
if (!lastWasSeparator && out.length) {
|
|
110
|
+
out += "-";
|
|
111
|
+
}
|
|
112
|
+
lastWasSeparator = true;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
out += ch;
|
|
116
|
+
lastWasSeparator = false;
|
|
117
|
+
}
|
|
118
|
+
return out.endsWith("-") ? out.slice(0, -1) : out;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* `data/pypi-pkg-aliases.json` maps module name -> distribution name. Inverting it
|
|
123
|
+
* gives the mapping consumers actually need: distribution -> the module names it
|
|
124
|
+
* installs. Built once on first use.
|
|
125
|
+
*
|
|
126
|
+
* @type {Map<string, string[]> | undefined}
|
|
127
|
+
*/
|
|
128
|
+
let invertedAliases;
|
|
129
|
+
|
|
130
|
+
function packageToModules() {
|
|
131
|
+
if (invertedAliases) {
|
|
132
|
+
return invertedAliases;
|
|
133
|
+
}
|
|
134
|
+
invertedAliases = new Map();
|
|
135
|
+
for (const [moduleName, pkgName] of Object.entries(
|
|
136
|
+
PYPI_MODULE_PACKAGE_MAPPING,
|
|
137
|
+
)) {
|
|
138
|
+
const key = normalizePypiName(pkgName);
|
|
139
|
+
if (!key || !moduleName) {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const existing = invertedAliases.get(key);
|
|
143
|
+
if (existing) {
|
|
144
|
+
if (!existing.includes(moduleName)) {
|
|
145
|
+
existing.push(moduleName);
|
|
146
|
+
}
|
|
147
|
+
} else {
|
|
148
|
+
invertedAliases.set(key, [moduleName]);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return invertedAliases;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Best-known import module names for a PyPI distribution.
|
|
156
|
+
*
|
|
157
|
+
* Two sources, in order: the curated alias map (which is where the cases that
|
|
158
|
+
* cannot be derived from the name live — `pyyaml` installs `yaml`, `absl-py`
|
|
159
|
+
* installs `absl`, `beautifulsoup4` installs `bs4`), and the name-derived form,
|
|
160
|
+
* which is correct for the large majority of distributions. Callers that need to
|
|
161
|
+
* distinguish the two should read the alias map themselves; for tagging purposes
|
|
162
|
+
* both are usable and a wrong extra candidate simply matches nothing.
|
|
163
|
+
*
|
|
164
|
+
* This is deliberately not a guess at *every* module a distribution ships — only
|
|
165
|
+
* at its top-level import names.
|
|
166
|
+
*
|
|
167
|
+
* @param {string} name Distribution name, e.g. "PyYAML"
|
|
168
|
+
* @returns {string[]} Candidate top-level module names, deduplicated
|
|
169
|
+
*/
|
|
170
|
+
export function pypiModulesForPackage(name) {
|
|
171
|
+
const normalized = normalizePypiName(name);
|
|
172
|
+
if (!normalized) {
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
const modules = [...(packageToModules().get(normalized) || [])];
|
|
176
|
+
// The name-derived form. PEP 503 collapses `.` and `_` to `-`; imports use `_`.
|
|
177
|
+
const derived = normalized.replaceAll("-", "_");
|
|
178
|
+
if (derived && !modules.includes(derived)) {
|
|
179
|
+
modules.push(derived);
|
|
180
|
+
}
|
|
181
|
+
return modules;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Interpret a PyPI classifier list.
|
|
186
|
+
*
|
|
187
|
+
* @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
|
|
188
|
+
* @returns {{frameworks: string[], tags: string[]}} Declared frameworks (lowercased,
|
|
189
|
+
* e.g. `["django"]`) and the tag vocabulary they and the `Topic ::` lines imply
|
|
190
|
+
*/
|
|
191
|
+
export function parsePypiClassifiers(classifiers) {
|
|
192
|
+
const frameworks = [];
|
|
193
|
+
const tags = new Set();
|
|
194
|
+
if (!Array.isArray(classifiers)) {
|
|
195
|
+
return { frameworks, tags: [] };
|
|
196
|
+
}
|
|
197
|
+
for (const classifier of classifiers) {
|
|
198
|
+
if (!classifier || typeof classifier !== "string") {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (classifier.startsWith("Framework :: ")) {
|
|
202
|
+
// "Framework :: Django :: 5.0" -> "django". The version segment is dropped:
|
|
203
|
+
// it is the framework version the package supports, not a distinct signal.
|
|
204
|
+
const declared = classifier.split(" :: ")[1]?.trim();
|
|
205
|
+
if (!declared) {
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const name = declared.toLowerCase().replaceAll(" ", "-");
|
|
209
|
+
if (!frameworks.includes(name)) {
|
|
210
|
+
frameworks.push(name);
|
|
211
|
+
}
|
|
212
|
+
tags.add(name);
|
|
213
|
+
if (!PYPI_NON_APPLICATION_FRAMEWORKS.has(name)) {
|
|
214
|
+
tags.add("framework");
|
|
215
|
+
}
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
if (!classifier.startsWith("Topic :: ")) {
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
for (const [prefix, topicTags] of TOPIC_TAGS) {
|
|
222
|
+
if (classifier === prefix || classifier.startsWith(`${prefix} ::`)) {
|
|
223
|
+
for (const tag of topicTags) {
|
|
224
|
+
tags.add(tag);
|
|
225
|
+
}
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return { frameworks, tags: Array.from(tags) };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function addProperty(pkg, name, value) {
|
|
234
|
+
if (!value) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (!pkg.properties) {
|
|
238
|
+
pkg.properties = [];
|
|
239
|
+
}
|
|
240
|
+
const existing = pkg.properties.find((prop) => prop.name === name);
|
|
241
|
+
if (existing) {
|
|
242
|
+
existing.value = value;
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
pkg.properties.push({ name, value });
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Attach classifier-derived tags and properties to a package.
|
|
250
|
+
*
|
|
251
|
+
* Tags are merged into any the package already carries rather than replacing
|
|
252
|
+
* them — `lib/managers/binary.js` sets `tags: ["source"]` as an identity marker,
|
|
253
|
+
* and losing it would change how that package is reconciled later.
|
|
254
|
+
*
|
|
255
|
+
* @param {Object} pkg Package object, mutated in place
|
|
256
|
+
* @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
|
|
257
|
+
* @returns {Object} The same package object
|
|
258
|
+
*/
|
|
259
|
+
export function applyPypiClassifierMetadata(pkg, classifiers) {
|
|
260
|
+
if (!pkg || !Array.isArray(classifiers) || !classifiers.length) {
|
|
261
|
+
return pkg;
|
|
262
|
+
}
|
|
263
|
+
const { frameworks, tags } = parsePypiClassifiers(classifiers);
|
|
264
|
+
if (tags.length) {
|
|
265
|
+
const merged = new Set(Array.isArray(pkg.tags) ? pkg.tags : []);
|
|
266
|
+
for (const tag of tags) {
|
|
267
|
+
merged.add(tag);
|
|
268
|
+
}
|
|
269
|
+
pkg.tags = Array.from(merged);
|
|
270
|
+
}
|
|
271
|
+
if (frameworks.length) {
|
|
272
|
+
addProperty(pkg, "cdx:pypi:frameworks", frameworks.join(","));
|
|
273
|
+
}
|
|
274
|
+
// The full list is retained so that consumers wanting the whole taxonomy are
|
|
275
|
+
// not limited to the tag vocabulary mapped above. Classifiers come from a
|
|
276
|
+
// closed PyPI-maintained set, so there is no free-text or secret exposure.
|
|
277
|
+
addProperty(
|
|
278
|
+
pkg,
|
|
279
|
+
"cdx:pypi:classifiers",
|
|
280
|
+
classifiers.filter((c) => typeof c === "string" && c.length).join("\n"),
|
|
281
|
+
);
|
|
282
|
+
return pkg;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Attach the distribution's top-level import module names as `internal:Namespaces`.
|
|
287
|
+
*
|
|
288
|
+
* This is the same property jar, gem, nuget and composer components already carry,
|
|
289
|
+
* so downstream consumers that map namespaces back to components need no
|
|
290
|
+
* pypi-specific path. Unlike the jar case the value is name-derived and
|
|
291
|
+
* alias-curated rather than read out of the artifact, which is why
|
|
292
|
+
* `cdx:pypi:modulesFrom` records the provenance.
|
|
293
|
+
*
|
|
294
|
+
* @param {Object} pkg Package object, mutated in place
|
|
295
|
+
* @returns {Object} The same package object
|
|
296
|
+
*/
|
|
297
|
+
export function applyPypiModuleNames(pkg) {
|
|
298
|
+
if (!pkg?.name) {
|
|
299
|
+
return pkg;
|
|
300
|
+
}
|
|
301
|
+
// Extras (`requests[security]`) are not part of the distribution name.
|
|
302
|
+
const name = pkg.name.includes("[") ? pkg.name.split("[")[0] : pkg.name;
|
|
303
|
+
if (name.includes("https")) {
|
|
304
|
+
return pkg;
|
|
305
|
+
}
|
|
306
|
+
const normalized = normalizePypiName(name);
|
|
307
|
+
const modules = pypiModulesForPackage(name);
|
|
308
|
+
if (!modules.length) {
|
|
309
|
+
return pkg;
|
|
310
|
+
}
|
|
311
|
+
// Most alias-map entries map a distribution to the module its own name already
|
|
312
|
+
// implies, so the map having an entry says nothing. The provenance worth
|
|
313
|
+
// recording is whether it contributed a module the name could not have given -
|
|
314
|
+
// that is the difference between `flask` and `pyyaml` -> `yaml`.
|
|
315
|
+
const derived = normalized.replaceAll("-", "_");
|
|
316
|
+
const curatedAddedSomething = (packageToModules().get(normalized) || []).some(
|
|
317
|
+
(moduleName) => moduleName !== derived,
|
|
318
|
+
);
|
|
319
|
+
addProperty(pkg, "internal:Namespaces", modules.join("\n"));
|
|
320
|
+
addProperty(
|
|
321
|
+
pkg,
|
|
322
|
+
"cdx:pypi:modulesFrom",
|
|
323
|
+
curatedAddedSomething ? "alias-map,distribution-name" : "distribution-name",
|
|
324
|
+
);
|
|
325
|
+
return pkg;
|
|
326
|
+
}
|