@cdxgen/cdxgen 13.0.0 → 13.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -13
- package/bin/audit.js +6 -0
- package/bin/cdxgen.js +140 -8
- package/bin/convert.js +7 -0
- package/bin/evinse.js +1 -1
- package/bin/hbom.js +13 -4
- package/bin/tracebom.js +9 -1
- package/data/component-tags.json +8 -1
- package/data/frameworks-list.json +86 -1
- package/data/remediations.json +1144 -0
- package/data/rules/asar-archives.yaml +1 -1
- package/data/rules/build-fidelity.yaml +782 -0
- package/data/rules/package-integrity.yaml +13 -13
- package/lib/audit/index.js +48 -4
- package/lib/audit/scoring.js +2 -8
- package/lib/cli/bomAssembly.js +83 -15
- package/lib/cli/cliOptions.js +124 -7
- package/lib/cli/index.js +76 -48
- package/lib/cli/jsBom.js +32 -3
- package/lib/cli/jvmBom.js +1494 -1315
- package/lib/cli/managedBom.js +97 -28
- package/lib/cli/nativeBom.js +189 -10
- package/lib/core/buildLedger.js +799 -0
- package/lib/core/deferredExit.js +141 -0
- package/lib/core/env.js +69 -1
- package/lib/core/fs.js +60 -0
- package/lib/core/logger.js +76 -25
- package/lib/core/severity.js +16 -0
- package/lib/core/syncFileWriter.js +106 -0
- package/lib/core/ui.js +46 -2
- package/lib/ecosystems/ecosystems.js +9 -0
- package/lib/ecosystems/gradleutils.js +355 -15
- package/lib/ecosystems/parsers-js.js +26 -0
- package/lib/ecosystems/parsers-jvm.js +9 -0
- package/lib/ecosystems/parsers-misc.js +16 -2
- package/lib/ecosystems/parsers-python.js +171 -35
- package/lib/ecosystems/parsers-rust.js +12 -2
- package/lib/ecosystems/pypiClassifiers.js +326 -0
- package/lib/ecosystems/pythonutils.js +49 -1
- package/lib/ecosystems/rubyutils.js +13 -0
- package/lib/ecosystems/utils.js +9 -1
- package/lib/evinser/evinser.js +30 -12
- package/lib/helpers/remote/dependency-track.js +81 -14
- package/lib/inventory/auditCategories.js +9 -0
- package/lib/inventory/depsUtils.js +72 -0
- package/lib/inventory/dosai.js +431 -22
- package/lib/inventory/envcontext.js +211 -19
- package/lib/inventory/fetchBatch.js +57 -0
- package/lib/inventory/jvmToolEnv.js +593 -0
- package/lib/inventory/osinfo.js +123 -16
- package/lib/inventory/table.js +9 -5
- package/lib/inventory/toolRequirements.js +903 -0
- package/lib/managers/binary.js +216 -40
- package/lib/managers/docker.js +26 -5
- package/lib/server/server.js +1 -1
- package/lib/stages/postgen/annotationBuilder.js +97 -0
- package/lib/stages/postgen/auditBom.js +76 -79
- package/lib/stages/postgen/introspection/annotate.js +320 -0
- package/lib/stages/postgen/introspection/commandFacts.js +293 -0
- package/lib/stages/postgen/introspection/emit.js +192 -0
- package/lib/stages/postgen/introspection/formulationEvidence.js +427 -0
- package/lib/stages/postgen/introspection/markers.js +227 -0
- package/lib/stages/postgen/introspection/reflect.js +1214 -0
- package/lib/stages/postgen/introspection/report.js +1208 -0
- package/lib/stages/postgen/introspection/score.js +1067 -0
- package/lib/stages/postgen/introspection/shapeCommand.js +304 -0
- package/lib/stages/postgen/parentEdge.js +156 -0
- package/lib/stages/postgen/postgen.js +115 -2
- package/lib/stages/postgen/ruleEngine.js +17 -0
- package/lib/stages/pregen/envAudit.js +42 -2
- package/lib/stages/pregen/pregen.js +1171 -6
- package/lib/validator/bomValidator.js +11 -2
- package/package.json +14 -14
- package/types/lib/audit/index.d.ts.map +1 -1
- package/types/lib/audit/scoring.d.ts +2 -8
- package/types/lib/audit/scoring.d.ts.map +1 -1
- package/types/lib/cli/bomAssembly.d.ts +14 -0
- package/types/lib/cli/bomAssembly.d.ts.map +1 -1
- package/types/lib/cli/cliOptions.d.ts +2 -1
- package/types/lib/cli/cliOptions.d.ts.map +1 -1
- package/types/lib/cli/index.d.ts +8 -0
- package/types/lib/cli/index.d.ts.map +1 -1
- package/types/lib/cli/jsBom.d.ts.map +1 -1
- package/types/lib/cli/jvmBom.d.ts.map +1 -1
- package/types/lib/cli/managedBom.d.ts.map +1 -1
- package/types/lib/cli/nativeBom.d.ts.map +1 -1
- package/types/lib/core/buildLedger.d.ts +272 -0
- package/types/lib/core/buildLedger.d.ts.map +1 -0
- package/types/lib/core/deferredExit.d.ts +84 -0
- package/types/lib/core/deferredExit.d.ts.map +1 -0
- package/types/lib/core/env.d.ts +22 -1
- package/types/lib/core/env.d.ts.map +1 -1
- package/types/lib/core/fs.d.ts +21 -0
- package/types/lib/core/fs.d.ts.map +1 -1
- package/types/lib/core/logger.d.ts +14 -1
- package/types/lib/core/logger.d.ts.map +1 -1
- package/types/lib/core/severity.d.ts +11 -0
- package/types/lib/core/severity.d.ts.map +1 -0
- package/types/lib/core/syncFileWriter.d.ts +33 -0
- package/types/lib/core/syncFileWriter.d.ts.map +1 -0
- package/types/lib/core/ui.d.ts +1 -1
- package/types/lib/core/ui.d.ts.map +1 -1
- package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
- package/types/lib/ecosystems/gradleutils.d.ts +33 -0
- package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
- package/types/lib/ecosystems/pypiClassifiers.d.ts +80 -0
- package/types/lib/ecosystems/pypiClassifiers.d.ts.map +1 -0
- package/types/lib/ecosystems/pythonutils.d.ts +20 -0
- package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
- package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
- package/types/lib/ecosystems/utils.d.ts +4 -4
- package/types/lib/ecosystems/utils.d.ts.map +1 -1
- package/types/lib/evinser/evinser.d.ts +5 -1
- package/types/lib/evinser/evinser.d.ts.map +1 -1
- package/types/lib/helpers/remote/dependency-track.d.ts +45 -4
- package/types/lib/helpers/remote/dependency-track.d.ts.map +1 -1
- package/types/lib/inventory/auditCategories.d.ts +8 -0
- package/types/lib/inventory/auditCategories.d.ts.map +1 -1
- package/types/lib/inventory/depsUtils.d.ts.map +1 -1
- package/types/lib/inventory/dosai.d.ts +20 -0
- package/types/lib/inventory/dosai.d.ts.map +1 -1
- package/types/lib/inventory/envcontext.d.ts.map +1 -1
- package/types/lib/inventory/fetchBatch.d.ts +9 -0
- package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
- package/types/lib/inventory/jvmToolEnv.d.ts +189 -0
- package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -0
- package/types/lib/inventory/osinfo.d.ts +83 -2
- package/types/lib/inventory/osinfo.d.ts.map +1 -1
- package/types/lib/inventory/table.d.ts +4 -1
- package/types/lib/inventory/table.d.ts.map +1 -1
- package/types/lib/inventory/toolRequirements.d.ts +216 -0
- package/types/lib/inventory/toolRequirements.d.ts.map +1 -0
- package/types/lib/managers/binary.d.ts +71 -0
- package/types/lib/managers/binary.d.ts.map +1 -1
- package/types/lib/managers/docker.d.ts.map +1 -1
- package/types/lib/stages/postgen/annotationBuilder.d.ts +53 -0
- package/types/lib/stages/postgen/annotationBuilder.d.ts.map +1 -0
- package/types/lib/stages/postgen/auditBom.d.ts.map +1 -1
- package/types/lib/stages/postgen/introspection/annotate.d.ts +81 -0
- package/types/lib/stages/postgen/introspection/annotate.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts +32 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts +71 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts +154 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts +57 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts +77 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/report.d.ts +85 -0
- package/types/lib/stages/postgen/introspection/report.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/score.d.ts +149 -0
- package/types/lib/stages/postgen/introspection/score.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts +79 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts.map +1 -0
- package/types/lib/stages/postgen/parentEdge.d.ts +35 -0
- package/types/lib/stages/postgen/parentEdge.d.ts.map +1 -0
- package/types/lib/stages/postgen/postgen.d.ts +2 -2
- package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
- package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
- package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
- package/types/lib/stages/pregen/pregen.d.ts +131 -0
- package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
- package/types/lib/validator/bomValidator.d.ts.map +1 -1
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// PyPI Trove classifier interpretation and package -> module-name resolution.
|
|
2
|
+
//
|
|
3
|
+
// PyPI responses carry two structured signals that were previously read only for
|
|
4
|
+
// their license lines and then discarded:
|
|
5
|
+
//
|
|
6
|
+
// * `Framework :: X` — the author's declaration that the distribution belongs to
|
|
7
|
+
// framework X. It is more reliable than any name-prefix heuristic: it catches
|
|
8
|
+
// `flask-talisman`, `dj-database-url` and `starlette-context` without anyone
|
|
9
|
+
// maintaining a list, and it never fires on a package that merely has a
|
|
10
|
+
// framework's name embedded in its own.
|
|
11
|
+
// * `Topic :: ...` — a curated taxonomy of what the distribution does, which maps
|
|
12
|
+
// cleanly onto the description-tag vocabulary in `data/component-tags.json`.
|
|
13
|
+
//
|
|
14
|
+
// Tags derived here flow into `component.tags` and, for the `framework` tag, into
|
|
15
|
+
// `determinePackageType()` in `lib/cli/bomAssembly.js`, which already promotes any
|
|
16
|
+
// package carrying a `framework` tag to `type: "framework"`.
|
|
17
|
+
|
|
18
|
+
import { PYPI_MODULE_PACKAGE_MAPPING } from "../core/state.js";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `Framework ::` values that do not mean "an application is built on this".
|
|
22
|
+
*
|
|
23
|
+
* Two groups, both of which would otherwise put a `framework` tag — and therefore
|
|
24
|
+
* `type: "framework"` — on a large share of an ordinary dependency tree, drowning
|
|
25
|
+
* the runtime frameworks the tag exists to surface:
|
|
26
|
+
*
|
|
27
|
+
* * Development, build, test and documentation tooling. A pytest plugin
|
|
28
|
+
* declaring `Framework :: Pytest` is making a true statement, but it belongs
|
|
29
|
+
* to the dev dependencies, not to what the application runs on.
|
|
30
|
+
* * Concurrency runtimes. `Framework :: AsyncIO` is declared by a very large
|
|
31
|
+
* number of libraries to advertise that they work under asyncio; observed on
|
|
32
|
+
* `fastapi`, and `Framework :: AnyIO` likewise on `starlette`. It describes an
|
|
33
|
+
* execution model, not a framework the application is written against.
|
|
34
|
+
*
|
|
35
|
+
* The declared name is still emitted as a tag in every case, and still recorded in
|
|
36
|
+
* `cdx:pypi:frameworks`; only the generic `framework` tag is withheld.
|
|
37
|
+
*/
|
|
38
|
+
export const PYPI_NON_APPLICATION_FRAMEWORKS = new Set([
|
|
39
|
+
"anyio",
|
|
40
|
+
"asyncio",
|
|
41
|
+
"buildout",
|
|
42
|
+
"cookiecutter",
|
|
43
|
+
"flake8",
|
|
44
|
+
"hatch",
|
|
45
|
+
"hypothesis",
|
|
46
|
+
"mkdocs",
|
|
47
|
+
"pytest",
|
|
48
|
+
"robotframework",
|
|
49
|
+
"setuptools",
|
|
50
|
+
"sphinx",
|
|
51
|
+
"tox",
|
|
52
|
+
"trio",
|
|
53
|
+
"wheel",
|
|
54
|
+
]);
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* `Topic ::` prefix -> tags, longest prefix wins.
|
|
58
|
+
*
|
|
59
|
+
* Every value on the right-hand side is an existing entry in
|
|
60
|
+
* `data/component-tags.json` so that a tag derived from a classifier and the same
|
|
61
|
+
* tag derived from a description are indistinguishable to consumers.
|
|
62
|
+
*/
|
|
63
|
+
const TOPIC_TAGS = [
|
|
64
|
+
[
|
|
65
|
+
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
|
66
|
+
["framework"],
|
|
67
|
+
],
|
|
68
|
+
["Topic :: Software Development :: Object Brokering", ["rpc"]],
|
|
69
|
+
["Topic :: Software Development :: Testing", ["test"]],
|
|
70
|
+
["Topic :: Internet :: WWW/HTTP :: WSGI", ["web", "http", "wsgi"]],
|
|
71
|
+
["Topic :: Internet :: WWW/HTTP :: Session", ["web", "http", "auth"]],
|
|
72
|
+
["Topic :: Internet :: WWW/HTTP :: Dynamic Content", ["web", "http"]],
|
|
73
|
+
["Topic :: Internet :: WWW/HTTP", ["web", "http"]],
|
|
74
|
+
["Topic :: Internet :: XML-RPC", ["rpc", "xml"]],
|
|
75
|
+
["Topic :: Internet :: Proxy Servers", ["web", "traffic"]],
|
|
76
|
+
["Topic :: Database :: Front-Ends", ["database"]],
|
|
77
|
+
["Topic :: Database", ["database"]],
|
|
78
|
+
["Topic :: Security :: Cryptography", ["crypto", "security"]],
|
|
79
|
+
["Topic :: Security", ["security"]],
|
|
80
|
+
["Topic :: System :: Logging", ["logging"]],
|
|
81
|
+
["Topic :: System :: Distributed Computing", ["rpc"]],
|
|
82
|
+
["Topic :: Communications :: Email", ["mail"]],
|
|
83
|
+
["Topic :: Text Processing :: Markup :: HTML", ["html", "template"]],
|
|
84
|
+
["Topic :: Text Processing :: Markup :: XML", ["xml"]],
|
|
85
|
+
["Topic :: Scientific/Engineering :: Artificial Intelligence", ["ai", "ml"]],
|
|
86
|
+
[
|
|
87
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
88
|
+
["ml", "object-detect"],
|
|
89
|
+
],
|
|
90
|
+
["Topic :: Office/Business :: Financial", ["finance"]],
|
|
91
|
+
["Topic :: Terminals", ["cli"]],
|
|
92
|
+
["Topic :: Utilities", ["tools"]],
|
|
93
|
+
];
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Normalize a distribution name per PEP 503.
|
|
97
|
+
*
|
|
98
|
+
* @param {string} name Distribution name
|
|
99
|
+
* @returns {string} Normalized name, or an empty string for unusable input
|
|
100
|
+
*/
|
|
101
|
+
export function normalizePypiName(name) {
|
|
102
|
+
if (!name || typeof name !== "string") {
|
|
103
|
+
return "";
|
|
104
|
+
}
|
|
105
|
+
let out = "";
|
|
106
|
+
let lastWasSeparator = false;
|
|
107
|
+
for (const ch of name.trim().toLowerCase()) {
|
|
108
|
+
if (ch === "-" || ch === "_" || ch === ".") {
|
|
109
|
+
if (!lastWasSeparator && out.length) {
|
|
110
|
+
out += "-";
|
|
111
|
+
}
|
|
112
|
+
lastWasSeparator = true;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
out += ch;
|
|
116
|
+
lastWasSeparator = false;
|
|
117
|
+
}
|
|
118
|
+
return out.endsWith("-") ? out.slice(0, -1) : out;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* `data/pypi-pkg-aliases.json` maps module name -> distribution name. Inverting it
|
|
123
|
+
* gives the mapping consumers actually need: distribution -> the module names it
|
|
124
|
+
* installs. Built once on first use.
|
|
125
|
+
*
|
|
126
|
+
* @type {Map<string, string[]> | undefined}
|
|
127
|
+
*/
|
|
128
|
+
let invertedAliases;
|
|
129
|
+
|
|
130
|
+
function packageToModules() {
|
|
131
|
+
if (invertedAliases) {
|
|
132
|
+
return invertedAliases;
|
|
133
|
+
}
|
|
134
|
+
invertedAliases = new Map();
|
|
135
|
+
for (const [moduleName, pkgName] of Object.entries(
|
|
136
|
+
PYPI_MODULE_PACKAGE_MAPPING,
|
|
137
|
+
)) {
|
|
138
|
+
const key = normalizePypiName(pkgName);
|
|
139
|
+
if (!key || !moduleName) {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const existing = invertedAliases.get(key);
|
|
143
|
+
if (existing) {
|
|
144
|
+
if (!existing.includes(moduleName)) {
|
|
145
|
+
existing.push(moduleName);
|
|
146
|
+
}
|
|
147
|
+
} else {
|
|
148
|
+
invertedAliases.set(key, [moduleName]);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return invertedAliases;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Best-known import module names for a PyPI distribution.
|
|
156
|
+
*
|
|
157
|
+
* Two sources, in order: the curated alias map (which is where the cases that
|
|
158
|
+
* cannot be derived from the name live — `pyyaml` installs `yaml`, `absl-py`
|
|
159
|
+
* installs `absl`, `beautifulsoup4` installs `bs4`), and the name-derived form,
|
|
160
|
+
* which is correct for the large majority of distributions. Callers that need to
|
|
161
|
+
* distinguish the two should read the alias map themselves; for tagging purposes
|
|
162
|
+
* both are usable and a wrong extra candidate simply matches nothing.
|
|
163
|
+
*
|
|
164
|
+
* This is deliberately not a guess at *every* module a distribution ships — only
|
|
165
|
+
* at its top-level import names.
|
|
166
|
+
*
|
|
167
|
+
* @param {string} name Distribution name, e.g. "PyYAML"
|
|
168
|
+
* @returns {string[]} Candidate top-level module names, deduplicated
|
|
169
|
+
*/
|
|
170
|
+
export function pypiModulesForPackage(name) {
|
|
171
|
+
const normalized = normalizePypiName(name);
|
|
172
|
+
if (!normalized) {
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
const modules = [...(packageToModules().get(normalized) || [])];
|
|
176
|
+
// The name-derived form. PEP 503 collapses `.` and `_` to `-`; imports use `_`.
|
|
177
|
+
const derived = normalized.replaceAll("-", "_");
|
|
178
|
+
if (derived && !modules.includes(derived)) {
|
|
179
|
+
modules.push(derived);
|
|
180
|
+
}
|
|
181
|
+
return modules;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Interpret a PyPI classifier list.
|
|
186
|
+
*
|
|
187
|
+
* @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
|
|
188
|
+
* @returns {{frameworks: string[], tags: string[]}} Declared frameworks (lowercased,
|
|
189
|
+
* e.g. `["django"]`) and the tag vocabulary they and the `Topic ::` lines imply
|
|
190
|
+
*/
|
|
191
|
+
export function parsePypiClassifiers(classifiers) {
|
|
192
|
+
const frameworks = [];
|
|
193
|
+
const tags = new Set();
|
|
194
|
+
if (!Array.isArray(classifiers)) {
|
|
195
|
+
return { frameworks, tags: [] };
|
|
196
|
+
}
|
|
197
|
+
for (const classifier of classifiers) {
|
|
198
|
+
if (!classifier || typeof classifier !== "string") {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (classifier.startsWith("Framework :: ")) {
|
|
202
|
+
// "Framework :: Django :: 5.0" -> "django". The version segment is dropped:
|
|
203
|
+
// it is the framework version the package supports, not a distinct signal.
|
|
204
|
+
const declared = classifier.split(" :: ")[1]?.trim();
|
|
205
|
+
if (!declared) {
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const name = declared.toLowerCase().replaceAll(" ", "-");
|
|
209
|
+
if (!frameworks.includes(name)) {
|
|
210
|
+
frameworks.push(name);
|
|
211
|
+
}
|
|
212
|
+
tags.add(name);
|
|
213
|
+
if (!PYPI_NON_APPLICATION_FRAMEWORKS.has(name)) {
|
|
214
|
+
tags.add("framework");
|
|
215
|
+
}
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
if (!classifier.startsWith("Topic :: ")) {
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
for (const [prefix, topicTags] of TOPIC_TAGS) {
|
|
222
|
+
if (classifier === prefix || classifier.startsWith(`${prefix} ::`)) {
|
|
223
|
+
for (const tag of topicTags) {
|
|
224
|
+
tags.add(tag);
|
|
225
|
+
}
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return { frameworks, tags: Array.from(tags) };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function addProperty(pkg, name, value) {
|
|
234
|
+
if (!value) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (!pkg.properties) {
|
|
238
|
+
pkg.properties = [];
|
|
239
|
+
}
|
|
240
|
+
const existing = pkg.properties.find((prop) => prop.name === name);
|
|
241
|
+
if (existing) {
|
|
242
|
+
existing.value = value;
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
pkg.properties.push({ name, value });
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Attach classifier-derived tags and properties to a package.
|
|
250
|
+
*
|
|
251
|
+
* Tags are merged into any the package already carries rather than replacing
|
|
252
|
+
* them — `lib/managers/binary.js` sets `tags: ["source"]` as an identity marker,
|
|
253
|
+
* and losing it would change how that package is reconciled later.
|
|
254
|
+
*
|
|
255
|
+
* @param {Object} pkg Package object, mutated in place
|
|
256
|
+
* @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
|
|
257
|
+
* @returns {Object} The same package object
|
|
258
|
+
*/
|
|
259
|
+
export function applyPypiClassifierMetadata(pkg, classifiers) {
|
|
260
|
+
if (!pkg || !Array.isArray(classifiers) || !classifiers.length) {
|
|
261
|
+
return pkg;
|
|
262
|
+
}
|
|
263
|
+
const { frameworks, tags } = parsePypiClassifiers(classifiers);
|
|
264
|
+
if (tags.length) {
|
|
265
|
+
const merged = new Set(Array.isArray(pkg.tags) ? pkg.tags : []);
|
|
266
|
+
for (const tag of tags) {
|
|
267
|
+
merged.add(tag);
|
|
268
|
+
}
|
|
269
|
+
pkg.tags = Array.from(merged);
|
|
270
|
+
}
|
|
271
|
+
if (frameworks.length) {
|
|
272
|
+
addProperty(pkg, "cdx:pypi:frameworks", frameworks.join(","));
|
|
273
|
+
}
|
|
274
|
+
// The full list is retained so that consumers wanting the whole taxonomy are
|
|
275
|
+
// not limited to the tag vocabulary mapped above. Classifiers come from a
|
|
276
|
+
// closed PyPI-maintained set, so there is no free-text or secret exposure.
|
|
277
|
+
addProperty(
|
|
278
|
+
pkg,
|
|
279
|
+
"cdx:pypi:classifiers",
|
|
280
|
+
classifiers.filter((c) => typeof c === "string" && c.length).join("\n"),
|
|
281
|
+
);
|
|
282
|
+
return pkg;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Attach the distribution's top-level import module names as `internal:Namespaces`.
|
|
287
|
+
*
|
|
288
|
+
* This is the same property jar, gem, nuget and composer components already carry,
|
|
289
|
+
* so downstream consumers that map namespaces back to components need no
|
|
290
|
+
* pypi-specific path. Unlike the jar case the value is name-derived and
|
|
291
|
+
* alias-curated rather than read out of the artifact, which is why
|
|
292
|
+
* `cdx:pypi:modulesFrom` records the provenance.
|
|
293
|
+
*
|
|
294
|
+
* @param {Object} pkg Package object, mutated in place
|
|
295
|
+
* @returns {Object} The same package object
|
|
296
|
+
*/
|
|
297
|
+
export function applyPypiModuleNames(pkg) {
|
|
298
|
+
if (!pkg?.name) {
|
|
299
|
+
return pkg;
|
|
300
|
+
}
|
|
301
|
+
// Extras (`requests[security]`) are not part of the distribution name.
|
|
302
|
+
const name = pkg.name.includes("[") ? pkg.name.split("[")[0] : pkg.name;
|
|
303
|
+
if (name.includes("https")) {
|
|
304
|
+
return pkg;
|
|
305
|
+
}
|
|
306
|
+
const normalized = normalizePypiName(name);
|
|
307
|
+
const modules = pypiModulesForPackage(name);
|
|
308
|
+
if (!modules.length) {
|
|
309
|
+
return pkg;
|
|
310
|
+
}
|
|
311
|
+
// Most alias-map entries map a distribution to the module its own name already
|
|
312
|
+
// implies, so the map having an entry says nothing. The provenance worth
|
|
313
|
+
// recording is whether it contributed a module the name could not have given -
|
|
314
|
+
// that is the difference between `flask` and `pyyaml` -> `yaml`.
|
|
315
|
+
const derived = normalized.replaceAll("-", "_");
|
|
316
|
+
const curatedAddedSomething = (packageToModules().get(normalized) || []).some(
|
|
317
|
+
(moduleName) => moduleName !== derived,
|
|
318
|
+
);
|
|
319
|
+
addProperty(pkg, "internal:Namespaces", modules.join("\n"));
|
|
320
|
+
addProperty(
|
|
321
|
+
pkg,
|
|
322
|
+
"cdx:pypi:modulesFrom",
|
|
323
|
+
curatedAddedSomething ? "alias-map,distribution-name" : "distribution-name",
|
|
324
|
+
);
|
|
325
|
+
return pkg;
|
|
326
|
+
}
|
|
@@ -331,6 +331,53 @@ export function get_python_command_from_env(env) {
|
|
|
331
331
|
return fallbackCmd;
|
|
332
332
|
}
|
|
333
333
|
|
|
334
|
+
/**
|
|
335
|
+
* Is this import satisfied by a module that lives in the project itself?
|
|
336
|
+
*
|
|
337
|
+
* `atom parsedeps` reports every non-stdlib import it sees, including imports of the
|
|
338
|
+
* project's own packages. Attributing those to a distribution invents dependencies:
|
|
339
|
+
* `data/pypi-pkg-aliases.json` maps the module `app` to `zope2`, `test` to `pytabix`,
|
|
340
|
+
* `models` to `asposestorage` and `src` to `auto-mix-prep`, so any project with a
|
|
341
|
+
* local package of that name acquires an obscure component it never depended on -
|
|
342
|
+
* along with every namespace and tag that component carries downstream. Filtering by
|
|
343
|
+
* name would need a denylist of every plausible first-party name; asking the
|
|
344
|
+
* filesystem whether the module is ours answers it exactly.
|
|
345
|
+
*
|
|
346
|
+
* Checks the project root and a `src/` layout for a module file, a package directory,
|
|
347
|
+
* and a PEP 420 namespace package directory.
|
|
348
|
+
*
|
|
349
|
+
* @param {string} src Path to the project being analysed
|
|
350
|
+
* @param {string} name Imported top-level module name
|
|
351
|
+
* @returns {boolean} `true` if the project supplies this module itself
|
|
352
|
+
*/
|
|
353
|
+
export function isFirstPartyModule(src, name) {
|
|
354
|
+
if (!src || !name || name.includes("/") || name.includes("\\")) {
|
|
355
|
+
return false;
|
|
356
|
+
}
|
|
357
|
+
for (const root of [src, join(src, "src")]) {
|
|
358
|
+
if (
|
|
359
|
+
safeExistsSync(join(root, `${name}.py`)) ||
|
|
360
|
+
safeExistsSync(join(root, name, "__init__.py")) ||
|
|
361
|
+
safeExistsSync(join(root, name, "__main__.py"))
|
|
362
|
+
) {
|
|
363
|
+
return true;
|
|
364
|
+
}
|
|
365
|
+
// PEP 420 namespace package: a directory of the right name holding python
|
|
366
|
+
// sources but no __init__.py.
|
|
367
|
+
const candidate = join(root, name);
|
|
368
|
+
if (safeExistsSync(candidate)) {
|
|
369
|
+
try {
|
|
370
|
+
if (readdirSync(candidate).some((f) => f.endsWith(".py"))) {
|
|
371
|
+
return true;
|
|
372
|
+
}
|
|
373
|
+
} catch {
|
|
374
|
+
// not a readable directory - treat as not first-party
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return false;
|
|
379
|
+
}
|
|
380
|
+
|
|
334
381
|
/**
|
|
335
382
|
* Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
|
|
336
383
|
*
|
|
@@ -362,7 +409,8 @@ export async function getPyModules(src, epkgList, options) {
|
|
|
362
409
|
(x) =>
|
|
363
410
|
!pyDefaultModules.has(x.name.toLowerCase()) &&
|
|
364
411
|
!x.name.startsWith("_") &&
|
|
365
|
-
!x.name.startsWith(".")
|
|
412
|
+
!x.name.startsWith(".") &&
|
|
413
|
+
!isFirstPartyModule(src, x.name),
|
|
366
414
|
);
|
|
367
415
|
let pkgList = modList.map((p) => {
|
|
368
416
|
const apkg = {
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
prefetchEnabled,
|
|
20
20
|
prefetchedResponse,
|
|
21
21
|
prefetchJson,
|
|
22
|
+
recordPolicyDegradationFromError,
|
|
22
23
|
} from "../inventory/fetchBatch.js";
|
|
23
24
|
|
|
24
25
|
// FIXME. This has to get removed, once we improve the module detection one-liner.
|
|
@@ -780,6 +781,7 @@ export async function getRubyGemsMetadata(pkgList) {
|
|
|
780
781
|
continue;
|
|
781
782
|
}
|
|
782
783
|
} catch (err) {
|
|
784
|
+
recordPolicyDegradationFromError(err);
|
|
783
785
|
if (DEBUG_MODE) {
|
|
784
786
|
console.error(name, err);
|
|
785
787
|
}
|
|
@@ -832,6 +834,7 @@ async function enrichGemFromVersionEndpoint(
|
|
|
832
834
|
}
|
|
833
835
|
applyGemMetadata(p, body);
|
|
834
836
|
} catch (err) {
|
|
837
|
+
recordPolicyDegradationFromError(err);
|
|
835
838
|
if (DEBUG_MODE) {
|
|
836
839
|
console.error(p, err);
|
|
837
840
|
}
|
|
@@ -1241,6 +1244,16 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
|
|
|
1241
1244
|
pkg.externalReferences.push({ type: refType, url: value, comment: key });
|
|
1242
1245
|
}
|
|
1243
1246
|
}
|
|
1247
|
+
// `s.homepage = "..."` is the project website. Keep it as an external
|
|
1248
|
+
// reference; the transient `homepage` key is a plain string that never
|
|
1249
|
+
// reaches the BOM because component conversion expects the `{ url }` shape.
|
|
1250
|
+
if (pkg.homepage) {
|
|
1251
|
+
pkg.externalReferences = pkg.externalReferences || [];
|
|
1252
|
+
if (!pkg.externalReferences.some((ref) => ref.url === pkg.homepage)) {
|
|
1253
|
+
pkg.externalReferences.push({ type: "website", url: pkg.homepage });
|
|
1254
|
+
}
|
|
1255
|
+
delete pkg.homepage;
|
|
1256
|
+
}
|
|
1244
1257
|
if (pkg.name) {
|
|
1245
1258
|
const purlString = toGemPurl(pkg.name, pkg.version, gemPlatform);
|
|
1246
1259
|
pkg.purl = purlString;
|
package/lib/ecosystems/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Deprecated re-export barrel.
|
|
3
3
|
*
|
|
4
|
-
* `utils.js` declares nothing of its own. It re-exports
|
|
4
|
+
* `utils.js` declares nothing of its own. It re-exports every public name that
|
|
5
5
|
* used to live here so existing imports keep working for one major version. Each
|
|
6
6
|
* implementation, and its JSDoc, now lives in the leaf module it was moved to
|
|
7
7
|
* (`purl.js`, `spdx.js`, `deps.js`, `ecosystems.js`, `parsers-*.js`, `core-*.js`,
|
|
@@ -191,11 +191,13 @@ import {
|
|
|
191
191
|
import { getMavenCommand, getMillCommand } from "./gradleutils.js";
|
|
192
192
|
import { createNpmWorkspacePurl } from "./npmutils.js";
|
|
193
193
|
import {
|
|
194
|
+
getCentralPackageVersions,
|
|
194
195
|
getPropertyGroupTextNodes,
|
|
195
196
|
parseCsPkgData,
|
|
196
197
|
parseCsPkgLockData,
|
|
197
198
|
parseCsProjAssetsData,
|
|
198
199
|
parseCsProjData,
|
|
200
|
+
parseDirectoryPackagesProps,
|
|
199
201
|
parseNupkg,
|
|
200
202
|
parseNuspecData,
|
|
201
203
|
parsePaketLockData,
|
|
@@ -220,12 +222,14 @@ import {
|
|
|
220
222
|
parsePackageJsonName,
|
|
221
223
|
parsePkgJson,
|
|
222
224
|
parsePkgLock,
|
|
225
|
+
parsePnpmAliasRef,
|
|
223
226
|
parsePnpmLock,
|
|
224
227
|
parsePnpmWorkspace,
|
|
225
228
|
parseYarnLock,
|
|
226
229
|
parseYarnWorkspace,
|
|
227
230
|
pnpmMetadata,
|
|
228
231
|
splitCommandArgs,
|
|
232
|
+
stripPnpmPeerSuffix,
|
|
229
233
|
yarnLockToIdentMap,
|
|
230
234
|
} from "./parsers-js.js";
|
|
231
235
|
import {
|
|
@@ -369,6 +373,7 @@ export {
|
|
|
369
373
|
getAllFiles,
|
|
370
374
|
getAllFilesWithIgnore,
|
|
371
375
|
getAtomCommand,
|
|
376
|
+
getCentralPackageVersions,
|
|
372
377
|
getCratesMetadata,
|
|
373
378
|
getDartMetadata,
|
|
374
379
|
getDefaultBomAuditCategories,
|
|
@@ -465,6 +470,7 @@ export {
|
|
|
465
470
|
parseCsProjAssetsData,
|
|
466
471
|
parseCsProjData,
|
|
467
472
|
parseCUsageSlice,
|
|
473
|
+
parseDirectoryPackagesProps,
|
|
468
474
|
parseEdnData,
|
|
469
475
|
parseFlakeLock,
|
|
470
476
|
parseFlakeNix,
|
|
@@ -503,6 +509,7 @@ export {
|
|
|
503
509
|
parsePixiTomlFile,
|
|
504
510
|
parsePkgJson,
|
|
505
511
|
parsePkgLock,
|
|
512
|
+
parsePnpmAliasRef,
|
|
506
513
|
parsePnpmLock,
|
|
507
514
|
parsePnpmWorkspace,
|
|
508
515
|
parsePodfileLock,
|
|
@@ -565,6 +572,7 @@ export {
|
|
|
565
572
|
shouldFetchVCS,
|
|
566
573
|
shouldRunPredictiveBomAudit,
|
|
567
574
|
splitCommandArgs,
|
|
575
|
+
stripPnpmPeerSuffix,
|
|
568
576
|
TABLE_BORDER_STYLE,
|
|
569
577
|
TIMEOUT_MS,
|
|
570
578
|
toCamel,
|
package/lib/evinser/evinser.js
CHANGED
|
@@ -42,12 +42,15 @@ import {
|
|
|
42
42
|
import { collectMvnDependencies } from "../inventory/deps.js";
|
|
43
43
|
import { mergeServices } from "../inventory/depsUtils.js";
|
|
44
44
|
import {
|
|
45
|
+
collectDosaiAiComponents,
|
|
45
46
|
collectDosaiDataFlowFrames,
|
|
46
47
|
collectDosaiPurlEvidence,
|
|
48
|
+
collectDosaiServiceComponents,
|
|
47
49
|
collectDosaiServicesFromMethods,
|
|
48
50
|
createDosaiDataFlowSlice,
|
|
49
51
|
createDosaiMethodsSlice,
|
|
50
52
|
isDosaiDotnetLanguage,
|
|
53
|
+
normalizeDosaiServiceMap,
|
|
51
54
|
persistDosaiSemanticsReport,
|
|
52
55
|
} from "../inventory/dosai.js";
|
|
53
56
|
import { parseOccurrenceEvidenceLocation } from "../inventory/evidenceUtils.js";
|
|
@@ -662,6 +665,7 @@ export async function analyzeProject(dbObjMap, options) {
|
|
|
662
665
|
const bomJson = JSON.parse(fs.readFileSync(bomFile, "utf8"));
|
|
663
666
|
const components = bomJson.components || [];
|
|
664
667
|
let cryptoComponents = [];
|
|
668
|
+
let aiComponents = [];
|
|
665
669
|
let cryptoGeneratePurls = {};
|
|
666
670
|
let openapiSpecFile;
|
|
667
671
|
// Load any existing purl-location information from the sbom.
|
|
@@ -750,6 +754,13 @@ export async function analyzeProject(dbObjMap, options) {
|
|
|
750
754
|
purlLocationMap[purl].add(location);
|
|
751
755
|
}
|
|
752
756
|
}
|
|
757
|
+
// Rich Services[] inventory first (bom-refs, trust zones, data); endpoint
|
|
758
|
+
// derivation fills any services the providers did not emit.
|
|
759
|
+
// AI components must be carried out of analyzeProject explicitly. Mutating the local
|
|
760
|
+
// `components` array here had no effect: the return below omits it and createEvinseFile
|
|
761
|
+
// re-reads the BOM from disk, so the whole AI-BOM contribution was silently discarded.
|
|
762
|
+
collectDosaiAiComponents(usageSlice, aiComponents);
|
|
763
|
+
servicesMap = collectDosaiServiceComponents(usageSlice, servicesMap);
|
|
753
764
|
servicesMap = collectDosaiServicesFromMethods(usageSlice, servicesMap);
|
|
754
765
|
userDefinedTypesMap = {};
|
|
755
766
|
}
|
|
@@ -802,6 +813,7 @@ export async function analyzeProject(dbObjMap, options) {
|
|
|
802
813
|
tempDirOwned: retMap?.tempDirOwned,
|
|
803
814
|
userDefinedTypesMap,
|
|
804
815
|
cryptoComponents,
|
|
816
|
+
aiComponents,
|
|
805
817
|
cryptoGeneratePurls,
|
|
806
818
|
};
|
|
807
819
|
}
|
|
@@ -832,6 +844,7 @@ export async function analyzeProject(dbObjMap, options) {
|
|
|
832
844
|
const retMap = collectReachableFrames(language, reachablesSlice);
|
|
833
845
|
dataFlowFrames = retMap.dataFlowFrames;
|
|
834
846
|
cryptoComponents = retMap.cryptoComponents;
|
|
847
|
+
aiComponents = retMap.aiComponents || [];
|
|
835
848
|
cryptoGeneratePurls = retMap.cryptoGeneratePurls;
|
|
836
849
|
}
|
|
837
850
|
// Reuse existing usages slices
|
|
@@ -960,6 +973,7 @@ export async function analyzeProject(dbObjMap, options) {
|
|
|
960
973
|
tempDirOwned: retMap?.tempDirOwned,
|
|
961
974
|
userDefinedTypesMap,
|
|
962
975
|
cryptoComponents,
|
|
976
|
+
aiComponents,
|
|
963
977
|
cryptoGeneratePurls,
|
|
964
978
|
openapiSpecFile,
|
|
965
979
|
};
|
|
@@ -1804,7 +1818,7 @@ export function extractEndpoints(language, code) {
|
|
|
1804
1818
|
* @param {Object} options Command line options
|
|
1805
1819
|
* @returns
|
|
1806
1820
|
*/
|
|
1807
|
-
export function createEvinseFile(sliceArtefacts, options) {
|
|
1821
|
+
export async function createEvinseFile(sliceArtefacts, options) {
|
|
1808
1822
|
const {
|
|
1809
1823
|
tempDir,
|
|
1810
1824
|
tempDirOwned,
|
|
@@ -1817,6 +1831,7 @@ export function createEvinseFile(sliceArtefacts, options) {
|
|
|
1817
1831
|
componentPropertiesMap = {},
|
|
1818
1832
|
metadataProperties = [],
|
|
1819
1833
|
cryptoComponents,
|
|
1834
|
+
aiComponents,
|
|
1820
1835
|
cryptoGeneratePurls,
|
|
1821
1836
|
} = sliceArtefacts;
|
|
1822
1837
|
const bomFile = options.input;
|
|
@@ -1873,16 +1888,12 @@ export function createEvinseFile(sliceArtefacts, options) {
|
|
|
1873
1888
|
}
|
|
1874
1889
|
} // for
|
|
1875
1890
|
if (servicesMap && Object.keys(servicesMap).length) {
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
"x-trust-boundary": servicesMap[serviceName].xTrustBoundary,
|
|
1883
|
-
properties: servicesMap[serviceName].properties,
|
|
1884
|
-
});
|
|
1885
|
-
}
|
|
1891
|
+
// normalizeDosaiServiceMap is the single mapping from the internal service
|
|
1892
|
+
// definitions to CycloneDX services[]. The OpenAPI/Java/Python detectors set
|
|
1893
|
+
// the camelCase `xTrustBoundary` while dosai's provider inventory (schema
|
|
1894
|
+
// 4.0.0) carries stable bom-refs, trust zones, data classifications, and
|
|
1895
|
+
// evidence occurrences; both shapes flow through the same normalizer.
|
|
1896
|
+
const services = normalizeDosaiServiceMap(servicesMap);
|
|
1886
1897
|
// Add to existing services while preserving CycloneDX uniqueItems validity
|
|
1887
1898
|
bomJson.services = mergeServices(bomJson.services || [], services);
|
|
1888
1899
|
servicesPresent = true;
|
|
@@ -1891,6 +1902,10 @@ export function createEvinseFile(sliceArtefacts, options) {
|
|
|
1891
1902
|
if (cryptoComponents?.length) {
|
|
1892
1903
|
bomJson.components = bomJson.components.concat(cryptoComponents);
|
|
1893
1904
|
}
|
|
1905
|
+
// Add the AI components (machine-learning-model, dataset) discovered by dosai
|
|
1906
|
+
if (aiComponents?.length) {
|
|
1907
|
+
bomJson.components = bomJson.components.concat(aiComponents);
|
|
1908
|
+
}
|
|
1894
1909
|
// Fix the dependencies section with provides information
|
|
1895
1910
|
if (
|
|
1896
1911
|
cryptoGeneratePurls &&
|
|
@@ -1943,7 +1958,10 @@ export function createEvinseFile(sliceArtefacts, options) {
|
|
|
1943
1958
|
// Redo post-processing with evinse data. `executeOsQuery` has to be injected
|
|
1944
1959
|
// because postgen no longer imports it directly (stages is below managers);
|
|
1945
1960
|
// without it the crypto formulation path receives `undefined` and throws.
|
|
1946
|
-
const bomNSData = postProcess(
|
|
1961
|
+
const bomNSData = await postProcess(
|
|
1962
|
+
{ bomJson },
|
|
1963
|
+
{ ...options, executeOsQuery },
|
|
1964
|
+
);
|
|
1947
1965
|
safeWriteSync(
|
|
1948
1966
|
evinseOutFile,
|
|
1949
1967
|
JSON.stringify(bomNSData.bomJson, null, options.jsonPretty ? 2 : null),
|