@cdxgen/cdxgen 13.0.0 → 13.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +26 -13
  2. package/bin/audit.js +6 -0
  3. package/bin/cdxgen.js +140 -8
  4. package/bin/convert.js +7 -0
  5. package/bin/evinse.js +1 -1
  6. package/bin/hbom.js +13 -4
  7. package/bin/tracebom.js +9 -1
  8. package/data/component-tags.json +8 -1
  9. package/data/frameworks-list.json +86 -1
  10. package/data/remediations.json +1144 -0
  11. package/data/rules/asar-archives.yaml +1 -1
  12. package/data/rules/build-fidelity.yaml +782 -0
  13. package/data/rules/package-integrity.yaml +13 -13
  14. package/lib/audit/index.js +48 -4
  15. package/lib/audit/scoring.js +2 -8
  16. package/lib/cli/bomAssembly.js +83 -15
  17. package/lib/cli/cliOptions.js +124 -7
  18. package/lib/cli/index.js +76 -48
  19. package/lib/cli/jsBom.js +32 -3
  20. package/lib/cli/jvmBom.js +1494 -1315
  21. package/lib/cli/managedBom.js +97 -28
  22. package/lib/cli/nativeBom.js +189 -10
  23. package/lib/core/buildLedger.js +799 -0
  24. package/lib/core/deferredExit.js +141 -0
  25. package/lib/core/env.js +69 -1
  26. package/lib/core/fs.js +60 -0
  27. package/lib/core/logger.js +76 -25
  28. package/lib/core/severity.js +16 -0
  29. package/lib/core/syncFileWriter.js +106 -0
  30. package/lib/core/ui.js +46 -2
  31. package/lib/ecosystems/ecosystems.js +9 -0
  32. package/lib/ecosystems/gradleutils.js +355 -15
  33. package/lib/ecosystems/parsers-js.js +26 -0
  34. package/lib/ecosystems/parsers-jvm.js +9 -0
  35. package/lib/ecosystems/parsers-misc.js +16 -2
  36. package/lib/ecosystems/parsers-python.js +171 -35
  37. package/lib/ecosystems/parsers-rust.js +12 -2
  38. package/lib/ecosystems/pypiClassifiers.js +326 -0
  39. package/lib/ecosystems/pythonutils.js +49 -1
  40. package/lib/ecosystems/rubyutils.js +13 -0
  41. package/lib/ecosystems/utils.js +9 -1
  42. package/lib/evinser/evinser.js +30 -12
  43. package/lib/helpers/remote/dependency-track.js +81 -14
  44. package/lib/inventory/auditCategories.js +9 -0
  45. package/lib/inventory/depsUtils.js +72 -0
  46. package/lib/inventory/dosai.js +431 -22
  47. package/lib/inventory/envcontext.js +211 -19
  48. package/lib/inventory/fetchBatch.js +57 -0
  49. package/lib/inventory/jvmToolEnv.js +593 -0
  50. package/lib/inventory/osinfo.js +123 -16
  51. package/lib/inventory/table.js +9 -5
  52. package/lib/inventory/toolRequirements.js +903 -0
  53. package/lib/managers/binary.js +216 -40
  54. package/lib/managers/docker.js +26 -5
  55. package/lib/server/server.js +1 -1
  56. package/lib/stages/postgen/annotationBuilder.js +97 -0
  57. package/lib/stages/postgen/auditBom.js +76 -79
  58. package/lib/stages/postgen/introspection/annotate.js +320 -0
  59. package/lib/stages/postgen/introspection/commandFacts.js +293 -0
  60. package/lib/stages/postgen/introspection/emit.js +192 -0
  61. package/lib/stages/postgen/introspection/formulationEvidence.js +427 -0
  62. package/lib/stages/postgen/introspection/markers.js +227 -0
  63. package/lib/stages/postgen/introspection/reflect.js +1214 -0
  64. package/lib/stages/postgen/introspection/report.js +1208 -0
  65. package/lib/stages/postgen/introspection/score.js +1067 -0
  66. package/lib/stages/postgen/introspection/shapeCommand.js +304 -0
  67. package/lib/stages/postgen/parentEdge.js +156 -0
  68. package/lib/stages/postgen/postgen.js +115 -2
  69. package/lib/stages/postgen/ruleEngine.js +17 -0
  70. package/lib/stages/pregen/envAudit.js +42 -2
  71. package/lib/stages/pregen/pregen.js +1171 -6
  72. package/lib/validator/bomValidator.js +11 -2
  73. package/package.json +14 -14
  74. package/types/lib/audit/index.d.ts.map +1 -1
  75. package/types/lib/audit/scoring.d.ts +2 -8
  76. package/types/lib/audit/scoring.d.ts.map +1 -1
  77. package/types/lib/cli/bomAssembly.d.ts +14 -0
  78. package/types/lib/cli/bomAssembly.d.ts.map +1 -1
  79. package/types/lib/cli/cliOptions.d.ts +2 -1
  80. package/types/lib/cli/cliOptions.d.ts.map +1 -1
  81. package/types/lib/cli/index.d.ts +8 -0
  82. package/types/lib/cli/index.d.ts.map +1 -1
  83. package/types/lib/cli/jsBom.d.ts.map +1 -1
  84. package/types/lib/cli/jvmBom.d.ts.map +1 -1
  85. package/types/lib/cli/managedBom.d.ts.map +1 -1
  86. package/types/lib/cli/nativeBom.d.ts.map +1 -1
  87. package/types/lib/core/buildLedger.d.ts +272 -0
  88. package/types/lib/core/buildLedger.d.ts.map +1 -0
  89. package/types/lib/core/deferredExit.d.ts +84 -0
  90. package/types/lib/core/deferredExit.d.ts.map +1 -0
  91. package/types/lib/core/env.d.ts +22 -1
  92. package/types/lib/core/env.d.ts.map +1 -1
  93. package/types/lib/core/fs.d.ts +21 -0
  94. package/types/lib/core/fs.d.ts.map +1 -1
  95. package/types/lib/core/logger.d.ts +14 -1
  96. package/types/lib/core/logger.d.ts.map +1 -1
  97. package/types/lib/core/severity.d.ts +11 -0
  98. package/types/lib/core/severity.d.ts.map +1 -0
  99. package/types/lib/core/syncFileWriter.d.ts +33 -0
  100. package/types/lib/core/syncFileWriter.d.ts.map +1 -0
  101. package/types/lib/core/ui.d.ts +1 -1
  102. package/types/lib/core/ui.d.ts.map +1 -1
  103. package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
  104. package/types/lib/ecosystems/gradleutils.d.ts +33 -0
  105. package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
  106. package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
  107. package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
  108. package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
  109. package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
  110. package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
  111. package/types/lib/ecosystems/pypiClassifiers.d.ts +80 -0
  112. package/types/lib/ecosystems/pypiClassifiers.d.ts.map +1 -0
  113. package/types/lib/ecosystems/pythonutils.d.ts +20 -0
  114. package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
  115. package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
  116. package/types/lib/ecosystems/utils.d.ts +4 -4
  117. package/types/lib/ecosystems/utils.d.ts.map +1 -1
  118. package/types/lib/evinser/evinser.d.ts +5 -1
  119. package/types/lib/evinser/evinser.d.ts.map +1 -1
  120. package/types/lib/helpers/remote/dependency-track.d.ts +45 -4
  121. package/types/lib/helpers/remote/dependency-track.d.ts.map +1 -1
  122. package/types/lib/inventory/auditCategories.d.ts +8 -0
  123. package/types/lib/inventory/auditCategories.d.ts.map +1 -1
  124. package/types/lib/inventory/depsUtils.d.ts.map +1 -1
  125. package/types/lib/inventory/dosai.d.ts +20 -0
  126. package/types/lib/inventory/dosai.d.ts.map +1 -1
  127. package/types/lib/inventory/envcontext.d.ts.map +1 -1
  128. package/types/lib/inventory/fetchBatch.d.ts +9 -0
  129. package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
  130. package/types/lib/inventory/jvmToolEnv.d.ts +189 -0
  131. package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -0
  132. package/types/lib/inventory/osinfo.d.ts +83 -2
  133. package/types/lib/inventory/osinfo.d.ts.map +1 -1
  134. package/types/lib/inventory/table.d.ts +4 -1
  135. package/types/lib/inventory/table.d.ts.map +1 -1
  136. package/types/lib/inventory/toolRequirements.d.ts +216 -0
  137. package/types/lib/inventory/toolRequirements.d.ts.map +1 -0
  138. package/types/lib/managers/binary.d.ts +71 -0
  139. package/types/lib/managers/binary.d.ts.map +1 -1
  140. package/types/lib/managers/docker.d.ts.map +1 -1
  141. package/types/lib/stages/postgen/annotationBuilder.d.ts +53 -0
  142. package/types/lib/stages/postgen/annotationBuilder.d.ts.map +1 -0
  143. package/types/lib/stages/postgen/auditBom.d.ts.map +1 -1
  144. package/types/lib/stages/postgen/introspection/annotate.d.ts +81 -0
  145. package/types/lib/stages/postgen/introspection/annotate.d.ts.map +1 -0
  146. package/types/lib/stages/postgen/introspection/commandFacts.d.ts +32 -0
  147. package/types/lib/stages/postgen/introspection/commandFacts.d.ts.map +1 -0
  148. package/types/lib/stages/postgen/introspection/emit.d.ts +71 -0
  149. package/types/lib/stages/postgen/introspection/emit.d.ts.map +1 -0
  150. package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts +154 -0
  151. package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts.map +1 -0
  152. package/types/lib/stages/postgen/introspection/markers.d.ts +57 -0
  153. package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -0
  154. package/types/lib/stages/postgen/introspection/reflect.d.ts +77 -0
  155. package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -0
  156. package/types/lib/stages/postgen/introspection/report.d.ts +85 -0
  157. package/types/lib/stages/postgen/introspection/report.d.ts.map +1 -0
  158. package/types/lib/stages/postgen/introspection/score.d.ts +149 -0
  159. package/types/lib/stages/postgen/introspection/score.d.ts.map +1 -0
  160. package/types/lib/stages/postgen/introspection/shapeCommand.d.ts +79 -0
  161. package/types/lib/stages/postgen/introspection/shapeCommand.d.ts.map +1 -0
  162. package/types/lib/stages/postgen/parentEdge.d.ts +35 -0
  163. package/types/lib/stages/postgen/parentEdge.d.ts.map +1 -0
  164. package/types/lib/stages/postgen/postgen.d.ts +2 -2
  165. package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
  166. package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
  167. package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
  168. package/types/lib/stages/pregen/pregen.d.ts +131 -0
  169. package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
  170. package/types/lib/validator/bomValidator.d.ts.map +1 -1
@@ -0,0 +1,326 @@
1
+ // PyPI Trove classifier interpretation and package -> module-name resolution.
2
+ //
3
+ // PyPI responses carry two structured signals that were previously read only for
4
+ // their license lines and then discarded:
5
+ //
6
+ // * `Framework :: X` — the author's declaration that the distribution belongs to
7
+ // framework X. It is more reliable than any name-prefix heuristic: it catches
8
+ // `flask-talisman`, `dj-database-url` and `starlette-context` without anyone
9
+ // maintaining a list, and it never fires on a package that merely has a
10
+ // framework's name embedded in its own.
11
+ // * `Topic :: ...` — a curated taxonomy of what the distribution does, which maps
12
+ // cleanly onto the description-tag vocabulary in `data/component-tags.json`.
13
+ //
14
+ // Tags derived here flow into `component.tags` and, for the `framework` tag, into
15
+ // `determinePackageType()` in `lib/cli/bomAssembly.js`, which already promotes any
16
+ // package carrying a `framework` tag to `type: "framework"`.
17
+
18
+ import { PYPI_MODULE_PACKAGE_MAPPING } from "../core/state.js";
19
+
20
+ /**
21
+ * `Framework ::` values that do not mean "an application is built on this".
22
+ *
23
+ * Two groups, both of which would otherwise put a `framework` tag — and therefore
24
+ * `type: "framework"` — on a large share of an ordinary dependency tree, drowning
25
+ * the runtime frameworks the tag exists to surface:
26
+ *
27
+ * * Development, build, test and documentation tooling. A pytest plugin
28
+ * declaring `Framework :: Pytest` is making a true statement, but it belongs
29
+ * to the dev dependencies, not to what the application runs on.
30
+ * * Concurrency runtimes. `Framework :: AsyncIO` is declared by a very large
31
+ * number of libraries to advertise that they work under asyncio; observed on
32
+ * `fastapi`, and `Framework :: AnyIO` likewise on `starlette`. It describes an
33
+ * execution model, not a framework the application is written against.
34
+ *
35
+ * The declared name is still emitted as a tag in every case, and still recorded in
36
+ * `cdx:pypi:frameworks`; only the generic `framework` tag is withheld.
37
+ */
38
+ export const PYPI_NON_APPLICATION_FRAMEWORKS = new Set([
39
+ "anyio",
40
+ "asyncio",
41
+ "buildout",
42
+ "cookiecutter",
43
+ "flake8",
44
+ "hatch",
45
+ "hypothesis",
46
+ "mkdocs",
47
+ "pytest",
48
+ "robotframework",
49
+ "setuptools",
50
+ "sphinx",
51
+ "tox",
52
+ "trio",
53
+ "wheel",
54
+ ]);
55
+
56
+ /**
57
+ * `Topic ::` prefix -> tags, longest prefix wins.
58
+ *
59
+ * Every value on the right-hand side is an existing entry in
60
+ * `data/component-tags.json` so that a tag derived from a classifier and the same
61
+ * tag derived from a description are indistinguishable to consumers.
62
+ */
63
+ const TOPIC_TAGS = [
64
+ [
65
+ "Topic :: Software Development :: Libraries :: Application Frameworks",
66
+ ["framework"],
67
+ ],
68
+ ["Topic :: Software Development :: Object Brokering", ["rpc"]],
69
+ ["Topic :: Software Development :: Testing", ["test"]],
70
+ ["Topic :: Internet :: WWW/HTTP :: WSGI", ["web", "http", "wsgi"]],
71
+ ["Topic :: Internet :: WWW/HTTP :: Session", ["web", "http", "auth"]],
72
+ ["Topic :: Internet :: WWW/HTTP :: Dynamic Content", ["web", "http"]],
73
+ ["Topic :: Internet :: WWW/HTTP", ["web", "http"]],
74
+ ["Topic :: Internet :: XML-RPC", ["rpc", "xml"]],
75
+ ["Topic :: Internet :: Proxy Servers", ["web", "traffic"]],
76
+ ["Topic :: Database :: Front-Ends", ["database"]],
77
+ ["Topic :: Database", ["database"]],
78
+ ["Topic :: Security :: Cryptography", ["crypto", "security"]],
79
+ ["Topic :: Security", ["security"]],
80
+ ["Topic :: System :: Logging", ["logging"]],
81
+ ["Topic :: System :: Distributed Computing", ["rpc"]],
82
+ ["Topic :: Communications :: Email", ["mail"]],
83
+ ["Topic :: Text Processing :: Markup :: HTML", ["html", "template"]],
84
+ ["Topic :: Text Processing :: Markup :: XML", ["xml"]],
85
+ ["Topic :: Scientific/Engineering :: Artificial Intelligence", ["ai", "ml"]],
86
+ [
87
+ "Topic :: Scientific/Engineering :: Image Recognition",
88
+ ["ml", "object-detect"],
89
+ ],
90
+ ["Topic :: Office/Business :: Financial", ["finance"]],
91
+ ["Topic :: Terminals", ["cli"]],
92
+ ["Topic :: Utilities", ["tools"]],
93
+ ];
94
+
95
+ /**
96
+ * Normalize a distribution name per PEP 503.
97
+ *
98
+ * @param {string} name Distribution name
99
+ * @returns {string} Normalized name, or an empty string for unusable input
100
+ */
101
+ export function normalizePypiName(name) {
102
+ if (!name || typeof name !== "string") {
103
+ return "";
104
+ }
105
+ let out = "";
106
+ let lastWasSeparator = false;
107
+ for (const ch of name.trim().toLowerCase()) {
108
+ if (ch === "-" || ch === "_" || ch === ".") {
109
+ if (!lastWasSeparator && out.length) {
110
+ out += "-";
111
+ }
112
+ lastWasSeparator = true;
113
+ continue;
114
+ }
115
+ out += ch;
116
+ lastWasSeparator = false;
117
+ }
118
+ return out.endsWith("-") ? out.slice(0, -1) : out;
119
+ }
120
+
121
+ /**
122
+ * `data/pypi-pkg-aliases.json` maps module name -> distribution name. Inverting it
123
+ * gives the mapping consumers actually need: distribution -> the module names it
124
+ * installs. Built once on first use.
125
+ *
126
+ * @type {Map<string, string[]> | undefined}
127
+ */
128
+ let invertedAliases;
129
+
130
+ function packageToModules() {
131
+ if (invertedAliases) {
132
+ return invertedAliases;
133
+ }
134
+ invertedAliases = new Map();
135
+ for (const [moduleName, pkgName] of Object.entries(
136
+ PYPI_MODULE_PACKAGE_MAPPING,
137
+ )) {
138
+ const key = normalizePypiName(pkgName);
139
+ if (!key || !moduleName) {
140
+ continue;
141
+ }
142
+ const existing = invertedAliases.get(key);
143
+ if (existing) {
144
+ if (!existing.includes(moduleName)) {
145
+ existing.push(moduleName);
146
+ }
147
+ } else {
148
+ invertedAliases.set(key, [moduleName]);
149
+ }
150
+ }
151
+ return invertedAliases;
152
+ }
153
+
154
+ /**
155
+ * Best-known import module names for a PyPI distribution.
156
+ *
157
+ * Two sources, in order: the curated alias map (which is where the cases that
158
+ * cannot be derived from the name live — `pyyaml` installs `yaml`, `absl-py`
159
+ * installs `absl`, `beautifulsoup4` installs `bs4`), and the name-derived form,
160
+ * which is correct for the large majority of distributions. Callers that need to
161
+ * distinguish the two should read the alias map themselves; for tagging purposes
162
+ * both are usable and a wrong extra candidate simply matches nothing.
163
+ *
164
+ * This is deliberately not a guess at *every* module a distribution ships — only
165
+ * at its top-level import names.
166
+ *
167
+ * @param {string} name Distribution name, e.g. "PyYAML"
168
+ * @returns {string[]} Candidate top-level module names, deduplicated
169
+ */
170
+ export function pypiModulesForPackage(name) {
171
+ const normalized = normalizePypiName(name);
172
+ if (!normalized) {
173
+ return [];
174
+ }
175
+ const modules = [...(packageToModules().get(normalized) || [])];
176
+ // The name-derived form. PEP 503 collapses `.` and `_` to `-`; imports use `_`.
177
+ const derived = normalized.replaceAll("-", "_");
178
+ if (derived && !modules.includes(derived)) {
179
+ modules.push(derived);
180
+ }
181
+ return modules;
182
+ }
183
+
184
+ /**
185
+ * Interpret a PyPI classifier list.
186
+ *
187
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
188
+ * @returns {{frameworks: string[], tags: string[]}} Declared frameworks (lowercased,
189
+ * e.g. `["django"]`) and the tag vocabulary they and the `Topic ::` lines imply
190
+ */
191
+ export function parsePypiClassifiers(classifiers) {
192
+ const frameworks = [];
193
+ const tags = new Set();
194
+ if (!Array.isArray(classifiers)) {
195
+ return { frameworks, tags: [] };
196
+ }
197
+ for (const classifier of classifiers) {
198
+ if (!classifier || typeof classifier !== "string") {
199
+ continue;
200
+ }
201
+ if (classifier.startsWith("Framework :: ")) {
202
+ // "Framework :: Django :: 5.0" -> "django". The version segment is dropped:
203
+ // it is the framework version the package supports, not a distinct signal.
204
+ const declared = classifier.split(" :: ")[1]?.trim();
205
+ if (!declared) {
206
+ continue;
207
+ }
208
+ const name = declared.toLowerCase().replaceAll(" ", "-");
209
+ if (!frameworks.includes(name)) {
210
+ frameworks.push(name);
211
+ }
212
+ tags.add(name);
213
+ if (!PYPI_NON_APPLICATION_FRAMEWORKS.has(name)) {
214
+ tags.add("framework");
215
+ }
216
+ continue;
217
+ }
218
+ if (!classifier.startsWith("Topic :: ")) {
219
+ continue;
220
+ }
221
+ for (const [prefix, topicTags] of TOPIC_TAGS) {
222
+ if (classifier === prefix || classifier.startsWith(`${prefix} ::`)) {
223
+ for (const tag of topicTags) {
224
+ tags.add(tag);
225
+ }
226
+ break;
227
+ }
228
+ }
229
+ }
230
+ return { frameworks, tags: Array.from(tags) };
231
+ }
232
+
233
+ function addProperty(pkg, name, value) {
234
+ if (!value) {
235
+ return;
236
+ }
237
+ if (!pkg.properties) {
238
+ pkg.properties = [];
239
+ }
240
+ const existing = pkg.properties.find((prop) => prop.name === name);
241
+ if (existing) {
242
+ existing.value = value;
243
+ return;
244
+ }
245
+ pkg.properties.push({ name, value });
246
+ }
247
+
248
+ /**
249
+ * Attach classifier-derived tags and properties to a package.
250
+ *
251
+ * Tags are merged into any the package already carries rather than replacing
252
+ * them — `lib/managers/binary.js` sets `tags: ["source"]` as an identity marker,
253
+ * and losing it would change how that package is reconciled later.
254
+ *
255
+ * @param {Object} pkg Package object, mutated in place
256
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
257
+ * @returns {Object} The same package object
258
+ */
259
+ export function applyPypiClassifierMetadata(pkg, classifiers) {
260
+ if (!pkg || !Array.isArray(classifiers) || !classifiers.length) {
261
+ return pkg;
262
+ }
263
+ const { frameworks, tags } = parsePypiClassifiers(classifiers);
264
+ if (tags.length) {
265
+ const merged = new Set(Array.isArray(pkg.tags) ? pkg.tags : []);
266
+ for (const tag of tags) {
267
+ merged.add(tag);
268
+ }
269
+ pkg.tags = Array.from(merged);
270
+ }
271
+ if (frameworks.length) {
272
+ addProperty(pkg, "cdx:pypi:frameworks", frameworks.join(","));
273
+ }
274
+ // The full list is retained so that consumers wanting the whole taxonomy are
275
+ // not limited to the tag vocabulary mapped above. Classifiers come from a
276
+ // closed PyPI-maintained set, so there is no free-text or secret exposure.
277
+ addProperty(
278
+ pkg,
279
+ "cdx:pypi:classifiers",
280
+ classifiers.filter((c) => typeof c === "string" && c.length).join("\n"),
281
+ );
282
+ return pkg;
283
+ }
284
+
285
+ /**
286
+ * Attach the distribution's top-level import module names as `internal:Namespaces`.
287
+ *
288
+ * This is the same property jar, gem, nuget and composer components already carry,
289
+ * so downstream consumers that map namespaces back to components need no
290
+ * pypi-specific path. Unlike the jar case the value is name-derived and
291
+ * alias-curated rather than read out of the artifact, which is why
292
+ * `cdx:pypi:modulesFrom` records the provenance.
293
+ *
294
+ * @param {Object} pkg Package object, mutated in place
295
+ * @returns {Object} The same package object
296
+ */
297
+ export function applyPypiModuleNames(pkg) {
298
+ if (!pkg?.name) {
299
+ return pkg;
300
+ }
301
+ // Extras (`requests[security]`) are not part of the distribution name.
302
+ const name = pkg.name.includes("[") ? pkg.name.split("[")[0] : pkg.name;
303
+ if (name.includes("https")) {
304
+ return pkg;
305
+ }
306
+ const normalized = normalizePypiName(name);
307
+ const modules = pypiModulesForPackage(name);
308
+ if (!modules.length) {
309
+ return pkg;
310
+ }
311
+ // Most alias-map entries map a distribution to the module its own name already
312
+ // implies, so the map having an entry says nothing. The provenance worth
313
+ // recording is whether it contributed a module the name could not have given -
314
+ // that is the difference between `flask` and `pyyaml` -> `yaml`.
315
+ const derived = normalized.replaceAll("-", "_");
316
+ const curatedAddedSomething = (packageToModules().get(normalized) || []).some(
317
+ (moduleName) => moduleName !== derived,
318
+ );
319
+ addProperty(pkg, "internal:Namespaces", modules.join("\n"));
320
+ addProperty(
321
+ pkg,
322
+ "cdx:pypi:modulesFrom",
323
+ curatedAddedSomething ? "alias-map,distribution-name" : "distribution-name",
324
+ );
325
+ return pkg;
326
+ }
@@ -331,6 +331,53 @@ export function get_python_command_from_env(env) {
331
331
  return fallbackCmd;
332
332
  }
333
333
 
334
+ /**
335
+ * Is this import satisfied by a module that lives in the project itself?
336
+ *
337
+ * `atom parsedeps` reports every non-stdlib import it sees, including imports of the
338
+ * project's own packages. Attributing those to a distribution invents dependencies:
339
+ * `data/pypi-pkg-aliases.json` maps the module `app` to `zope2`, `test` to `pytabix`,
340
+ * `models` to `asposestorage` and `src` to `auto-mix-prep`, so any project with a
341
+ * local package of that name acquires an obscure component it never depended on -
342
+ * along with every namespace and tag that component carries downstream. Filtering by
343
+ * name would need a denylist of every plausible first-party name; asking the
344
+ * filesystem whether the module is ours answers it exactly.
345
+ *
346
+ * Checks the project root and a `src/` layout for a module file, a package directory,
347
+ * and a PEP 420 namespace package directory.
348
+ *
349
+ * @param {string} src Path to the project being analysed
350
+ * @param {string} name Imported top-level module name
351
+ * @returns {boolean} `true` if the project supplies this module itself
352
+ */
353
+ export function isFirstPartyModule(src, name) {
354
+ if (!src || !name || name.includes("/") || name.includes("\\")) {
355
+ return false;
356
+ }
357
+ for (const root of [src, join(src, "src")]) {
358
+ if (
359
+ safeExistsSync(join(root, `${name}.py`)) ||
360
+ safeExistsSync(join(root, name, "__init__.py")) ||
361
+ safeExistsSync(join(root, name, "__main__.py"))
362
+ ) {
363
+ return true;
364
+ }
365
+ // PEP 420 namespace package: a directory of the right name holding python
366
+ // sources but no __init__.py.
367
+ const candidate = join(root, name);
368
+ if (safeExistsSync(candidate)) {
369
+ try {
370
+ if (readdirSync(candidate).some((f) => f.endsWith(".py"))) {
371
+ return true;
372
+ }
373
+ } catch {
374
+ // not a readable directory - treat as not first-party
375
+ }
376
+ }
377
+ }
378
+ return false;
379
+ }
380
+
334
381
  /**
335
382
  * Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
336
383
  *
@@ -362,7 +409,8 @@ export async function getPyModules(src, epkgList, options) {
362
409
  (x) =>
363
410
  !pyDefaultModules.has(x.name.toLowerCase()) &&
364
411
  !x.name.startsWith("_") &&
365
- !x.name.startsWith("."),
412
+ !x.name.startsWith(".") &&
413
+ !isFirstPartyModule(src, x.name),
366
414
  );
367
415
  let pkgList = modList.map((p) => {
368
416
  const apkg = {
@@ -19,6 +19,7 @@ import {
19
19
  prefetchEnabled,
20
20
  prefetchedResponse,
21
21
  prefetchJson,
22
+ recordPolicyDegradationFromError,
22
23
  } from "../inventory/fetchBatch.js";
23
24
 
24
25
  // FIXME. This has to get removed, once we improve the module detection one-liner.
@@ -780,6 +781,7 @@ export async function getRubyGemsMetadata(pkgList) {
780
781
  continue;
781
782
  }
782
783
  } catch (err) {
784
+ recordPolicyDegradationFromError(err);
783
785
  if (DEBUG_MODE) {
784
786
  console.error(name, err);
785
787
  }
@@ -832,6 +834,7 @@ async function enrichGemFromVersionEndpoint(
832
834
  }
833
835
  applyGemMetadata(p, body);
834
836
  } catch (err) {
837
+ recordPolicyDegradationFromError(err);
835
838
  if (DEBUG_MODE) {
836
839
  console.error(p, err);
837
840
  }
@@ -1241,6 +1244,16 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
1241
1244
  pkg.externalReferences.push({ type: refType, url: value, comment: key });
1242
1245
  }
1243
1246
  }
1247
+ // `s.homepage = "..."` is the project website. Keep it as an external
1248
+ // reference; the transient `homepage` key is a plain string that never
1249
+ // reaches the BOM because component conversion expects the `{ url }` shape.
1250
+ if (pkg.homepage) {
1251
+ pkg.externalReferences = pkg.externalReferences || [];
1252
+ if (!pkg.externalReferences.some((ref) => ref.url === pkg.homepage)) {
1253
+ pkg.externalReferences.push({ type: "website", url: pkg.homepage });
1254
+ }
1255
+ delete pkg.homepage;
1256
+ }
1244
1257
  if (pkg.name) {
1245
1258
  const purlString = toGemPurl(pkg.name, pkg.version, gemPlatform);
1246
1259
  pkg.purl = purlString;
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Deprecated re-export barrel.
3
3
  *
4
- * `utils.js` declares nothing of its own. It re-exports the 261 public names that
4
+ * `utils.js` declares nothing of its own. It re-exports every public name that
5
5
  * used to live here so existing imports keep working for one major version. Each
6
6
  * implementation, and its JSDoc, now lives in the leaf module it was moved to
7
7
  * (`purl.js`, `spdx.js`, `deps.js`, `ecosystems.js`, `parsers-*.js`, `core-*.js`,
@@ -191,11 +191,13 @@ import {
191
191
  import { getMavenCommand, getMillCommand } from "./gradleutils.js";
192
192
  import { createNpmWorkspacePurl } from "./npmutils.js";
193
193
  import {
194
+ getCentralPackageVersions,
194
195
  getPropertyGroupTextNodes,
195
196
  parseCsPkgData,
196
197
  parseCsPkgLockData,
197
198
  parseCsProjAssetsData,
198
199
  parseCsProjData,
200
+ parseDirectoryPackagesProps,
199
201
  parseNupkg,
200
202
  parseNuspecData,
201
203
  parsePaketLockData,
@@ -220,12 +222,14 @@ import {
220
222
  parsePackageJsonName,
221
223
  parsePkgJson,
222
224
  parsePkgLock,
225
+ parsePnpmAliasRef,
223
226
  parsePnpmLock,
224
227
  parsePnpmWorkspace,
225
228
  parseYarnLock,
226
229
  parseYarnWorkspace,
227
230
  pnpmMetadata,
228
231
  splitCommandArgs,
232
+ stripPnpmPeerSuffix,
229
233
  yarnLockToIdentMap,
230
234
  } from "./parsers-js.js";
231
235
  import {
@@ -369,6 +373,7 @@ export {
369
373
  getAllFiles,
370
374
  getAllFilesWithIgnore,
371
375
  getAtomCommand,
376
+ getCentralPackageVersions,
372
377
  getCratesMetadata,
373
378
  getDartMetadata,
374
379
  getDefaultBomAuditCategories,
@@ -465,6 +470,7 @@ export {
465
470
  parseCsProjAssetsData,
466
471
  parseCsProjData,
467
472
  parseCUsageSlice,
473
+ parseDirectoryPackagesProps,
468
474
  parseEdnData,
469
475
  parseFlakeLock,
470
476
  parseFlakeNix,
@@ -503,6 +509,7 @@ export {
503
509
  parsePixiTomlFile,
504
510
  parsePkgJson,
505
511
  parsePkgLock,
512
+ parsePnpmAliasRef,
506
513
  parsePnpmLock,
507
514
  parsePnpmWorkspace,
508
515
  parsePodfileLock,
@@ -565,6 +572,7 @@ export {
565
572
  shouldFetchVCS,
566
573
  shouldRunPredictiveBomAudit,
567
574
  splitCommandArgs,
575
+ stripPnpmPeerSuffix,
568
576
  TABLE_BORDER_STYLE,
569
577
  TIMEOUT_MS,
570
578
  toCamel,
@@ -42,12 +42,15 @@ import {
42
42
  import { collectMvnDependencies } from "../inventory/deps.js";
43
43
  import { mergeServices } from "../inventory/depsUtils.js";
44
44
  import {
45
+ collectDosaiAiComponents,
45
46
  collectDosaiDataFlowFrames,
46
47
  collectDosaiPurlEvidence,
48
+ collectDosaiServiceComponents,
47
49
  collectDosaiServicesFromMethods,
48
50
  createDosaiDataFlowSlice,
49
51
  createDosaiMethodsSlice,
50
52
  isDosaiDotnetLanguage,
53
+ normalizeDosaiServiceMap,
51
54
  persistDosaiSemanticsReport,
52
55
  } from "../inventory/dosai.js";
53
56
  import { parseOccurrenceEvidenceLocation } from "../inventory/evidenceUtils.js";
@@ -662,6 +665,7 @@ export async function analyzeProject(dbObjMap, options) {
662
665
  const bomJson = JSON.parse(fs.readFileSync(bomFile, "utf8"));
663
666
  const components = bomJson.components || [];
664
667
  let cryptoComponents = [];
668
+ let aiComponents = [];
665
669
  let cryptoGeneratePurls = {};
666
670
  let openapiSpecFile;
667
671
  // Load any existing purl-location information from the sbom.
@@ -750,6 +754,13 @@ export async function analyzeProject(dbObjMap, options) {
750
754
  purlLocationMap[purl].add(location);
751
755
  }
752
756
  }
757
+ // Rich Services[] inventory first (bom-refs, trust zones, data); endpoint
758
+ // derivation fills any services the providers did not emit.
759
+ // AI components must be carried out of analyzeProject explicitly. Mutating the local
760
+ // `components` array here had no effect: the return below omits it and createEvinseFile
761
+ // re-reads the BOM from disk, so the whole AI-BOM contribution was silently discarded.
762
+ collectDosaiAiComponents(usageSlice, aiComponents);
763
+ servicesMap = collectDosaiServiceComponents(usageSlice, servicesMap);
753
764
  servicesMap = collectDosaiServicesFromMethods(usageSlice, servicesMap);
754
765
  userDefinedTypesMap = {};
755
766
  }
@@ -802,6 +813,7 @@ export async function analyzeProject(dbObjMap, options) {
802
813
  tempDirOwned: retMap?.tempDirOwned,
803
814
  userDefinedTypesMap,
804
815
  cryptoComponents,
816
+ aiComponents,
805
817
  cryptoGeneratePurls,
806
818
  };
807
819
  }
@@ -832,6 +844,7 @@ export async function analyzeProject(dbObjMap, options) {
832
844
  const retMap = collectReachableFrames(language, reachablesSlice);
833
845
  dataFlowFrames = retMap.dataFlowFrames;
834
846
  cryptoComponents = retMap.cryptoComponents;
847
+ aiComponents = retMap.aiComponents || [];
835
848
  cryptoGeneratePurls = retMap.cryptoGeneratePurls;
836
849
  }
837
850
  // Reuse existing usages slices
@@ -960,6 +973,7 @@ export async function analyzeProject(dbObjMap, options) {
960
973
  tempDirOwned: retMap?.tempDirOwned,
961
974
  userDefinedTypesMap,
962
975
  cryptoComponents,
976
+ aiComponents,
963
977
  cryptoGeneratePurls,
964
978
  openapiSpecFile,
965
979
  };
@@ -1804,7 +1818,7 @@ export function extractEndpoints(language, code) {
1804
1818
  * @param {Object} options Command line options
1805
1819
  * @returns
1806
1820
  */
1807
- export function createEvinseFile(sliceArtefacts, options) {
1821
+ export async function createEvinseFile(sliceArtefacts, options) {
1808
1822
  const {
1809
1823
  tempDir,
1810
1824
  tempDirOwned,
@@ -1817,6 +1831,7 @@ export function createEvinseFile(sliceArtefacts, options) {
1817
1831
  componentPropertiesMap = {},
1818
1832
  metadataProperties = [],
1819
1833
  cryptoComponents,
1834
+ aiComponents,
1820
1835
  cryptoGeneratePurls,
1821
1836
  } = sliceArtefacts;
1822
1837
  const bomFile = options.input;
@@ -1873,16 +1888,12 @@ export function createEvinseFile(sliceArtefacts, options) {
1873
1888
  }
1874
1889
  } // for
1875
1890
  if (servicesMap && Object.keys(servicesMap).length) {
1876
- const services = [];
1877
- for (const serviceName of Object.keys(servicesMap)) {
1878
- services.push({
1879
- name: serviceName,
1880
- endpoints: Array.from(servicesMap[serviceName].endpoints),
1881
- authenticated: servicesMap[serviceName].authenticated,
1882
- "x-trust-boundary": servicesMap[serviceName].xTrustBoundary,
1883
- properties: servicesMap[serviceName].properties,
1884
- });
1885
- }
1891
+ // normalizeDosaiServiceMap is the single mapping from the internal service
1892
+ // definitions to CycloneDX services[]. The OpenAPI/Java/Python detectors set
1893
+ // the camelCase `xTrustBoundary` while dosai's provider inventory (schema
1894
+ // 4.0.0) carries stable bom-refs, trust zones, data classifications, and
1895
+ // evidence occurrences; both shapes flow through the same normalizer.
1896
+ const services = normalizeDosaiServiceMap(servicesMap);
1886
1897
  // Add to existing services while preserving CycloneDX uniqueItems validity
1887
1898
  bomJson.services = mergeServices(bomJson.services || [], services);
1888
1899
  servicesPresent = true;
@@ -1891,6 +1902,10 @@ export function createEvinseFile(sliceArtefacts, options) {
1891
1902
  if (cryptoComponents?.length) {
1892
1903
  bomJson.components = bomJson.components.concat(cryptoComponents);
1893
1904
  }
1905
+ // Add the AI components (machine-learning-model, dataset) discovered by dosai
1906
+ if (aiComponents?.length) {
1907
+ bomJson.components = bomJson.components.concat(aiComponents);
1908
+ }
1894
1909
  // Fix the dependencies section with provides information
1895
1910
  if (
1896
1911
  cryptoGeneratePurls &&
@@ -1943,7 +1958,10 @@ export function createEvinseFile(sliceArtefacts, options) {
1943
1958
  // Redo post-processing with evinse data. `executeOsQuery` has to be injected
1944
1959
  // because postgen no longer imports it directly (stages is below managers);
1945
1960
  // without it the crypto formulation path receives `undefined` and throws.
1946
- const bomNSData = postProcess({ bomJson }, { ...options, executeOsQuery });
1961
+ const bomNSData = await postProcess(
1962
+ { bomJson },
1963
+ { ...options, executeOsQuery },
1964
+ );
1947
1965
  safeWriteSync(
1948
1966
  evinseOutFile,
1949
1967
  JSON.stringify(bomNSData.bomJson, null, options.jsonPretty ? 2 : null),