@cdxgen/cdxgen 13.3.0 → 13.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/README.md +9 -6
  2. package/bin/cdxgen.js +90 -26
  3. package/bin/evinse.js +18 -1
  4. package/bin/repl.js +86 -0
  5. package/bin/sign.js +45 -15
  6. package/bin/tracebom.js +5 -1
  7. package/bin/validate.js +84 -18
  8. package/bin/verify.js +155 -86
  9. package/data/helpers/init.gradle +46 -21
  10. package/data/remediations.json +61 -0
  11. package/data/rules/dependency-sources.yaml +67 -0
  12. package/data/rules/scala-evidence.yaml +115 -0
  13. package/lib/audit/index.js +38 -6
  14. package/lib/cli/index.js +113 -54
  15. package/lib/cli/jsBom.js +50 -29
  16. package/lib/cli/jvmBom.js +665 -138
  17. package/lib/cli/managedBom.js +320 -200
  18. package/lib/cli/nativeBom.js +139 -30
  19. package/lib/core/activity.js +12 -0
  20. package/lib/core/env.js +17 -9
  21. package/lib/core/fs.js +3 -2
  22. package/lib/core/httpClient.js +159 -24
  23. package/lib/core/pluginRun.js +184 -0
  24. package/lib/core/processTree.js +100 -0
  25. package/lib/core/runState.js +39 -0
  26. package/lib/core/toml.js +72 -0
  27. package/lib/ecosystems/asarutils.js +4 -2
  28. package/lib/ecosystems/bunutils.js +5 -7
  29. package/lib/ecosystems/cargoCache.js +231 -8
  30. package/lib/ecosystems/cmakeResolver.js +10 -21
  31. package/lib/ecosystems/cppBuildContext.js +631 -0
  32. package/lib/ecosystems/cppEvidence.js +337 -22
  33. package/lib/ecosystems/cppIncludeResolver.js +294 -0
  34. package/lib/ecosystems/denoutils.js +93 -66
  35. package/lib/ecosystems/ecosystems.js +1967 -788
  36. package/lib/ecosystems/golem.js +28 -12
  37. package/lib/ecosystems/gradleutils.js +314 -30
  38. package/lib/ecosystems/jsEvidence.js +85 -24
  39. package/lib/ecosystems/kosi.js +21 -4
  40. package/lib/ecosystems/npmutils.js +64 -0
  41. package/lib/ecosystems/parsers-dotnet.js +567 -151
  42. package/lib/ecosystems/parsers-elm.js +46 -28
  43. package/lib/ecosystems/parsers-gleam.js +3 -4
  44. package/lib/ecosystems/parsers-go.js +324 -61
  45. package/lib/ecosystems/parsers-js.js +92 -26
  46. package/lib/ecosystems/parsers-julia.js +2 -3
  47. package/lib/ecosystems/parsers-jvm.js +204 -12
  48. package/lib/ecosystems/parsers-misc.js +182 -55
  49. package/lib/ecosystems/parsers-python.js +141 -10
  50. package/lib/ecosystems/parsers-rust.js +4 -4
  51. package/lib/ecosystems/parsers-terraform.js +1573 -19
  52. package/lib/ecosystems/pythonutils.js +102 -6
  53. package/lib/ecosystems/remote/huggingface.js +3 -0
  54. package/lib/ecosystems/rubyutils.js +67 -17
  55. package/lib/ecosystems/rusi.js +22 -9
  56. package/lib/ecosystems/sbtutils.js +243 -187
  57. package/lib/ecosystems/scalacliutils.js +342 -0
  58. package/lib/ecosystems/scalasem.js +2618 -0
  59. package/lib/ecosystems/terraformRegistry.js +288 -0
  60. package/lib/ecosystems/utils.js +2 -0
  61. package/lib/ecosystems/vendoredCode.js +156 -0
  62. package/lib/evinser/evinser.js +578 -69
  63. package/lib/helpers/agenticToolutils.js +2 -2
  64. package/lib/helpers/bomSigner.js +576 -128
  65. package/lib/inventory/aiCollector.js +4 -1
  66. package/lib/inventory/analyzer.js +110 -127
  67. package/lib/inventory/atomUtils.js +171 -48
  68. package/lib/inventory/auditCategories.js +7 -0
  69. package/lib/inventory/buildToolRateLimit.js +144 -0
  70. package/lib/inventory/cacheCommands.js +2 -1
  71. package/lib/inventory/cmakeBuildDirs.js +254 -0
  72. package/lib/inventory/deps.js +501 -124
  73. package/lib/inventory/dosai.js +248 -44
  74. package/lib/inventory/dosaiParsers.js +310 -15
  75. package/lib/inventory/dynamic.js +52 -6
  76. package/lib/inventory/evidenceUtils.js +154 -109
  77. package/lib/inventory/fetchBatch.js +574 -38
  78. package/lib/inventory/fetchDiskCache.js +597 -0
  79. package/lib/inventory/forgeEnricher.js +14 -5
  80. package/lib/inventory/jvmLocalRepos.js +917 -0
  81. package/lib/inventory/mcpConfigParser.js +2 -3
  82. package/lib/inventory/parentSelection.js +146 -0
  83. package/lib/inventory/rustFormulationParser.js +3 -4
  84. package/lib/inventory/scalaCoords.js +153 -0
  85. package/lib/inventory/source.js +8 -8
  86. package/lib/inventory/spdx.js +101 -0
  87. package/lib/inventory/traceRunner.js +88 -7
  88. package/lib/inventory/tracedCrypto.js +294 -0
  89. package/lib/managers/docker.js +33 -32
  90. package/lib/managers/oci.js +24 -2
  91. package/lib/parsers/cmakeCache.js +69 -0
  92. package/lib/parsers/cmakeLists.js +16 -2
  93. package/lib/parsers/cmakePresets.js +400 -0
  94. package/lib/parsers/compileCommands.js +557 -0
  95. package/lib/parsers/strictJson.js +322 -0
  96. package/lib/parsers/terraformConfig.js +1073 -0
  97. package/lib/parsers/terraformSource.js +652 -0
  98. package/lib/server/server.js +23 -0
  99. package/lib/stages/postgen/introspection/markers.js +1 -1
  100. package/lib/stages/postgen/postgen.js +15 -1
  101. package/lib/stages/postgen/specVersionCompat.js +32 -0
  102. package/lib/validator/index.js +25 -10
  103. package/lib/validator/reporters/json.js +1 -0
  104. package/package.json +21 -21
  105. package/types/lib/audit/index.d.ts.map +1 -1
  106. package/types/lib/cli/index.d.ts.map +1 -1
  107. package/types/lib/cli/jsBom.d.ts.map +1 -1
  108. package/types/lib/cli/jvmBom.d.ts +24 -1
  109. package/types/lib/cli/jvmBom.d.ts.map +1 -1
  110. package/types/lib/cli/managedBom.d.ts +7 -4
  111. package/types/lib/cli/managedBom.d.ts.map +1 -1
  112. package/types/lib/cli/nativeBom.d.ts +3 -1
  113. package/types/lib/cli/nativeBom.d.ts.map +1 -1
  114. package/types/lib/core/activity.d.ts.map +1 -1
  115. package/types/lib/core/env.d.ts +9 -0
  116. package/types/lib/core/env.d.ts.map +1 -1
  117. package/types/lib/core/fs.d.ts.map +1 -1
  118. package/types/lib/core/httpClient.d.ts +13 -8
  119. package/types/lib/core/httpClient.d.ts.map +1 -1
  120. package/types/lib/core/pluginRun.d.ts +97 -0
  121. package/types/lib/core/pluginRun.d.ts.map +1 -0
  122. package/types/lib/core/processTree.d.ts +36 -0
  123. package/types/lib/core/processTree.d.ts.map +1 -0
  124. package/types/lib/core/runState.d.ts +29 -0
  125. package/types/lib/core/runState.d.ts.map +1 -0
  126. package/types/lib/core/toml.d.ts +20 -0
  127. package/types/lib/core/toml.d.ts.map +1 -0
  128. package/types/lib/ecosystems/asarutils.d.ts.map +1 -1
  129. package/types/lib/ecosystems/bunutils.d.ts.map +1 -1
  130. package/types/lib/ecosystems/cargoCache.d.ts +12 -0
  131. package/types/lib/ecosystems/cargoCache.d.ts.map +1 -1
  132. package/types/lib/ecosystems/cmakeResolver.d.ts +5 -3
  133. package/types/lib/ecosystems/cmakeResolver.d.ts.map +1 -1
  134. package/types/lib/ecosystems/cppBuildContext.d.ts +55 -0
  135. package/types/lib/ecosystems/cppBuildContext.d.ts.map +1 -0
  136. package/types/lib/ecosystems/cppEvidence.d.ts +18 -2
  137. package/types/lib/ecosystems/cppEvidence.d.ts.map +1 -1
  138. package/types/lib/ecosystems/cppIncludeResolver.d.ts +92 -0
  139. package/types/lib/ecosystems/cppIncludeResolver.d.ts.map +1 -0
  140. package/types/lib/ecosystems/denoutils.d.ts +9 -3
  141. package/types/lib/ecosystems/denoutils.d.ts.map +1 -1
  142. package/types/lib/ecosystems/ecosystems.d.ts +62 -13
  143. package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
  144. package/types/lib/ecosystems/golem.d.ts.map +1 -1
  145. package/types/lib/ecosystems/gradleutils.d.ts +50 -0
  146. package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
  147. package/types/lib/ecosystems/jsEvidence.d.ts.map +1 -1
  148. package/types/lib/ecosystems/kosi.d.ts.map +1 -1
  149. package/types/lib/ecosystems/npmutils.d.ts +15 -0
  150. package/types/lib/ecosystems/npmutils.d.ts.map +1 -1
  151. package/types/lib/ecosystems/parsers-dotnet.d.ts +118 -4
  152. package/types/lib/ecosystems/parsers-dotnet.d.ts.map +1 -1
  153. package/types/lib/ecosystems/parsers-elm.d.ts.map +1 -1
  154. package/types/lib/ecosystems/parsers-gleam.d.ts.map +1 -1
  155. package/types/lib/ecosystems/parsers-go.d.ts +44 -2
  156. package/types/lib/ecosystems/parsers-go.d.ts.map +1 -1
  157. package/types/lib/ecosystems/parsers-js.d.ts +6 -1
  158. package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
  159. package/types/lib/ecosystems/parsers-julia.d.ts.map +1 -1
  160. package/types/lib/ecosystems/parsers-jvm.d.ts +46 -1
  161. package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
  162. package/types/lib/ecosystems/parsers-misc.d.ts +3 -1
  163. package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
  164. package/types/lib/ecosystems/parsers-python.d.ts +53 -3
  165. package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
  166. package/types/lib/ecosystems/parsers-terraform.d.ts +38 -21
  167. package/types/lib/ecosystems/parsers-terraform.d.ts.map +1 -1
  168. package/types/lib/ecosystems/pythonutils.d.ts +27 -2
  169. package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
  170. package/types/lib/ecosystems/remote/huggingface.d.ts.map +1 -1
  171. package/types/lib/ecosystems/rubyutils.d.ts +21 -1
  172. package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
  173. package/types/lib/ecosystems/rusi.d.ts.map +1 -1
  174. package/types/lib/ecosystems/sbtutils.d.ts +88 -2
  175. package/types/lib/ecosystems/sbtutils.d.ts.map +1 -1
  176. package/types/lib/ecosystems/scalacliutils.d.ts +56 -0
  177. package/types/lib/ecosystems/scalacliutils.d.ts.map +1 -0
  178. package/types/lib/ecosystems/scalasem.d.ts +311 -0
  179. package/types/lib/ecosystems/scalasem.d.ts.map +1 -0
  180. package/types/lib/ecosystems/terraformRegistry.d.ts +9 -0
  181. package/types/lib/ecosystems/terraformRegistry.d.ts.map +1 -0
  182. package/types/lib/ecosystems/utils.d.ts +2 -2
  183. package/types/lib/ecosystems/utils.d.ts.map +1 -1
  184. package/types/lib/ecosystems/vendoredCode.d.ts +32 -0
  185. package/types/lib/ecosystems/vendoredCode.d.ts.map +1 -0
  186. package/types/lib/evinser/evinser.d.ts +75 -8
  187. package/types/lib/evinser/evinser.d.ts.map +1 -1
  188. package/types/lib/helpers/bomSigner.d.ts +80 -4
  189. package/types/lib/helpers/bomSigner.d.ts.map +1 -1
  190. package/types/lib/inventory/aiCollector.d.ts.map +1 -1
  191. package/types/lib/inventory/analyzer.d.ts.map +1 -1
  192. package/types/lib/inventory/atomUtils.d.ts +82 -10
  193. package/types/lib/inventory/atomUtils.d.ts.map +1 -1
  194. package/types/lib/inventory/auditCategories.d.ts +3 -0
  195. package/types/lib/inventory/auditCategories.d.ts.map +1 -1
  196. package/types/lib/inventory/buildToolRateLimit.d.ts +38 -0
  197. package/types/lib/inventory/buildToolRateLimit.d.ts.map +1 -0
  198. package/types/lib/inventory/cacheCommands.d.ts +2 -1
  199. package/types/lib/inventory/cacheCommands.d.ts.map +1 -1
  200. package/types/lib/inventory/cmakeBuildDirs.d.ts +58 -0
  201. package/types/lib/inventory/cmakeBuildDirs.d.ts.map +1 -0
  202. package/types/lib/inventory/deps.d.ts +59 -1
  203. package/types/lib/inventory/deps.d.ts.map +1 -1
  204. package/types/lib/inventory/dosai.d.ts +57 -8
  205. package/types/lib/inventory/dosai.d.ts.map +1 -1
  206. package/types/lib/inventory/dosaiParsers.d.ts +53 -7
  207. package/types/lib/inventory/dosaiParsers.d.ts.map +1 -1
  208. package/types/lib/inventory/dynamic.d.ts +10 -0
  209. package/types/lib/inventory/dynamic.d.ts.map +1 -1
  210. package/types/lib/inventory/evidenceUtils.d.ts +19 -4
  211. package/types/lib/inventory/evidenceUtils.d.ts.map +1 -1
  212. package/types/lib/inventory/fetchBatch.d.ts +83 -14
  213. package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
  214. package/types/lib/inventory/fetchDiskCache.d.ts +169 -0
  215. package/types/lib/inventory/fetchDiskCache.d.ts.map +1 -0
  216. package/types/lib/inventory/forgeEnricher.d.ts.map +1 -1
  217. package/types/lib/inventory/jvmLocalRepos.d.ts +178 -0
  218. package/types/lib/inventory/jvmLocalRepos.d.ts.map +1 -0
  219. package/types/lib/inventory/mcpConfigParser.d.ts.map +1 -1
  220. package/types/lib/inventory/parentSelection.d.ts +34 -0
  221. package/types/lib/inventory/parentSelection.d.ts.map +1 -0
  222. package/types/lib/inventory/rustFormulationParser.d.ts.map +1 -1
  223. package/types/lib/inventory/scalaCoords.d.ts +80 -0
  224. package/types/lib/inventory/scalaCoords.d.ts.map +1 -0
  225. package/types/lib/inventory/source.d.ts.map +1 -1
  226. package/types/lib/inventory/spdx.d.ts +9 -0
  227. package/types/lib/inventory/spdx.d.ts.map +1 -1
  228. package/types/lib/inventory/traceRunner.d.ts +12 -0
  229. package/types/lib/inventory/traceRunner.d.ts.map +1 -1
  230. package/types/lib/inventory/tracedCrypto.d.ts +42 -0
  231. package/types/lib/inventory/tracedCrypto.d.ts.map +1 -0
  232. package/types/lib/managers/docker.d.ts.map +1 -1
  233. package/types/lib/managers/oci.d.ts +3 -1
  234. package/types/lib/managers/oci.d.ts.map +1 -1
  235. package/types/lib/parsers/cmakeCache.d.ts +16 -0
  236. package/types/lib/parsers/cmakeCache.d.ts.map +1 -1
  237. package/types/lib/parsers/cmakeLists.d.ts +2 -1
  238. package/types/lib/parsers/cmakeLists.d.ts.map +1 -1
  239. package/types/lib/parsers/cmakePresets.d.ts +78 -0
  240. package/types/lib/parsers/cmakePresets.d.ts.map +1 -0
  241. package/types/lib/parsers/compileCommands.d.ts +115 -0
  242. package/types/lib/parsers/compileCommands.d.ts.map +1 -0
  243. package/types/lib/parsers/strictJson.d.ts +46 -0
  244. package/types/lib/parsers/strictJson.d.ts.map +1 -0
  245. package/types/lib/parsers/terraformConfig.d.ts +70 -0
  246. package/types/lib/parsers/terraformConfig.d.ts.map +1 -0
  247. package/types/lib/parsers/terraformSource.d.ts +66 -0
  248. package/types/lib/parsers/terraformSource.d.ts.map +1 -0
  249. package/types/lib/server/server.d.ts.map +1 -1
  250. package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
  251. package/types/lib/stages/postgen/specVersionCompat.d.ts.map +1 -1
  252. package/types/lib/validator/index.d.ts +8 -0
  253. package/types/lib/validator/index.d.ts.map +1 -1
  254. package/types/lib/validator/reporters/json.d.ts.map +1 -1
  255. package/lib/third-party/arborist/snapshots/calc-dep-flags.test.cjs +0 -462
@@ -4,6 +4,7 @@ import {
4
4
  sep as _sep,
5
5
  basename,
6
6
  dirname,
7
+ isAbsolute,
7
8
  join,
8
9
  relative,
9
10
  resolve,
@@ -12,11 +13,7 @@ import {
12
13
  import { build, Purl } from "@cdxgen/cdx-purl";
13
14
  import { parse as _load } from "yaml";
14
15
 
15
- import {
16
- cdxgenAgent,
17
- DEBUG_MODE,
18
- readEnvironmentVariable,
19
- } from "../core/activity.js";
16
+ import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
20
17
  import { recordDegradation } from "../core/buildLedger.js";
21
18
  import { deferFailOnError } from "../core/deferredExit.js";
22
19
  import { shouldFetchLicense } from "../core/env.js";
@@ -28,10 +25,12 @@ import {
28
25
  } from "../core/fs.js";
29
26
  import { thoughtLog } from "../core/logger.js";
30
27
  import { isWin } from "../core/paths.js";
28
+ import { registerRunReset } from "../core/runState.js";
31
29
  import { mesonWrapDB } from "../core/state.js";
32
30
  import { isGitHubUrl } from "../core/urls.js";
33
31
  import { parseWorkflowFile } from "../inventory/ciParsers/githubActions.js";
34
32
  import {
33
+ gatedGet,
35
34
  prefetchEnabled,
36
35
  prefetchedResponse,
37
36
  prefetchJson,
@@ -59,6 +58,22 @@ import { parsePkgJson } from "./parsers-js.js";
59
58
  */
60
59
  const MAX_FLAKE_INPUTS_BLOCK_LENGTH = 16 * 1024;
61
60
 
61
+ /**
62
+ * Whether a package registry can hold a pubspec.lock entry.
63
+ *
64
+ * The entry's `source` names where the package comes from. `sdk` packages ship
65
+ * with the Dart or Flutter SDK, `path` packages live in the working tree, and
66
+ * `git` packages come from a repository. None is published on a registry, and
67
+ * asking about them leaks their names and imports a look-alike's metadata.
68
+ *
69
+ * @param {Object} packageData One pubspec.lock `packages` entry.
70
+ * @returns {boolean}
71
+ */
72
+ function canPubRegistryServe(packageData) {
73
+ const source = packageData?.source;
74
+ return source === undefined || source === "hosted";
75
+ }
76
+
62
77
  /**
63
78
  * Method to parse pubspec.lock files.
64
79
  *
@@ -71,10 +86,11 @@ export async function parsePubLockData(pubLockData, lockFile) {
71
86
  if (!pubLockData) {
72
87
  return [];
73
88
  }
74
- let pkgList = [];
89
+ const pkgList = [];
75
90
  const rootList = [];
76
91
  const data = _load(pubLockData);
77
92
  const packages = data.packages;
93
+ const lookupList = [];
78
94
  for (const [packageName, packageData] of Object.entries(packages)) {
79
95
  const pkg = {
80
96
  name: packageName,
@@ -131,9 +147,15 @@ export async function parsePubLockData(pubLockData, lockFile) {
131
147
  };
132
148
  }
133
149
  pkgList.push(pkg);
150
+ if (canPubRegistryServe(packageData)) {
151
+ lookupList.push(pkg);
152
+ }
134
153
  }
135
154
  if (shouldFetchLicense()) {
136
- pkgList = await getDartMetadata(pkgList);
155
+ // getDartMetadata enriches in place, so the sdk, path and git packages
156
+ // stay in pkgList with the lockfile's answers and are simply not handed
157
+ // to it.
158
+ await getDartMetadata(lookupList);
137
159
  }
138
160
  return { rootList, pkgList };
139
161
  }
@@ -3002,14 +3024,67 @@ export function parseMakeDFile(dfile) {
3002
3024
  return pkgFilesMap;
3003
3025
  }
3004
3026
 
3027
+ /**
3028
+ * Resolve a path a lockfile names against the lockfile's directory, refusing
3029
+ * absolute paths and any that leave the scanned directory. The value comes
3030
+ * from the scanned project, so it must not steer the scan at files outside
3031
+ * it. The bound is the scan root, not the lockfile's directory: a React
3032
+ * Native `ios/Podfile` names its pods as `../node_modules/...`.
3033
+ *
3034
+ * @param {String} projectPath Directory of the Podfile.lock
3035
+ * @param {String} value Path as the lockfile writes it
3036
+ * @param {String} [scanRoot] Directory the scan was asked for; the lockfile's
3037
+ * directory when not given
3038
+ * @returns {String|undefined} The resolved path, or undefined when refused
3039
+ */
3040
+ function resolveInsideScan(projectPath, value, scanRoot = projectPath) {
3041
+ if (typeof value !== "string" || !value.length) {
3042
+ return undefined;
3043
+ }
3044
+ if (
3045
+ isAbsolute(value) ||
3046
+ /^[a-zA-Z]:[\\/]/.test(value) ||
3047
+ value.startsWith("\\\\")
3048
+ ) {
3049
+ return undefined;
3050
+ }
3051
+ const resolved = resolve(projectPath, value);
3052
+ const back = relative(resolve(scanRoot), resolved);
3053
+ if (back.startsWith("..") || isAbsolute(back)) {
3054
+ return undefined;
3055
+ }
3056
+ return resolved;
3057
+ }
3058
+
3059
+ /**
3060
+ * Whether a pod name can be used as a file name. Pod names come from the
3061
+ * lockfile and are joined onto directories.
3062
+ *
3063
+ * @param {String} podName Root pod name
3064
+ * @returns {Boolean}
3065
+ */
3066
+ function isSafePodFileName(podName) {
3067
+ return (
3068
+ typeof podName === "string" &&
3069
+ podName.length > 0 &&
3070
+ !podName.includes("/") &&
3071
+ !podName.includes("\\") &&
3072
+ !podName.includes("\0") &&
3073
+ podName !== "." &&
3074
+ podName !== ".."
3075
+ );
3076
+ }
3077
+
3005
3078
  /**
3006
3079
  * Parse the contents of a 'Podfile.lock'
3007
3080
  *
3008
3081
  * @param {Object} podfileLock The content of the podfile.lock as an Object
3009
3082
  * @param {String} projectPath The path to the project root
3083
+ * @param {String} [scanRoot] Directory the scan was asked for. Local `:path`
3084
+ * and `:podspec` sources outside it are not read. Defaults to projectPath.
3010
3085
  * @returns {Map} Map of all dependencies with their direct dependencies
3011
3086
  */
3012
- export async function parsePodfileLock(podfileLock, projectPath) {
3087
+ export async function parsePodfileLock(podfileLock, projectPath, scanRoot) {
3013
3088
  const dependencies = new Map();
3014
3089
  for (const pod of podfileLock["PODS"]) {
3015
3090
  const dependency = {};
@@ -3031,58 +3106,103 @@ export async function parsePodfileLock(podfileLock, projectPath) {
3031
3106
  dependency.metadata.name.indexOf("/"),
3032
3107
  )
3033
3108
  : dependency.metadata.name;
3034
- if (podfileLock["EXTERNAL SOURCES"]?.[podName]) {
3109
+ if (
3110
+ podfileLock["EXTERNAL SOURCES"]?.[podName] &&
3111
+ isSafePodFileName(podName)
3112
+ ) {
3035
3113
  const externalPod = podfileLock["EXTERNAL SOURCES"][podName];
3036
3114
  if (externalPod[":git"]) {
3037
- let projectRepo = externalPod[":git"];
3038
- if (isGitHubUrl(projectRepo)) {
3039
- projectRepo = projectRepo.replace(
3040
- "github.com",
3041
- "raw.githubusercontent.com",
3042
- );
3043
- }
3044
- if (projectRepo.endsWith(".git")) {
3045
- projectRepo = projectRepo.substring(0, projectRepo.length - 4);
3115
+ // The podspec of a checked out pod, which `pod install` writes
3116
+ // alongside the sources it downloaded. Reading it needs no request,
3117
+ // so it answers before any raw repository URL.
3118
+ const localPodspec = join(
3119
+ projectPath,
3120
+ "Pods",
3121
+ "Local Podspecs",
3122
+ `${podName}.podspec.json`,
3123
+ );
3124
+ if (safeExistsSync(localPodspec)) {
3125
+ dependency.metadata.properties = [
3126
+ {
3127
+ name: "cdx:pods:podspecLocation",
3128
+ value: localPodspec,
3129
+ },
3130
+ ];
3131
+ } else {
3132
+ let projectRepo = externalPod[":git"];
3133
+ if (isGitHubUrl(projectRepo)) {
3134
+ projectRepo = projectRepo.replace(
3135
+ "github.com",
3136
+ "raw.githubusercontent.com",
3137
+ );
3138
+ }
3139
+ if (projectRepo.endsWith(".git")) {
3140
+ projectRepo = projectRepo.substring(0, projectRepo.length - 4);
3141
+ }
3142
+ // The commit the lockfile recorded in CHECKOUT OPTIONS names the
3143
+ // exact revision CocoaPods checked out, so its podspec is asked
3144
+ // for directly instead of probing the default branch names. It
3145
+ // wins over a branch, whose head may have moved since.
3146
+ const checkoutCommit =
3147
+ podfileLock["CHECKOUT OPTIONS"]?.[podName]?.[":commit"];
3148
+ const projectRepoBranchOrTag =
3149
+ typeof checkoutCommit === "string" &&
3150
+ /^[0-9a-f]{7,64}$/i.test(checkoutCommit)
3151
+ ? checkoutCommit
3152
+ : externalPod[":tag"]
3153
+ ? `refs/tags/${externalPod[":tag"]}`
3154
+ : externalPod[":branch"]
3155
+ ? `refs/heads/${externalPod[":branch"]}`
3156
+ : "refs/heads/<DEFAULT>";
3157
+ dependency.metadata.properties = [
3158
+ {
3159
+ name: "cdx:pods:podspecLocation",
3160
+ value: `${projectRepo}/${projectRepoBranchOrTag}/${podName}.podspec`,
3161
+ },
3162
+ ];
3046
3163
  }
3047
- const projectRepoBranchOrTag = externalPod[":tag"]
3048
- ? `tags/${externalPod[":tag"]}`
3049
- : `heads/${externalPod[":branch"] ? externalPod[":branch"] : "<DEFAULT>"}`;
3050
- dependency.metadata.properties = [
3051
- {
3052
- name: "cdx:pods:podspecLocation",
3053
- value: `${projectRepo}/refs/${projectRepoBranchOrTag}/${podName}.podspec`,
3054
- },
3055
- ];
3056
3164
  } else if (externalPod[":path"]) {
3057
- const projectLocation = resolve(projectPath, externalPod[":path"]);
3058
- dependency.metadata.properties = [
3059
- {
3060
- name: "cdx:pods:projectDir",
3061
- value: projectLocation,
3062
- },
3063
- ];
3064
- let podspec = join(projectLocation, `${podName}.podspec`);
3065
- if (!safeExistsSync(podspec)) {
3066
- podspec = `${podspec}.json`;
3067
- }
3068
- if (safeExistsSync(podspec)) {
3069
- dependency.metadata.properties.push({
3070
- name: "cdx:pods:podspecLocation",
3071
- value: podspec,
3072
- });
3165
+ const projectLocation = resolveInsideScan(
3166
+ projectPath,
3167
+ externalPod[":path"],
3168
+ scanRoot,
3169
+ );
3170
+ if (projectLocation) {
3171
+ dependency.metadata.properties = [
3172
+ {
3173
+ name: "cdx:pods:projectDir",
3174
+ value: projectLocation,
3175
+ },
3176
+ ];
3177
+ let podspec = join(projectLocation, `${podName}.podspec`);
3178
+ if (!safeExistsSync(podspec)) {
3179
+ podspec = `${podspec}.json`;
3180
+ }
3181
+ if (safeExistsSync(podspec)) {
3182
+ dependency.metadata.properties.push({
3183
+ name: "cdx:pods:podspecLocation",
3184
+ value: podspec,
3185
+ });
3186
+ }
3073
3187
  }
3074
3188
  } else if (externalPod[":podspec"]) {
3075
- const podspecLocation = resolve(projectPath, externalPod[":podspec"]);
3076
- dependency.metadata.properties = [
3077
- {
3078
- name: "cdx:pods:projectDir",
3079
- value: dirname(podspecLocation),
3080
- },
3081
- {
3082
- name: "cdx:pods:podspecLocation",
3083
- value: podspecLocation,
3084
- },
3085
- ];
3189
+ const podspecLocation = resolveInsideScan(
3190
+ projectPath,
3191
+ externalPod[":podspec"],
3192
+ scanRoot,
3193
+ );
3194
+ if (podspecLocation) {
3195
+ dependency.metadata.properties = [
3196
+ {
3197
+ name: "cdx:pods:projectDir",
3198
+ value: dirname(podspecLocation),
3199
+ },
3200
+ {
3201
+ name: "cdx:pods:podspecLocation",
3202
+ value: podspecLocation,
3203
+ },
3204
+ ];
3205
+ }
3086
3206
  }
3087
3207
  }
3088
3208
  dependencies.set(dependency.metadata.name, dependency);
@@ -3408,6 +3528,8 @@ export function resetCocoaPodspecPrefetch() {
3408
3528
  cocoaPodspecPrefetch = new Map();
3409
3529
  }
3410
3530
 
3531
+ registerRunReset("cocoapods:podspecs", resetCocoaPodspecPrefetch);
3532
+
3411
3533
  /**
3412
3534
  * Prefetch the remote podspecs a full CocoaPods scan is about to read.
3413
3535
  *
@@ -3510,7 +3632,7 @@ async function fullScanCocoaPod(dependency, component, options) {
3510
3632
  try {
3511
3633
  httpResult =
3512
3634
  prefetchedResponse(cocoaPodspecPrefetch, candidate) ||
3513
- (await cdxgenAgent.get(candidate));
3635
+ (await gatedGet(candidate, { responseType: "text" }));
3514
3636
  podspecLocation = candidate;
3515
3637
  } catch (_err) {
3516
3638
  continue;
@@ -3525,6 +3647,11 @@ async function fullScanCocoaPod(dependency, component, options) {
3525
3647
  } else {
3526
3648
  podspecContent = readFileSync(podspecLocation, "utf-8");
3527
3649
  }
3650
+ // A location none of the candidates served contributes nothing: the
3651
+ // component keeps what the lockfile already said about the pod.
3652
+ if (podspecContent === undefined) {
3653
+ return;
3654
+ }
3528
3655
  for (const replacement of replacements) {
3529
3656
  const replacementPair = replacement.split("=");
3530
3657
  let match = replacementPair[0].replaceAll("<NEWLINE>", "\n");
@@ -4,13 +4,13 @@ import process from "node:process";
4
4
  import { URL } from "node:url";
5
5
 
6
6
  import { build } from "@cdxgen/cdx-purl";
7
- import * as toml from "smol-toml";
8
7
  import { parse as _load } from "yaml";
9
8
 
10
9
  import { DEBUG_MODE } from "../core/activity.js";
11
10
  import { recordDegradation } from "../core/buildLedger.js";
12
11
  import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
13
12
  import { PYTHON_STD_MODULES } from "../core/state.js";
13
+ import { parseToml } from "../core/toml.js";
14
14
  import { tryBuildPurl } from "../inventory/purl.js";
15
15
  import {
16
16
  createExternalReferenceKey,
@@ -95,8 +95,11 @@ export function parsePyRequiresDist(dist_string) {
95
95
  * Method to parse pipfile.lock data
96
96
  *
97
97
  * @param {Object} lockData JSON data from Pipfile.lock
98
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
99
+ * `*.dist-info/METADATA` files of this machine, which answer before PyPI
100
+ * is asked
98
101
  */
99
- export async function parsePiplockData(lockData) {
102
+ export async function parsePiplockData(lockData, installedMetadata) {
100
103
  const pkgList = [];
101
104
  Object.keys(lockData)
102
105
  .filter((i) => i !== "_meta")
@@ -112,6 +115,9 @@ export async function parsePiplockData(lockData) {
112
115
  }
113
116
  });
114
117
  });
118
+ // The installed distributions answer first, so PyPI is asked only about
119
+ // the packages they do not cover.
120
+ enrichFromInstalledMetadata(pkgList, installedMetadata);
115
121
  return await getPyMetadata(pkgList, false);
116
122
  }
117
123
 
@@ -274,8 +280,12 @@ function collectPythonManifestSource(pkg) {
274
280
  { kind: "url", value: pkg?.vcs?.url },
275
281
  { kind: "url", value: pkg?.source?.url },
276
282
  { kind: "path", value: pkg?.source?.path },
283
+ { kind: "path", value: pkg?.source?.directory },
277
284
  { kind: "path", value: pkg?.source?.editable },
278
285
  { kind: "path", value: pkg?.source?.virtual },
286
+ { kind: "path", value: pkg?.directory?.path },
287
+ { kind: "url", value: pkg?.archive?.url },
288
+ { kind: "path", value: pkg?.archive?.path },
279
289
  ];
280
290
  for (const candidate of sourceCandidates) {
281
291
  if (typeof candidate.value !== "string" || !candidate.value.trim()) {
@@ -395,7 +405,7 @@ export function parsePyProjectTomlFile(tomlFile) {
395
405
  const groupDepsKeys = {};
396
406
  const dependencySourceMap = {};
397
407
  try {
398
- tomlData = toml.parse(readFileSync(tomlFile, { encoding: "utf-8" }));
408
+ tomlData = parseToml(readFileSync(tomlFile, { encoding: "utf-8" }));
399
409
  } catch (err) {
400
410
  console.log(`Error while parsing the pyproject file ${tomlFile}.`, err);
401
411
  }
@@ -788,14 +798,114 @@ function collectPythonLockMetadataDistributionReferences(fileEntries) {
788
798
  return distributionReferences;
789
799
  }
790
800
 
801
+ /**
802
+ * Normalise a Python distribution name the way PEP 503 does, so an installed
803
+ * distribution and the lockfile entry it belongs to compare equal whatever
804
+ * separator and case each of them used.
805
+ *
806
+ * @param {string} value Distribution name
807
+ * @returns {string|undefined} The normalised name, or undefined for no name
808
+ */
809
+ export function normalizePep503Name(value) {
810
+ if (typeof value !== "string" || !value.trim()) {
811
+ return undefined;
812
+ }
813
+ return value
814
+ .trim()
815
+ .toLowerCase()
816
+ .replaceAll(/[-_.]+/g, "-");
817
+ }
818
+
819
+ /**
820
+ * The key an installed distribution and a lockfile entry are matched by: the
821
+ * PEP 503 normalised name and the version.
822
+ *
823
+ * @param {Object} p Component with `name` and `version`
824
+ * @returns {string|undefined} The key, or undefined without a name or version
825
+ */
826
+ export function installedMetadataKey(p) {
827
+ const name = normalizePep503Name(p?.name);
828
+ const version = typeof p?.version === "string" ? p.version.trim() : "";
829
+ return name && version ? `${name}@${version}` : undefined;
830
+ }
831
+
832
+ /**
833
+ * Enrich lockfile entries from the installed distributions on this machine,
834
+ * before PyPI is asked.
835
+ *
836
+ * Each entry of `installedMetadata` is a component as
837
+ * {@link parseBdistMetadata} builds it from an installed `*.dist-info/METADATA`
838
+ * file. An entry is matched to the lockfile component with the same PEP 503
839
+ * normalised name and the same version, and fills the fields it lacks, so the
840
+ * registry is left only what the installed distributions do not answer. An
841
+ * installed release of another version describes a different release, so it
842
+ * enriches nothing. The distributions enrich the entries in place; no new
843
+ * components are created.
844
+ *
845
+ * @param {Object[]} pkgList Lockfile components, enriched in place
846
+ * @param {Object[]} installedMetadata Components parsed from installed
847
+ * `*.dist-info/METADATA` files
848
+ * @returns {Set<string>} The keys of the installations that enriched an entry,
849
+ * as {@link installedMetadataKey} builds them, so a caller that also walks
850
+ * the METADATA files can skip those as components
851
+ */
852
+ export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
853
+ const byKey = new Map();
854
+ for (const p of pkgList || []) {
855
+ const key = installedMetadataKey(p);
856
+ if (key && !byKey.has(key)) {
857
+ byKey.set(key, p);
858
+ }
859
+ }
860
+ const enriched = new Set();
861
+ for (const md of installedMetadata || []) {
862
+ const key = installedMetadataKey(md);
863
+ const p = key ? byKey.get(key) : undefined;
864
+ if (!p) {
865
+ continue;
866
+ }
867
+ enriched.add(key);
868
+ if (!p.description?.trim() && md.description?.trim()) {
869
+ p.description = md.description;
870
+ }
871
+ if (!p.license && md.licenses?.length) {
872
+ // The licence field holds the names a lockfile consumer expects; the
873
+ // CycloneDX licence objects travel in `licenses`.
874
+ p.license = md.licenses
875
+ .map((alicense) => alicense.expression || alicense.license?.name)
876
+ .filter((aname) => aname?.length);
877
+ p.licenses = md.licenses;
878
+ }
879
+ if (!p.homepage && md.homepage) {
880
+ p.homepage = md.homepage;
881
+ }
882
+ if (!p.repository && md.repository) {
883
+ p.repository = md.repository;
884
+ }
885
+ if (!p.author && md.author) {
886
+ p.author = md.author;
887
+ }
888
+ }
889
+ return enriched;
890
+ }
891
+
791
892
  /**
792
893
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
793
894
  *
794
895
  * @param {string} lockData Raw TOML text from poetry.lock, pdm.lock, uv.lock, or pylock.toml
795
896
  * @param {string} lockFile Lock file name for evidence
796
897
  * @param {string} pyProjectFile pyproject.toml file
898
+ * @param {Object} [options] Options
899
+ * @param {Object[]} [options.installedMetadata] Components parsed from the
900
+ * installed `*.dist-info/METADATA` files of this machine, which answer
901
+ * before PyPI is asked
797
902
  */
798
- export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
903
+ export async function parsePyLockData(
904
+ lockData,
905
+ lockFile,
906
+ pyProjectFile,
907
+ options = {},
908
+ ) {
799
909
  let pkgList = [];
800
910
  const rootList = [];
801
911
  const dependenciesList = [];
@@ -942,7 +1052,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
942
1052
  }
943
1053
  let lockTomlObj;
944
1054
  try {
945
- lockTomlObj = toml.parse(lockData);
1055
+ lockTomlObj = parseToml(lockData);
946
1056
  } catch (err) {
947
1057
  if (lockFile) {
948
1058
  // The failing file names the manager whose lock command regenerates
@@ -1418,6 +1528,9 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
1418
1528
  dependsOn: [...dependsOnList].sort(),
1419
1529
  });
1420
1530
  }
1531
+ // The installed distributions answer first, so PyPI is asked only about the
1532
+ // packages they do not cover.
1533
+ enrichFromInstalledMetadata(pkgList, options.installedMetadata);
1421
1534
  pkgList = await getPyMetadata(pkgList, false);
1422
1535
  return {
1423
1536
  parentComponent,
@@ -1444,11 +1557,18 @@ const PEP440_VERSION_CLAUSE = /(===|==|!=|<=|>=|~=|<|>)\s*([0-9A-Za-z.*_+!-]*)/;
1444
1557
  *
1445
1558
  * @param {String} reqFile Requirements.txt file
1446
1559
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
1560
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
1561
+ * `*.dist-info/METADATA` files of this machine, which answer before PyPI
1562
+ * is asked
1447
1563
  *
1448
1564
  * @returns {Promise[Array<Object>]} List of direct dependencies from the requirements file
1449
1565
  */
1450
- export async function parseReqFile(reqFile, fetchDepsInfo = false) {
1451
- return await parseReqData(reqFile, null, fetchDepsInfo);
1566
+ export async function parseReqFile(
1567
+ reqFile,
1568
+ fetchDepsInfo = false,
1569
+ installedMetadata = undefined,
1570
+ ) {
1571
+ return await parseReqData(reqFile, null, fetchDepsInfo, installedMetadata);
1452
1572
  }
1453
1573
 
1454
1574
  const LICENSE_ID_COMMENTS_PATTERN =
@@ -1477,9 +1597,17 @@ function parseLicenseComment(comment) {
1477
1597
  * @param {String} reqFile Requirements.txt file
1478
1598
  * @param {Object} reqData Requirements.txt data for internal invocations from setup.py file etc.
1479
1599
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
1600
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
1601
+ * `*.dist-info/METADATA` files of this machine, which answer before PyPI
1602
+ * is asked
1480
1603
  * @returns {Promise<Array<Object>>} List of direct dependencies from the requirements file
1481
1604
  */
1482
- async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
1605
+ async function parseReqData(
1606
+ reqFile,
1607
+ reqData = null,
1608
+ fetchDepsInfo = false,
1609
+ installedMetadata = undefined,
1610
+ ) {
1483
1611
  const pkgList = [];
1484
1612
  let compScope;
1485
1613
  if (!reqFile && !reqData) {
@@ -1732,6 +1860,9 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
1732
1860
  pkgList.push(apkg);
1733
1861
  }
1734
1862
  }
1863
+ // The installed distributions answer first, so PyPI is asked only about
1864
+ // the packages they do not cover.
1865
+ enrichFromInstalledMetadata(pkgList, installedMetadata);
1735
1866
  return await getPyMetadata(pkgList, fetchDepsInfo);
1736
1867
  }
1737
1868
 
@@ -2042,7 +2173,7 @@ export function parsePixiTomlFile(pixiToml) {
2042
2173
  const pixiTomlFile = readFileSync(pixiToml, { encoding: "utf-8" });
2043
2174
  let tomlData;
2044
2175
  try {
2045
- tomlData = toml.parse(pixiTomlFile);
2176
+ tomlData = parseToml(pixiTomlFile);
2046
2177
  } catch (err) {
2047
2178
  console.log(`Error while parsing the pixi file ${pixiToml}.`, err);
2048
2179
  return {};
@@ -2118,7 +2249,7 @@ export function generatePixiLockFile(_path) {
2118
2249
  export function parseMojoProject(mojoProjectFile) {
2119
2250
  let manifest;
2120
2251
  try {
2121
- manifest = toml.parse(readFileSync(mojoProjectFile, "utf-8"));
2252
+ manifest = parseToml(readFileSync(mojoProjectFile, "utf-8"));
2122
2253
  } catch (error) {
2123
2254
  console.warn(`Failed to parse ${mojoProjectFile}: ${error.message}`);
2124
2255
  return { pkgList: [], parentComponent: {} };
@@ -3,12 +3,12 @@ import { sep as _sep, basename, dirname, join, resolve } from "node:path";
3
3
 
4
4
  import { build } from "@cdxgen/cdx-purl";
5
5
  import { globSync } from "glob";
6
- import * as toml from "smol-toml";
7
6
 
8
7
  import { DEBUG_MODE } from "../core/activity.js";
9
8
  import { shouldFetchLicense } from "../core/env.js";
10
9
  import { safeExistsSync } from "../core/fs.js";
11
10
  import { traceLog } from "../core/logger.js";
11
+ import { parseToml } from "../core/toml.js";
12
12
  import { getCratesMetadata, normalizeCargoIntegrity } from "./ecosystems.js";
13
13
 
14
14
  /**
@@ -204,7 +204,7 @@ function readCargoTomlData(cargoTomlFile) {
204
204
  return undefined;
205
205
  }
206
206
  try {
207
- return toml.parse(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
207
+ return parseToml(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
208
208
  } catch (error) {
209
209
  traceLog("cargo", {
210
210
  cargoTomlFile,
@@ -1082,7 +1082,7 @@ export async function parseCargoData(
1082
1082
  return pkgList;
1083
1083
  }
1084
1084
 
1085
- const cargoData = toml.parse(
1085
+ const cargoData = parseToml(
1086
1086
  readFileSync(cargoLockFile, { encoding: "utf-8" }),
1087
1087
  );
1088
1088
  if (!cargoData) {
@@ -1290,7 +1290,7 @@ export function parseCargoManifestDependencyData(cargoTomlFile, context = {}) {
1290
1290
  */
1291
1291
  export function parseCargoDependencyData(cargoLockData) {
1292
1292
  const purlFromPackageInfo = (pkg) => decodeURIComponent(buildCargoPurl(pkg));
1293
- const cargoData = toml.parse(cargoLockData);
1293
+ const cargoData = parseToml(cargoLockData);
1294
1294
  const packageNode = cargoData?.package;
1295
1295
  if (!packageNode || !Array.isArray(packageNode)) {
1296
1296
  return [];