@cdxgen/cdxgen 13.1.0 → 13.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/README.md +32 -15
  2. package/bin/cdxgen.js +243 -122
  3. package/bin/evinse.js +7 -0
  4. package/bin/sign.js +88 -14
  5. package/bin/tracebom.js +64 -4
  6. package/bin/validate.js +29 -7
  7. package/bin/verify.js +79 -39
  8. package/data/crypto-oid.json +88 -12
  9. package/data/remediations.json +47 -0
  10. package/data/rules/ai-bom.yaml +78 -0
  11. package/data/rules/cargo-rust.yaml +132 -0
  12. package/data/rules/mcp-servers.yaml +122 -0
  13. package/lib/audit/targets.js +6 -1
  14. package/lib/cli/bomAssembly.js +117 -15
  15. package/lib/cli/cliOptions.js +93 -1
  16. package/lib/cli/index.js +235 -5
  17. package/lib/cli/jsBom.js +108 -26
  18. package/lib/cli/jvmBom.js +68 -12
  19. package/lib/cli/managedBom.js +264 -41
  20. package/lib/cli/nativeBom.js +923 -15
  21. package/lib/core/activity.js +120 -2
  22. package/lib/core/env.js +44 -7
  23. package/lib/core/fs.js +174 -6
  24. package/lib/core/metadataProperties.js +97 -0
  25. package/lib/core/projectConfig.js +31 -4
  26. package/lib/core/userAgent.js +151 -0
  27. package/lib/ecosystems/bunutils.js +269 -101
  28. package/lib/ecosystems/cargoCache.js +364 -0
  29. package/lib/ecosystems/caxa.js +447 -7
  30. package/lib/ecosystems/cmakeResolver.js +66 -0
  31. package/lib/ecosystems/cppEvidence.js +104 -33
  32. package/lib/ecosystems/ecosystems.js +260 -99
  33. package/lib/ecosystems/golem.js +11 -1
  34. package/lib/ecosystems/gradleutils.js +185 -9
  35. package/lib/ecosystems/jsEvidence.js +19 -20
  36. package/lib/ecosystems/kosi.js +1263 -0
  37. package/lib/ecosystems/npmutils.js +110 -9
  38. package/lib/ecosystems/parsers-crystal.js +179 -0
  39. package/lib/ecosystems/parsers-dotnet.js +120 -49
  40. package/lib/ecosystems/parsers-elm.js +816 -0
  41. package/lib/ecosystems/parsers-gleam.js +1 -1
  42. package/lib/ecosystems/parsers-go.js +47 -9
  43. package/lib/ecosystems/parsers-js.js +171 -50
  44. package/lib/ecosystems/parsers-julia.js +268 -0
  45. package/lib/ecosystems/parsers-lua.js +218 -0
  46. package/lib/ecosystems/parsers-misc.js +919 -161
  47. package/lib/ecosystems/parsers-nim.js +221 -0
  48. package/lib/ecosystems/parsers-opam.js +180 -0
  49. package/lib/ecosystems/parsers-perl.js +277 -0
  50. package/lib/ecosystems/parsers-python.js +27 -15
  51. package/lib/ecosystems/parsers-r.js +148 -0
  52. package/lib/ecosystems/parsers-rebar.js +411 -0
  53. package/lib/ecosystems/parsers-rust.js +954 -33
  54. package/lib/ecosystems/parsers-spack.js +204 -0
  55. package/lib/ecosystems/parsers-stack.js +264 -0
  56. package/lib/ecosystems/parsers-terraform.js +194 -0
  57. package/lib/ecosystems/rubyutils.js +1426 -166
  58. package/lib/ecosystems/rusi.js +158 -7
  59. package/lib/evinser/evinser.js +547 -129
  60. package/lib/evinser/swiftsem.js +1914 -302
  61. package/lib/helpers/agenticToolutils.js +925 -0
  62. package/lib/helpers/bomSigner.js +652 -221
  63. package/lib/inventory/agentFormulationParser.js +57 -4
  64. package/lib/inventory/aiCollector.js +504 -10
  65. package/lib/inventory/aiInventory.js +12 -0
  66. package/lib/inventory/analyzer.js +488 -30
  67. package/lib/inventory/astWorker.js +6 -0
  68. package/lib/inventory/atomUtils.js +568 -31
  69. package/lib/inventory/auditCategories.js +8 -0
  70. package/lib/inventory/cbomutils.js +30 -0
  71. package/lib/inventory/cdxrs.js +110 -14
  72. package/lib/inventory/communityAiConfigParser.js +215 -2
  73. package/lib/inventory/depsUtils.js +183 -11
  74. package/lib/inventory/display.js +4 -5
  75. package/lib/inventory/dosai.js +627 -13
  76. package/lib/inventory/dynamic.js +39 -1
  77. package/lib/inventory/envcontext.js +9 -6
  78. package/lib/inventory/evidenceUtils.js +26 -5
  79. package/lib/inventory/fetchBatch.js +190 -19
  80. package/lib/inventory/fetchRate.js +74 -11
  81. package/lib/inventory/forgeEnricher.js +3 -2
  82. package/lib/inventory/jvmToolEnv.js +6 -5
  83. package/lib/inventory/mcpConfigParser.js +166 -2
  84. package/lib/inventory/mcpDiscovery.js +11 -0
  85. package/lib/inventory/mcpServerManifest.js +389 -0
  86. package/lib/inventory/mcpbBundle.js +419 -0
  87. package/lib/inventory/plugins.js +3 -0
  88. package/lib/inventory/protobom.js +62 -89
  89. package/lib/inventory/protobomLoader.js +4 -0
  90. package/lib/inventory/purl.js +152 -3
  91. package/lib/inventory/rustFormulationParser.js +1 -1
  92. package/lib/inventory/sbomDocument.js +2 -70
  93. package/lib/inventory/source.js +47 -23
  94. package/lib/inventory/stylePackageRefs.js +41 -0
  95. package/lib/inventory/svelteUtils.js +524 -0
  96. package/lib/inventory/traceRunner.js +210 -21
  97. package/lib/managers/binary.js +115 -24
  98. package/lib/managers/docker.js +440 -20
  99. package/lib/managers/oci.js +39 -1
  100. package/lib/parsers/cmakeLists.js +452 -0
  101. package/lib/parsers/huggingfaceUtils.js +15 -7
  102. package/lib/parsers/largeJson.js +454 -0
  103. package/lib/parsers/npmrc.js +19 -1
  104. package/lib/parsers/onnx.js +294 -0
  105. package/lib/parsers/safetensors.js +114 -0
  106. package/lib/stages/postgen/introspection/markers.js +29 -6
  107. package/lib/stages/postgen/postgen.js +237 -16
  108. package/lib/third-party/arborist/snapshots/calc-dep-flags.test.cjs +462 -0
  109. package/lib/validator/bomValidator.js +6 -16
  110. package/lib/validator/index.js +48 -10
  111. package/lib/validator/reporters/console.js +3 -0
  112. package/lib/validator/reporters/json.js +1 -0
  113. package/lib/validator/reporters/sarif.js +7 -0
  114. package/package.json +27 -27
  115. package/types/lib/audit/targets.d.ts.map +1 -1
  116. package/types/lib/cli/bomAssembly.d.ts +2 -1
  117. package/types/lib/cli/bomAssembly.d.ts.map +1 -1
  118. package/types/lib/cli/cliOptions.d.ts +33 -1
  119. package/types/lib/cli/cliOptions.d.ts.map +1 -1
  120. package/types/lib/cli/index.d.ts +2 -2
  121. package/types/lib/cli/index.d.ts.map +1 -1
  122. package/types/lib/cli/jsBom.d.ts +23 -0
  123. package/types/lib/cli/jsBom.d.ts.map +1 -1
  124. package/types/lib/cli/jvmBom.d.ts.map +1 -1
  125. package/types/lib/cli/managedBom.d.ts +33 -0
  126. package/types/lib/cli/managedBom.d.ts.map +1 -1
  127. package/types/lib/cli/nativeBom.d.ts +88 -8
  128. package/types/lib/cli/nativeBom.d.ts.map +1 -1
  129. package/types/lib/core/activity.d.ts.map +1 -1
  130. package/types/lib/core/env.d.ts +21 -0
  131. package/types/lib/core/env.d.ts.map +1 -1
  132. package/types/lib/core/fs.d.ts +55 -0
  133. package/types/lib/core/fs.d.ts.map +1 -1
  134. package/types/lib/core/metadataProperties.d.ts +39 -0
  135. package/types/lib/core/metadataProperties.d.ts.map +1 -0
  136. package/types/lib/core/projectConfig.d.ts +6 -2
  137. package/types/lib/core/projectConfig.d.ts.map +1 -1
  138. package/types/lib/core/userAgent.d.ts +56 -0
  139. package/types/lib/core/userAgent.d.ts.map +1 -0
  140. package/types/lib/ecosystems/bunutils.d.ts +12 -5
  141. package/types/lib/ecosystems/bunutils.d.ts.map +1 -1
  142. package/types/lib/ecosystems/cargoCache.d.ts +75 -0
  143. package/types/lib/ecosystems/cargoCache.d.ts.map +1 -0
  144. package/types/lib/ecosystems/caxa.d.ts +14 -0
  145. package/types/lib/ecosystems/caxa.d.ts.map +1 -1
  146. package/types/lib/ecosystems/cmakeResolver.d.ts +23 -0
  147. package/types/lib/ecosystems/cmakeResolver.d.ts.map +1 -1
  148. package/types/lib/ecosystems/cppEvidence.d.ts +2 -2
  149. package/types/lib/ecosystems/cppEvidence.d.ts.map +1 -1
  150. package/types/lib/ecosystems/ecosystems.d.ts +15 -1
  151. package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
  152. package/types/lib/ecosystems/golem.d.ts.map +1 -1
  153. package/types/lib/ecosystems/gradleutils.d.ts +40 -4
  154. package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
  155. package/types/lib/ecosystems/jsEvidence.d.ts +5 -2
  156. package/types/lib/ecosystems/jsEvidence.d.ts.map +1 -1
  157. package/types/lib/ecosystems/kosi.d.ts +86 -0
  158. package/types/lib/ecosystems/kosi.d.ts.map +1 -0
  159. package/types/lib/ecosystems/npmutils.d.ts +42 -0
  160. package/types/lib/ecosystems/npmutils.d.ts.map +1 -1
  161. package/types/lib/ecosystems/parsers-crystal.d.ts +27 -0
  162. package/types/lib/ecosystems/parsers-crystal.d.ts.map +1 -0
  163. package/types/lib/ecosystems/parsers-dotnet.d.ts +19 -0
  164. package/types/lib/ecosystems/parsers-dotnet.d.ts.map +1 -1
  165. package/types/lib/ecosystems/parsers-elm.d.ts +79 -0
  166. package/types/lib/ecosystems/parsers-elm.d.ts.map +1 -0
  167. package/types/lib/ecosystems/parsers-go.d.ts +10 -0
  168. package/types/lib/ecosystems/parsers-go.d.ts.map +1 -1
  169. package/types/lib/ecosystems/parsers-js.d.ts +10 -4
  170. package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
  171. package/types/lib/ecosystems/parsers-julia.d.ts +57 -0
  172. package/types/lib/ecosystems/parsers-julia.d.ts.map +1 -0
  173. package/types/lib/ecosystems/parsers-lua.d.ts +50 -0
  174. package/types/lib/ecosystems/parsers-lua.d.ts.map +1 -0
  175. package/types/lib/ecosystems/parsers-misc.d.ts +90 -3
  176. package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
  177. package/types/lib/ecosystems/parsers-nim.d.ts +43 -0
  178. package/types/lib/ecosystems/parsers-nim.d.ts.map +1 -0
  179. package/types/lib/ecosystems/parsers-opam.d.ts +31 -0
  180. package/types/lib/ecosystems/parsers-opam.d.ts.map +1 -0
  181. package/types/lib/ecosystems/parsers-perl.d.ts +36 -0
  182. package/types/lib/ecosystems/parsers-perl.d.ts.map +1 -0
  183. package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
  184. package/types/lib/ecosystems/parsers-r.d.ts +35 -0
  185. package/types/lib/ecosystems/parsers-r.d.ts.map +1 -0
  186. package/types/lib/ecosystems/parsers-rebar.d.ts +42 -0
  187. package/types/lib/ecosystems/parsers-rebar.d.ts.map +1 -0
  188. package/types/lib/ecosystems/parsers-rust.d.ts +199 -0
  189. package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
  190. package/types/lib/ecosystems/parsers-spack.d.ts +33 -0
  191. package/types/lib/ecosystems/parsers-spack.d.ts.map +1 -0
  192. package/types/lib/ecosystems/parsers-stack.d.ts +35 -0
  193. package/types/lib/ecosystems/parsers-stack.d.ts.map +1 -0
  194. package/types/lib/ecosystems/parsers-terraform.d.ts +34 -0
  195. package/types/lib/ecosystems/parsers-terraform.d.ts.map +1 -0
  196. package/types/lib/ecosystems/pythonutils.d.ts +1 -1
  197. package/types/lib/ecosystems/rubyutils.d.ts +143 -2
  198. package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
  199. package/types/lib/ecosystems/rusi.d.ts +13 -0
  200. package/types/lib/ecosystems/rusi.d.ts.map +1 -1
  201. package/types/lib/evinser/evinser.d.ts +81 -2
  202. package/types/lib/evinser/evinser.d.ts.map +1 -1
  203. package/types/lib/evinser/swiftsem.d.ts +280 -15
  204. package/types/lib/evinser/swiftsem.d.ts.map +1 -1
  205. package/types/lib/helpers/agenticToolutils.d.ts +87 -0
  206. package/types/lib/helpers/agenticToolutils.d.ts.map +1 -0
  207. package/types/lib/helpers/bomSigner.d.ts +106 -13
  208. package/types/lib/helpers/bomSigner.d.ts.map +1 -1
  209. package/types/lib/inventory/agentFormulationParser.d.ts.map +1 -1
  210. package/types/lib/inventory/aiCollector.d.ts.map +1 -1
  211. package/types/lib/inventory/aiInventory.d.ts.map +1 -1
  212. package/types/lib/inventory/analyzer.d.ts.map +1 -1
  213. package/types/lib/inventory/atomUtils.d.ts +145 -3
  214. package/types/lib/inventory/atomUtils.d.ts.map +1 -1
  215. package/types/lib/inventory/auditCategories.d.ts +4 -0
  216. package/types/lib/inventory/auditCategories.d.ts.map +1 -1
  217. package/types/lib/inventory/cbomutils.d.ts.map +1 -1
  218. package/types/lib/inventory/cdxrs.d.ts +8 -3
  219. package/types/lib/inventory/cdxrs.d.ts.map +1 -1
  220. package/types/lib/inventory/communityAiConfigParser.d.ts.map +1 -1
  221. package/types/lib/inventory/depsUtils.d.ts +15 -0
  222. package/types/lib/inventory/depsUtils.d.ts.map +1 -1
  223. package/types/lib/inventory/display.d.ts.map +1 -1
  224. package/types/lib/inventory/dosai.d.ts +87 -1
  225. package/types/lib/inventory/dosai.d.ts.map +1 -1
  226. package/types/lib/inventory/dynamic.d.ts.map +1 -1
  227. package/types/lib/inventory/envcontext.d.ts.map +1 -1
  228. package/types/lib/inventory/evidenceUtils.d.ts.map +1 -1
  229. package/types/lib/inventory/fetchBatch.d.ts +34 -0
  230. package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
  231. package/types/lib/inventory/fetchRate.d.ts +9 -6
  232. package/types/lib/inventory/fetchRate.d.ts.map +1 -1
  233. package/types/lib/inventory/forgeEnricher.d.ts.map +1 -1
  234. package/types/lib/inventory/jvmToolEnv.d.ts +5 -4
  235. package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -1
  236. package/types/lib/inventory/mcpConfigParser.d.ts.map +1 -1
  237. package/types/lib/inventory/mcpDiscovery.d.ts.map +1 -1
  238. package/types/lib/inventory/mcpServerManifest.d.ts +19 -0
  239. package/types/lib/inventory/mcpServerManifest.d.ts.map +1 -0
  240. package/types/lib/inventory/mcpbBundle.d.ts +19 -0
  241. package/types/lib/inventory/mcpbBundle.d.ts.map +1 -0
  242. package/types/lib/inventory/plugins.d.ts.map +1 -1
  243. package/types/lib/inventory/protobom.d.ts +18 -13
  244. package/types/lib/inventory/protobom.d.ts.map +1 -1
  245. package/types/lib/inventory/protobomLoader.d.ts.map +1 -1
  246. package/types/lib/inventory/purl.d.ts +71 -2
  247. package/types/lib/inventory/purl.d.ts.map +1 -1
  248. package/types/lib/inventory/sbomDocument.d.ts.map +1 -1
  249. package/types/lib/inventory/source.d.ts.map +1 -1
  250. package/types/lib/inventory/stylePackageRefs.d.ts +17 -0
  251. package/types/lib/inventory/stylePackageRefs.d.ts.map +1 -0
  252. package/types/lib/inventory/svelteUtils.d.ts +124 -0
  253. package/types/lib/inventory/svelteUtils.d.ts.map +1 -0
  254. package/types/lib/inventory/traceRunner.d.ts +47 -1
  255. package/types/lib/inventory/traceRunner.d.ts.map +1 -1
  256. package/types/lib/managers/binary.d.ts +30 -1
  257. package/types/lib/managers/binary.d.ts.map +1 -1
  258. package/types/lib/managers/docker.d.ts +29 -0
  259. package/types/lib/managers/docker.d.ts.map +1 -1
  260. package/types/lib/managers/oci.d.ts +14 -0
  261. package/types/lib/managers/oci.d.ts.map +1 -1
  262. package/types/lib/parsers/cmakeLists.d.ts +76 -0
  263. package/types/lib/parsers/cmakeLists.d.ts.map +1 -0
  264. package/types/lib/parsers/huggingfaceUtils.d.ts +8 -1
  265. package/types/lib/parsers/huggingfaceUtils.d.ts.map +1 -1
  266. package/types/lib/parsers/largeJson.d.ts +52 -0
  267. package/types/lib/parsers/largeJson.d.ts.map +1 -0
  268. package/types/lib/parsers/npmrc.d.ts +5 -0
  269. package/types/lib/parsers/npmrc.d.ts.map +1 -1
  270. package/types/lib/parsers/onnx.d.ts +28 -0
  271. package/types/lib/parsers/onnx.d.ts.map +1 -0
  272. package/types/lib/parsers/safetensors.d.ts +22 -0
  273. package/types/lib/parsers/safetensors.d.ts.map +1 -0
  274. package/types/lib/stages/postgen/introspection/markers.d.ts +3 -2
  275. package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -1
  276. package/types/lib/stages/postgen/postgen.d.ts +33 -0
  277. package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
  278. package/types/lib/validator/bomValidator.d.ts.map +1 -1
  279. package/types/lib/validator/index.d.ts +10 -1
  280. package/types/lib/validator/index.d.ts.map +1 -1
  281. package/types/lib/validator/reporters/console.d.ts.map +1 -1
  282. package/types/lib/validator/reporters/json.d.ts.map +1 -1
  283. package/types/lib/validator/reporters/sarif.d.ts.map +1 -1
@@ -34,6 +34,7 @@ export async function buildDynamicComponents(
34
34
  const result = await executeAndTrace(commandStr, workingDir, traceOptions);
35
35
  const libPaths = result.libPaths || [];
36
36
  const httpAccessEntries = result.httpAccessEntries || [];
37
+ const egressEntries = result.egressEntries || [];
37
38
  const cryptoComponents = result.cryptoComponents || [];
38
39
  const components = [];
39
40
 
@@ -141,5 +142,42 @@ export async function buildDynamicComponents(
141
142
  };
142
143
  });
143
144
 
144
- return { components, services };
145
+ // Egress seen by the safer-exec proxy: hostnames the command connected to
146
+ // (allowed) or tried to reach (denied). This works on every platform,
147
+ // including macOS where eBPF URL tracing is unavailable.
148
+ const servicesByName = new Map(services.map((svc) => [svc.name, svc]));
149
+ for (const egress of egressEntries) {
150
+ const name = `dynamic-${egress.host}-${egress.port}`;
151
+ const defaultPort = egress.protocol === "http" ? 80 : 443;
152
+ const endpoint = `${egress.protocol}://${egress.host}${egress.port !== defaultPort ? `:${egress.port}` : ""}`;
153
+ let svc = servicesByName.get(name);
154
+ if (!svc) {
155
+ svc = {
156
+ name,
157
+ bomRef: `urn:service:dynamic:${name}`,
158
+ endpoints: [],
159
+ properties: [],
160
+ };
161
+ servicesByName.set(name, svc);
162
+ services.push(svc);
163
+ }
164
+ if (!svc.endpoints.some((e) => e.startsWith(endpoint))) {
165
+ svc.endpoints.push(endpoint);
166
+ svc.endpoints.sort();
167
+ }
168
+ if (
169
+ !svc.properties.some(
170
+ (p) =>
171
+ p.name === "cdx:dynamic:egressDecision" &&
172
+ p.value === egress.decision,
173
+ )
174
+ ) {
175
+ svc.properties.push({
176
+ name: "cdx:dynamic:egressDecision",
177
+ value: egress.decision,
178
+ });
179
+ }
180
+ }
181
+
182
+ return { components, services, dryRunSummary: result.dryRunSummary };
145
183
  }
@@ -101,15 +101,18 @@ function hardenedGitEnv() {
101
101
  * @type {Object<string, string>}
102
102
  */
103
103
  export const SDKMAN_JAVA_TOOL_ALIASES = {
104
- java8: readEnvironmentVariable("JAVA8_TOOL") || "8.0.452-amzn", // Temurin no longer offers java8 :(
105
- java11: readEnvironmentVariable("JAVA11_TOOL") || "11.0.31-tem",
106
- java17: readEnvironmentVariable("JAVA17_TOOL") || "17.0.19-tem",
107
- java21: readEnvironmentVariable("JAVA21_TOOL") || "21.0.11-tem",
104
+ // Identifiers must be downloadable from the sdkman broker: the `sdk list`
105
+ // output advertises Temurin respin ids such as 21.0.12+1.1-tem, but the
106
+ // broker cannot serve ids containing '+', so the plain ids are pinned.
107
+ java8: readEnvironmentVariable("JAVA8_TOOL") || "8.0.504-zulu", // Temurin no longer offers java8 :(
108
+ java11: readEnvironmentVariable("JAVA11_TOOL") || "11.0.32-tem",
109
+ java17: readEnvironmentVariable("JAVA17_TOOL") || "17.0.20-tem",
110
+ java21: readEnvironmentVariable("JAVA21_TOOL") || "21.0.12-tem",
108
111
  java22: readEnvironmentVariable("JAVA22_TOOL") || "22.0.2-tem",
109
112
  java23: readEnvironmentVariable("JAVA23_TOOL") || "23.0.2-tem",
110
113
  java24: readEnvironmentVariable("JAVA24_TOOL") || "24.0.2-tem",
111
- java25: readEnvironmentVariable("JAVA25_TOOL") || "25.0.3-tem",
112
- java26: readEnvironmentVariable("JAVA26_TOOL") || "26.0.1-tem",
114
+ java25: readEnvironmentVariable("JAVA25_TOOL") || "25.0.4-tem",
115
+ java26: readEnvironmentVariable("JAVA26_TOOL") || "26.0.2-tem",
113
116
  };
114
117
 
115
118
  /**
@@ -1,4 +1,3 @@
1
- import { readFileSync } from "node:fs";
2
1
  import { platform } from "node:os";
3
2
  import { extname } from "node:path";
4
3
 
@@ -6,6 +5,10 @@ import { build } from "@cdxgen/cdx-purl";
6
5
 
7
6
  import { safeExistsSync } from "../core/fs.js";
8
7
  import { thoughtLog } from "../core/logger.js";
8
+ import {
9
+ addDosaiReachabilityProperties,
10
+ readDosaiMethodsReport,
11
+ } from "./dosai.js";
9
12
  import {
10
13
  addDosaiSetValue,
11
14
  buildDosaiPurlAliasMap,
@@ -256,6 +259,7 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
256
259
  const purlLocationMap = {};
257
260
  const purlModulesMap = {};
258
261
  const purlMethodsMap = {};
262
+ const purlReachabilityMap = {};
259
263
  const purlAliasMap = buildDosaiPurlAliasMap(pkgList);
260
264
  for (const apkg of pkgList) {
261
265
  if (apkg.properties && Array.isArray(apkg.properties)) {
@@ -273,10 +277,16 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
273
277
  });
274
278
  }
275
279
  }
276
- let slicesData;
277
- try {
278
- slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
279
- } catch (_err) {
280
+ // Only MethodCalls into a package .dll produce evidence below, so a report
281
+ // too large to parse whole keeps just those.
282
+ const slicesData = readDosaiMethodsReport(slicesFile, {
283
+ keepMethodCall: (methodCall) =>
284
+ Boolean(
285
+ methodCall?.Module?.endsWith(".dll") &&
286
+ pkgFilePurlMap[methodCall.Module],
287
+ ),
288
+ });
289
+ if (!slicesData) {
280
290
  return pkgList;
281
291
  }
282
292
  if (slicesData && Object.keys(slicesData)) {
@@ -340,6 +350,9 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
340
350
  if (!modPurl) {
341
351
  continue;
342
352
  }
353
+ // Confidence/evidence facts for this purl, applied to the component below so an
354
+ // unbuilt tree stays distinguishable from an only-imported package.
355
+ purlReachabilityMap[modPurl] ??= reachability;
343
356
  let hasExplicitSourceLocations = false;
344
357
  for (const sourceLocation of reachability.SourceLocations || []) {
345
358
  const location = dosaiSourceLocation(sourceLocation);
@@ -456,6 +469,14 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
456
469
  "I didn't find any occurrence evidence or detailed imported modules, even though there is good dependency slice data from dosai. This is surprising.",
457
470
  );
458
471
  }
472
+ // Reachability confidence facts (kind, confidence, evidence, reasons) as properties.
473
+ // Applied outside the location guard: a Low-confidence unbuilt tree must stay
474
+ // distinguishable from an only-imported package even when no locations were found.
475
+ for (const apkg of pkgList) {
476
+ if (purlReachabilityMap[apkg.purl]) {
477
+ addDosaiReachabilityProperties(apkg, purlReachabilityMap[apkg.purl]);
478
+ }
479
+ }
459
480
  return pkgList;
460
481
  }
461
482
 
@@ -248,33 +248,178 @@ export async function prefetchJson(requests, options = {}) {
248
248
  * - a non-JSON `responseType`, since the envelope has nowhere to put HTML;
249
249
  * - request-specific `headers`, since cdxrs derives its own Authorization from
250
250
  * `GITHUB_TOKEN` and would silently drop any other credential, turning an
251
- * authenticated lookup into an anonymous one.
251
+ * authenticated lookup into an anonymous one;
252
+ * - anything addressed to a host whose rate policy only one transport knows.
253
+ * crates.io's crawler policy allows one request per second with no
254
+ * parallelism, and package.elm-lang.org has a JS-only allowance because
255
+ * rate.rs carries no row for it. Both transports keep separate gates, so a
256
+ * batch split across the two would issue twice the agreed rate; the JS pool
257
+ * owns these hosts so there is one gate to reason about.
252
258
  *
253
259
  * The pool applies the same global and per-host concurrency and rate policy, so
254
260
  * the batching win is unaffected; only the on-disk cache is skipped.
255
261
  *
256
- * @param {{responseType?: string, headers?: Object}} request
262
+ * @param {{url?: string, responseType?: string, headers?: Object}} request
257
263
  * @returns {boolean}
258
264
  */
259
265
  function cdxrsCanServe(request) {
260
266
  if (request.responseType && request.responseType !== "json") {
261
267
  return false;
262
268
  }
269
+ if (isSingleGateHost(extractHost(request.url))) {
270
+ return false;
271
+ }
263
272
  return !request.headers || !Object.keys(request.headers).length;
264
273
  }
265
274
 
275
+ /**
276
+ * Whether a hostname belongs to crates.io, including its index and CDN hosts.
277
+ *
278
+ * @param {string|null} host Hostname, or null when the URL did not parse.
279
+ * @returns {boolean}
280
+ */
281
+ export function isCratesHost(host) {
282
+ if (!host) {
283
+ return false;
284
+ }
285
+ const lower = host.toLowerCase();
286
+ return lower === "crates.io" || lower.endsWith(".crates.io");
287
+ }
288
+
289
+ /**
290
+ * Whether a host's rate policy lives in the JS table alone and therefore needs
291
+ * every request to pass through the one gate the JS pool holds.
292
+ *
293
+ * @param {string|null} host Hostname, or null when the URL did not parse.
294
+ * @returns {boolean}
295
+ */
296
+ export function isSingleGateHost(host) {
297
+ if (!host) {
298
+ return false;
299
+ }
300
+ return isCratesHost(host) || host.toLowerCase() === "package.elm-lang.org";
301
+ }
302
+
303
+ /**
304
+ * Most URLs sent to one `cdxrs fetch` run. The envelope comes back as a single
305
+ * JSON string and V8 caps strings at ~512 MB, while a full npm packument is a
306
+ * few hundred KB on average and tens of MB for the largest packages. One run
307
+ * per workspace therefore outgrew the limit on large pnpm workspaces (issue
308
+ * 4393); runs of this size stay far below it for any realistic mix.
309
+ */
310
+ const RUST_FETCH_CHUNK_SIZE = 250;
311
+
266
312
  /**
267
313
  * Run the batch through the `cdxrs fetch` subprocess and translate its
268
314
  * envelope into the shared result shape.
269
315
  *
316
+ * The URLs go out in chunks of {@link RUST_FETCH_CHUNK_SIZE}, one run at a time
317
+ * so each host still sees a single rate gate. A chunk whose envelope is still
318
+ * too large to deliver is split in half and retried, which is cheap because
319
+ * the first attempt filled the cdxrs disk cache. A chunk that fails for any
320
+ * other reason falls back to the JS pool on its own, so one bad run never
321
+ * costs the rest of the batch its Rust fetch.
322
+ *
270
323
  * @param {Array<{url: string, accept?: string, authRealm?: string}>} unique
271
324
  * Already deduplicated by URL.
272
325
  * @param {Object} options Options with `timeoutMs`.
273
326
  * @returns {Promise<Map<string, BatchEntry>>}
274
327
  */
275
328
  async function rustBatchFetch(unique, options) {
329
+ const pending = [];
330
+ for (let i = 0; i < unique.length; i += RUST_FETCH_CHUNK_SIZE) {
331
+ pending.push(unique.slice(i, i + RUST_FETCH_CHUNK_SIZE));
332
+ }
333
+ const results = new Map();
334
+ const chunkStats = [];
335
+ while (pending.length) {
336
+ const chunk = pending.shift();
337
+ const outcome = await rustFetchChunk(chunk, options);
338
+ if (outcome.reason === "stdout-too-large" && chunk.length > 1) {
339
+ const middle = Math.ceil(chunk.length / 2);
340
+ pending.unshift(chunk.slice(0, middle), chunk.slice(middle));
341
+ continue;
342
+ }
343
+ if (!outcome.results) {
344
+ // Not "binary absent" (that never reaches here), so fall through to the
345
+ // JS pool rather than returning nothing: a transient cdxrs crash must
346
+ // not take this chunk's concurrency with it.
347
+ for (const [url, entry] of await jsBatchFetch(chunk)) {
348
+ results.set(url, entry);
349
+ }
350
+ chunkStats.push(_lastStats);
351
+ continue;
352
+ }
353
+ for (const [url, entry] of outcome.results) {
354
+ results.set(url, entry);
355
+ }
356
+ chunkStats.push(outcome.stats);
357
+ }
358
+ _lastStats = combineBatchStats(chunkStats);
359
+ if (DEBUG_MODE && _lastStats) {
360
+ const s = _lastStats;
361
+ console.log(
362
+ `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
363
+ );
364
+ }
365
+ return results;
366
+ }
367
+
368
+ /**
369
+ * Add up the stats of the runs that served one batch. Numeric fields sum
370
+ * because the runs are sequential, except peak concurrency, which is the
371
+ * widest run. Per-host entries under `hosts` are combined the same way.
372
+ *
373
+ * @param {Array<Object|null|undefined>} parts Stats from each run.
374
+ * @returns {Object|null} Combined stats, or null when no run reported any.
375
+ */
376
+ function combineBatchStats(parts) {
377
+ const reported = parts.filter(Boolean);
378
+ if (!reported.length) {
379
+ return null;
380
+ }
381
+ if (reported.length === 1) {
382
+ return reported[0];
383
+ }
384
+ const combined = addStats({}, reported);
385
+ const hosts = {};
386
+ for (const part of reported) {
387
+ for (const [host, entry] of Object.entries(part.hosts || {})) {
388
+ hosts[host] = addStats(hosts[host] || {}, [entry]);
389
+ }
390
+ }
391
+ if (Object.keys(hosts).length) {
392
+ combined.hosts = hosts;
393
+ }
394
+ return combined;
395
+ }
396
+
397
+ function addStats(target, parts) {
398
+ for (const part of parts) {
399
+ for (const [key, value] of Object.entries(part)) {
400
+ if (typeof value !== "number") {
401
+ continue;
402
+ }
403
+ target[key] =
404
+ key === "peakConcurrency"
405
+ ? Math.max(target[key] || 0, value)
406
+ : (target[key] || 0) + value;
407
+ }
408
+ }
409
+ return target;
410
+ }
411
+
412
+ /**
413
+ * Run one chunk through `cdxrs fetch`.
414
+ *
415
+ * @param {Array<{url: string, accept?: string, authRealm?: string}>} chunk
416
+ * @param {Object} options Options with `timeoutMs`.
417
+ * @returns {Promise<{results?: Map<string, BatchEntry>, stats?: Object, reason?: string}>}
418
+ * `results` when the run succeeded; otherwise the reason it did not.
419
+ */
420
+ async function rustFetchChunk(chunk, options) {
276
421
  const payload = { requests: [] };
277
- for (const request of unique) {
422
+ for (const request of chunk) {
278
423
  payload.requests.push({
279
424
  id: request.url,
280
425
  url: request.url,
@@ -291,20 +436,19 @@ async function rustBatchFetch(unique, options) {
291
436
  if (!ok) {
292
437
  if (DEBUG_MODE && reason !== "disabled" && reason !== "binary-not-found") {
293
438
  console.log(
294
- `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
439
+ reason === "stdout-too-large" && chunk.length > 1
440
+ ? `cdxrs fetch output for ${chunk.length} url(s) is too large; retrying in two halves.`
441
+ : `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
295
442
  );
296
443
  }
297
- // The Rust path failed in a way that is not "binary absent". Fall through
298
- // to the JS pool rather than returning an empty map, so a transient cdxrs
299
- // crash does not take the whole batch's concurrency with it.
300
- return await jsBatchFetch(unique);
444
+ return { reason };
301
445
  }
302
446
 
303
447
  let envelope;
304
448
  try {
305
449
  envelope = JSON.parse(stdout);
306
450
  } catch (_err) {
307
- return await jsBatchFetch(unique);
451
+ return { reason: "malformed-envelope" };
308
452
  }
309
453
  if (envelope?.schemaVersion !== BATCH_SCHEMA_VERSION) {
310
454
  // A binary from a different generation of this protocol is not something
@@ -314,10 +458,9 @@ async function rustBatchFetch(unique, options) {
314
458
  `cdxrs fetch envelope version ${envelope?.schemaVersion} != ${BATCH_SCHEMA_VERSION}; falling back to the JS batch pool.`,
315
459
  );
316
460
  }
317
- return await jsBatchFetch(unique);
461
+ return { reason: "version-mismatch" };
318
462
  }
319
463
 
320
- _lastStats = envelope.stats || null;
321
464
  const results = new Map();
322
465
  for (const result of envelope.results || []) {
323
466
  if (!result?.id) {
@@ -335,14 +478,7 @@ async function rustBatchFetch(unique, options) {
335
478
  const definite = status >= 400 && status < 500 && status !== 429;
336
479
  results.set(result.id, { ok: false, status, definite });
337
480
  }
338
-
339
- if (DEBUG_MODE && envelope.stats) {
340
- const s = envelope.stats;
341
- console.log(
342
- `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
343
- );
344
- }
345
- return results;
481
+ return { results, stats: envelope.stats || null };
346
482
  }
347
483
 
348
484
  /**
@@ -700,6 +836,41 @@ async function fetchOne(request, gauge) {
700
836
  }
701
837
  }
702
838
 
839
+ /**
840
+ * Run a caller's own request under the same per-host gate the batch pool uses.
841
+ *
842
+ * Not every registry lookup goes through the pool: a caller that misses the
843
+ * prefetch, or runs with prefetching disabled, issues `cdxgenAgent.get`
844
+ * directly. Those requests reach the same host and count against the same
845
+ * published budget, so they have to queue behind the same limiter rather than
846
+ * a second one — crates.io's one-request-per-second policy is not honoured by
847
+ * two independent gates that each allow one per second.
848
+ *
849
+ * The per-host semaphore and limiter are module state, so this shares one gate
850
+ * per host for the life of the process.
851
+ *
852
+ * @template T
853
+ * @param {string} url The URL about to be requested.
854
+ * @param {() => Promise<T>} issue Issues the request and resolves its result.
855
+ * @returns {Promise<T>} Whatever `issue` resolves to.
856
+ */
857
+ export async function withHostRateLimit(url, issue) {
858
+ const host = extractHost(url);
859
+ if (!host) {
860
+ return await issue();
861
+ }
862
+ const controls = getHostControls(host);
863
+ const releaseGlobal = await _globalSemaphore.acquire();
864
+ const releaseHost = await controls.sem.acquire();
865
+ try {
866
+ await controls.limiter.wait();
867
+ return await issue();
868
+ } finally {
869
+ releaseHost();
870
+ releaseGlobal();
871
+ }
872
+ }
873
+
703
874
  /**
704
875
  * Dedupe key for an in-flight request: the URL, plus everything that changes
705
876
  * which response the caller gets back. That is the headers which change what
@@ -2,18 +2,21 @@
2
2
  * Per-host rate policy for batched registry fetches.
3
3
  *
4
4
  * This is the single source of truth for the JS transport's concurrency and
5
- * minimum-interval decisions. The numbers are shared with the Rust transport
5
+ * minimum-interval decisions. Every number is shared with the Rust transport
6
6
  * (`cdxgen-plugins-bin/thirdparty/cdxrs/src/fetch/rate.rs`); a change to one
7
7
  * side must be reflected in the other, and `fetchRate.poku.js` pins them so a
8
- * drift is visible.
8
+ * drift is visible. crates.io and package.elm-lang.org are routed so that their
9
+ * requests never reach the Rust transport, which makes this table the one that
10
+ * governs them in practice — their rows are mirrored there regardless, because
11
+ * two tables that disagree are a bug waiting for the routing to change.
9
12
  *
10
13
  * Why a per-host table rather than a single global interval: a single-registry
11
14
  * project — which is to say, almost every npm project — has exactly one host,
12
15
  * so a conservative per-host cap *is* the global cap. Applying crates.io's
13
- * 250 ms to every host caps an npm run at 4 requests per second, which is
14
- * slower than the serial JS path once connection reuse is taken into account.
15
- * Hosts that publish a limit get an interval; hosts that do not are bounded by
16
- * the per-host concurrency cap alone, and by whatever `Retry-After` they send.
16
+ * one-per-second policy to every host would cap an npm run at one request a
17
+ * second, orders of magnitude slower than that registry asks for. Hosts that
18
+ * publish a limit get an interval; hosts that do not are bounded by the
19
+ * per-host concurrency cap alone, and by whatever `Retry-After` they send.
17
20
  */
18
21
 
19
22
  import { readEnvironmentVariable } from "../core/activity.js";
@@ -48,13 +51,18 @@ import { readEnvironmentVariable } from "../core/activity.js";
48
51
  * Suffix match is used so `index.crates.io` inherits `crates.io`.
49
52
  */
50
53
  const HOST_POLICIES = [
51
- // crates.io's published budget is per IP and does not improve with a token.
54
+ // crates.io's crawler policy asks for at most one request per second, made
55
+ // serially. It is a condition of use rather than a throughput hint, so the
56
+ // interval is a full second and the concurrency is 1: with more than one
57
+ // request in flight the gate would still average a second apiece but would
58
+ // deliver them in bursts, which is the thing the policy rules out. The
59
+ // budget is per IP and does not improve with a token.
52
60
  {
53
61
  host: "crates.io",
54
- anonInterval: 250,
55
- anonConcurrency: 4,
56
- authInterval: 250,
57
- authConcurrency: 4,
62
+ anonInterval: 1000,
63
+ anonConcurrency: 1,
64
+ authInterval: 1000,
65
+ authConcurrency: 1,
58
66
  },
59
67
  // GitHub: 60 req/h anonymous, 5000 req/h authenticated. The limit is an
60
68
  // hourly budget rather than a rate, so a batch of a few hundred lookups is
@@ -88,6 +96,61 @@ const HOST_POLICIES = [
88
96
  authInterval: 250,
89
97
  authConcurrency: 4,
90
98
  },
99
+ // package.elm-lang.org publishes no limit: its robots.txt carries no
100
+ // `Crawl-delay` and the registry has no terms of use or documented API. What
101
+ // it does have is a single community-run origin rather than a CDN, so it is
102
+ // held to the same gentle allowance as the other unpublished host in this
103
+ // table instead of the CDN default. Like crates.io, its requests are kept off
104
+ // the Rust transport to leave one gate in charge; rate.rs still carries the
105
+ // same row so the two tables state one policy.
106
+ // Maven Central publishes no numeric limit. What it publishes instead is a
107
+ // policy: consumption limits are enforced against aggregate traffic, the
108
+ // thresholds are unpublished and moving downward, and the named offenders are
109
+ // "CI/CD platforms, security scanners, and other services that generate
110
+ // outsized traffic" — which is cdxgen's exact job description. A block is an
111
+ // escalating ban of up to 30 days, not a per-request 429, so the cost of
112
+ // being slightly too fast is losing Maven resolution for a month rather than
113
+ // retrying a second later. That asymmetry, not a published number, is what
114
+ // sets these rows.
115
+ //
116
+ // repo1 is the artifact store: Cloudflare-fronted (`cf-cache-status` on every
117
+ // response), and a POM fetch that hits the edge cache never reaches Sonatype.
118
+ // It gets a real allowance — 100 ms and 8 in flight, so ~10 req/s sustained —
119
+ // which is half the CDN default and orders of magnitude below anything that
120
+ // would show up in the top few percent of Central's traffic.
121
+ {
122
+ host: "repo1.maven.org",
123
+ anonInterval: 100,
124
+ anonConcurrency: 8,
125
+ authInterval: 100,
126
+ authConcurrency: 8,
127
+ },
128
+ // The search API is the opposite: a Solr query against the origin, one
129
+ // request per unknown jar hash, cacheable by nobody. cdxgen already treats it
130
+ // as fragile — a single timeout or 429 disables jar search for the rest of
131
+ // the run — so the gate is a serial 500 ms (~2 req/s). A jar sweep stays
132
+ // usable while staying far below anything worth throttling.
133
+ {
134
+ host: "search.maven.org",
135
+ anonInterval: 500,
136
+ anonConcurrency: 1,
137
+ authInterval: 500,
138
+ authConcurrency: 1,
139
+ },
140
+ {
141
+ host: "central.sonatype.com",
142
+ anonInterval: 500,
143
+ anonConcurrency: 1,
144
+ authInterval: 500,
145
+ authConcurrency: 1,
146
+ },
147
+ {
148
+ host: "package.elm-lang.org",
149
+ anonInterval: 250,
150
+ anonConcurrency: 4,
151
+ authInterval: 250,
152
+ authConcurrency: 4,
153
+ },
91
154
  ];
92
155
 
93
156
  /**
@@ -1,4 +1,5 @@
1
1
  import { cdxgenAgent, readEnvironmentVariable } from "../core/activity.js";
2
+ import { getUserAgent } from "../core/userAgent.js";
2
3
  import { getOriginUrl } from "./envcontext.js";
3
4
  import { prefetchJson } from "./fetchBatch.js";
4
5
 
@@ -123,7 +124,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
123
124
  const githubHeaders = {
124
125
  Accept: "application/vnd.github+json",
125
126
  Authorization: `Bearer ${githubToken}`,
126
- "User-Agent": "cdxgen-oversight",
127
+ "User-Agent": getUserAgent("api.github.com"),
127
128
  };
128
129
  const pullsUrlFor = (commit) =>
129
130
  `https://api.github.com/repos/${owner}/${repo}/commits/${commit.hash}/pulls`;
@@ -214,7 +215,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
214
215
  const commitsToQuery = aiCommits.slice(0, 5);
215
216
  const gitlabHeaders = {
216
217
  "PRIVATE-TOKEN": gitlabToken,
217
- "User-Agent": "cdxgen-oversight",
218
+ "User-Agent": getUserAgent("gitlab.com"),
218
219
  };
219
220
  const mrUrlFor = (commit) =>
220
221
  `https://gitlab.com/api/v4/projects/${projectEncoded}/repository/commits/${commit.hash}/merge_requests`;
@@ -45,10 +45,11 @@ const JVM_BUILD_TOOL_CANDIDATES = ["maven", "gradle", "sbt", "scala"];
45
45
 
46
46
  /**
47
47
  * Anchored validator for sdkman version identifiers such as `3.9.9`,
48
- * `4.0.0-rc-5`, `8.14`, or `21.0.7-tem`. Versions are interpolated into a
49
- * shell command by the sdkman installer, so every identifier must be a single
50
- * safe token: it must start with a digit and may only contain digits, ascii
51
- * letters, dots, and hyphens.
48
+ * `4.0.0-rc-5`, `8.14`, `21.0.7-tem`, or `26.0.2+1.1-tem`. Versions are
49
+ * interpolated into a shell command by the sdkman installer, so every
50
+ * identifier must be a single safe token: it must start with a digit and may
51
+ * only contain digits, ascii letters, dots, hyphens, and plus signs (`+` has
52
+ * no shell meaning and sdkman uses it in Temurin respin identifiers).
52
53
  *
53
54
  * Every separator must be followed by at least one alphanumeric character, and
54
55
  * a single repeated group covers both the dotted numeric part and the vendor
@@ -63,7 +64,7 @@ export function isValidSdkmanVersion(version) {
63
64
  typeof version === "string" &&
64
65
  version.length > 0 &&
65
66
  version.length <= MAX_VERSION_TOKEN_LENGTH &&
66
- /^\d+(?:[-.][0-9A-Za-z]+)*$/.test(version)
67
+ /^\d+(?:[-.+][0-9A-Za-z]+)*$/.test(version)
67
68
  );
68
69
  }
69
70