@cdxgen/cdxgen 13.4.0 → 13.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (305) hide show
  1. package/README.md +9 -9
  2. package/bin/cdxgen-secure.js +15 -0
  3. package/bin/cdxgen.js +29 -5
  4. package/bin/evinse.js +8 -0
  5. package/bin/hbom.js +15 -7
  6. package/bin/repl.js +29 -0
  7. package/bin/validate.js +6 -0
  8. package/data/cbomosdb-queries.json +13 -1
  9. package/data/component-tags.json +8 -0
  10. package/data/license-aliases.json +24 -2
  11. package/data/queries-darwin.json +25 -1
  12. package/data/queries-win.json +19 -1
  13. package/data/queries.json +41 -0
  14. package/data/remediations.json +407 -1
  15. package/data/rules/ai-bom.yaml +1 -1
  16. package/data/rules/brotli-decoder.yaml +107 -0
  17. package/data/rules/cbom-security.yaml +235 -4
  18. package/data/rules/go-binaries.yaml +46 -0
  19. package/data/rules/golem-go.yaml +80 -0
  20. package/data/rules/hbom-compliance.yaml +40 -22
  21. package/data/rules/hbom-performance.yaml +7 -1
  22. package/data/rules/host-topology.yaml +1 -1
  23. package/data/rules/obom-runtime.yaml +31 -2
  24. package/data/rules/php-runtime.yaml +143 -0
  25. package/data/rules/vscode-extensions.yaml +75 -0
  26. package/data/spdx-license-list.json +121 -7
  27. package/data/spdx-licenses.json +69 -0
  28. package/data/spdx.schema.json +845 -2
  29. package/lib/audit/index.js +14 -0
  30. package/lib/cli/bomAssembly.js +120 -6
  31. package/lib/cli/cliOptions.js +110 -3
  32. package/lib/cli/index.js +182 -6
  33. package/lib/cli/jsBom.js +153 -64
  34. package/lib/cli/jvmBom.js +363 -66
  35. package/lib/cli/managedBom.js +853 -87
  36. package/lib/cli/nativeBom.js +267 -33
  37. package/lib/core/boundedFile.js +344 -0
  38. package/lib/core/brotliProbe.js +182 -0
  39. package/lib/core/env.js +38 -8
  40. package/lib/core/fs.js +40 -19
  41. package/lib/core/httpClient.js +172 -46
  42. package/lib/core/lockPaths.js +38 -0
  43. package/lib/core/paths.js +24 -1
  44. package/lib/core/processTree.js +5 -0
  45. package/lib/core/projectConfig.js +1 -0
  46. package/lib/core/projectTooling.js +97 -0
  47. package/lib/core/propertySanitizer.js +72 -0
  48. package/lib/core/toolRegistry.js +170 -0
  49. package/lib/ecosystems/cppEvidence.js +263 -106
  50. package/lib/ecosystems/cppIncludeResolver.js +16 -5
  51. package/lib/ecosystems/ecosystems.js +25 -10
  52. package/lib/ecosystems/gemHostProbe.js +301 -0
  53. package/lib/ecosystems/golem.js +550 -6
  54. package/lib/ecosystems/gradleutils.js +858 -244
  55. package/lib/ecosystems/parsers-bazel.js +107 -6
  56. package/lib/ecosystems/parsers-dotnet.js +117 -18
  57. package/lib/ecosystems/parsers-go.js +1090 -367
  58. package/lib/ecosystems/parsers-js.js +505 -22
  59. package/lib/ecosystems/parsers-jvm.js +259 -17
  60. package/lib/ecosystems/parsers-misc.js +438 -64
  61. package/lib/ecosystems/parsers-python.js +100 -12
  62. package/lib/ecosystems/pythonutils.js +128 -2
  63. package/lib/ecosystems/rubyutils.js +544 -34
  64. package/lib/ecosystems/rusi.js +10 -0
  65. package/lib/ecosystems/scalacliutils.js +4 -1
  66. package/lib/ecosystems/scalasem.js +798 -151
  67. package/lib/ecosystems/vendoredCode.js +237 -3
  68. package/lib/ecosystems/venvProbe.js +360 -0
  69. package/lib/evinser/evinser.js +472 -61
  70. package/lib/evinser/swiftsem.js +68 -5
  71. package/lib/helpers/agenticToolutils.js +413 -93
  72. package/lib/helpers/vsixutils.js +1574 -315
  73. package/lib/inventory/agentSkillSpec.js +17 -0
  74. package/lib/inventory/apkInstalledDb.js +243 -0
  75. package/lib/inventory/atomUtils.js +110 -0
  76. package/lib/inventory/auditCategories.js +3 -3
  77. package/lib/inventory/brotliDecoder.js +142 -0
  78. package/lib/inventory/buildToolRateLimit.js +1 -1
  79. package/lib/inventory/cacheDir.js +52 -9
  80. package/lib/inventory/cbomutils.js +56 -7
  81. package/lib/inventory/certificates.js +2371 -0
  82. package/lib/inventory/communityAiConfigParser.js +4 -9
  83. package/lib/inventory/deps.js +41 -4
  84. package/lib/inventory/display.js +200 -3
  85. package/lib/inventory/dosai.js +639 -63
  86. package/lib/inventory/dotnetRuntime.js +217 -0
  87. package/lib/inventory/dynamic.js +45 -3
  88. package/lib/inventory/envcontext.js +475 -18
  89. package/lib/inventory/evidenceUtils.js +447 -23
  90. package/lib/inventory/fetchBatch.js +1 -1
  91. package/lib/inventory/formulationParsers.js +32 -3
  92. package/lib/inventory/goBuildInfo.js +140 -0
  93. package/lib/inventory/gtfobins.js +13 -0
  94. package/lib/inventory/hbom.js +67 -7
  95. package/lib/inventory/hbomAnalysis.js +16 -0
  96. package/lib/inventory/hostRuntimeProbe.js +686 -0
  97. package/lib/inventory/hostTopology.js +21 -0
  98. package/lib/inventory/jvmLocalRepos.js +9 -2
  99. package/lib/inventory/jvmToolEnv.js +29 -0
  100. package/lib/inventory/licenseEnhancer.js +12 -0
  101. package/lib/inventory/mavenCentralMirror.js +534 -0
  102. package/lib/inventory/nodeBuildInfo.js +218 -0
  103. package/lib/inventory/nodeHostProbe.js +202 -0
  104. package/lib/inventory/osPackagePurl.js +244 -0
  105. package/lib/inventory/osPackageResolver.js +271 -87
  106. package/lib/inventory/osinfo.js +79 -31
  107. package/lib/inventory/osqueryTransform.js +114 -5
  108. package/lib/inventory/phpBuildInfo.js +438 -0
  109. package/lib/inventory/phpHostProbe.js +185 -0
  110. package/lib/inventory/rootfsCerts.js +426 -0
  111. package/lib/inventory/rubyBuildInfo.js +972 -0
  112. package/lib/inventory/rubyHostProbe.js +218 -0
  113. package/lib/inventory/scalaCoords.js +4 -3
  114. package/lib/inventory/scalaDedupe.js +440 -0
  115. package/lib/inventory/source.js +256 -5
  116. package/lib/inventory/spdx.js +144 -0
  117. package/lib/inventory/traceRunner.js +5 -1
  118. package/lib/inventory/tracedCrypto.js +129 -0
  119. package/lib/managers/binary.js +866 -186
  120. package/lib/managers/docker.js +245 -12
  121. package/lib/parsers/email.js +277 -0
  122. package/lib/parsers/largeJson.js +62 -13
  123. package/lib/parsers/vscodeExtensionId.js +183 -0
  124. package/lib/server/server.js +198 -2
  125. package/lib/stages/postgen/annotator.js +17 -1
  126. package/lib/stages/postgen/introspection/reflect.js +4 -1
  127. package/lib/stages/postgen/postgen.js +5 -5
  128. package/lib/stages/postgen/ruleEngine.js +71 -6
  129. package/lib/stages/postgen/specVersionCompat.js +30 -0
  130. package/lib/stages/pregen/envAudit.js +1 -0
  131. package/lib/stages/pregen/pregen.js +231 -62
  132. package/lib/validator/bomValidator.js +32 -6
  133. package/lib/validator/generated/runtime.mjs +13 -0
  134. package/lib/validator/generated/validate-1.6.mjs +27 -8
  135. package/lib/validator/generated/validate-1.7.mjs +32 -13
  136. package/lib/validator/generated/validate-2.0.mjs +31 -12
  137. package/lib/validator/index.js +1 -1
  138. package/package.json +19 -18
  139. package/types/bin/cdxgen-secure.d.ts +3 -0
  140. package/types/bin/cdxgen-secure.d.ts.map +1 -0
  141. package/types/bin/repl.d.ts.map +1 -1
  142. package/types/lib/audit/index.d.ts.map +1 -1
  143. package/types/lib/cli/bomAssembly.d.ts +22 -3
  144. package/types/lib/cli/bomAssembly.d.ts.map +1 -1
  145. package/types/lib/cli/cliOptions.d.ts +43 -0
  146. package/types/lib/cli/cliOptions.d.ts.map +1 -1
  147. package/types/lib/cli/index.d.ts.map +1 -1
  148. package/types/lib/cli/jsBom.d.ts +1 -17
  149. package/types/lib/cli/jsBom.d.ts.map +1 -1
  150. package/types/lib/cli/jvmBom.d.ts +20 -0
  151. package/types/lib/cli/jvmBom.d.ts.map +1 -1
  152. package/types/lib/cli/managedBom.d.ts.map +1 -1
  153. package/types/lib/cli/nativeBom.d.ts.map +1 -1
  154. package/types/lib/core/boundedFile.d.ts +70 -0
  155. package/types/lib/core/boundedFile.d.ts.map +1 -0
  156. package/types/lib/core/brotliProbe.d.ts +37 -0
  157. package/types/lib/core/brotliProbe.d.ts.map +1 -0
  158. package/types/lib/core/env.d.ts +22 -2
  159. package/types/lib/core/env.d.ts.map +1 -1
  160. package/types/lib/core/fs.d.ts.map +1 -1
  161. package/types/lib/core/httpClient.d.ts +24 -0
  162. package/types/lib/core/httpClient.d.ts.map +1 -1
  163. package/types/lib/core/lockPaths.d.ts +19 -0
  164. package/types/lib/core/lockPaths.d.ts.map +1 -0
  165. package/types/lib/core/processTree.d.ts +2 -0
  166. package/types/lib/core/processTree.d.ts.map +1 -1
  167. package/types/lib/core/projectConfig.d.ts.map +1 -1
  168. package/types/lib/core/projectTooling.d.ts +41 -0
  169. package/types/lib/core/projectTooling.d.ts.map +1 -0
  170. package/types/lib/core/propertySanitizer.d.ts +26 -0
  171. package/types/lib/core/propertySanitizer.d.ts.map +1 -1
  172. package/types/lib/core/toolRegistry.d.ts +36 -0
  173. package/types/lib/core/toolRegistry.d.ts.map +1 -0
  174. package/types/lib/ecosystems/cppEvidence.d.ts +40 -13
  175. package/types/lib/ecosystems/cppEvidence.d.ts.map +1 -1
  176. package/types/lib/ecosystems/cppIncludeResolver.d.ts +2 -1
  177. package/types/lib/ecosystems/cppIncludeResolver.d.ts.map +1 -1
  178. package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
  179. package/types/lib/ecosystems/gemHostProbe.d.ts +28 -0
  180. package/types/lib/ecosystems/gemHostProbe.d.ts.map +1 -0
  181. package/types/lib/ecosystems/golem.d.ts +81 -1
  182. package/types/lib/ecosystems/golem.d.ts.map +1 -1
  183. package/types/lib/ecosystems/gradleutils.d.ts +57 -21
  184. package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
  185. package/types/lib/ecosystems/parsers-bazel.d.ts.map +1 -1
  186. package/types/lib/ecosystems/parsers-dotnet.d.ts +40 -4
  187. package/types/lib/ecosystems/parsers-dotnet.d.ts.map +1 -1
  188. package/types/lib/ecosystems/parsers-go.d.ts +85 -13
  189. package/types/lib/ecosystems/parsers-go.d.ts.map +1 -1
  190. package/types/lib/ecosystems/parsers-js.d.ts +17 -20
  191. package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
  192. package/types/lib/ecosystems/parsers-jvm.d.ts +28 -2
  193. package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
  194. package/types/lib/ecosystems/parsers-misc.d.ts +37 -58
  195. package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
  196. package/types/lib/ecosystems/parsers-python.d.ts +14 -3
  197. package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
  198. package/types/lib/ecosystems/pythonutils.d.ts +19 -1
  199. package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
  200. package/types/lib/ecosystems/rubyutils.d.ts +144 -12
  201. package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
  202. package/types/lib/ecosystems/rusi.d.ts.map +1 -1
  203. package/types/lib/ecosystems/scalacliutils.d.ts.map +1 -1
  204. package/types/lib/ecosystems/scalasem.d.ts.map +1 -1
  205. package/types/lib/ecosystems/vendoredCode.d.ts +26 -0
  206. package/types/lib/ecosystems/vendoredCode.d.ts.map +1 -1
  207. package/types/lib/ecosystems/venvProbe.d.ts +40 -0
  208. package/types/lib/ecosystems/venvProbe.d.ts.map +1 -0
  209. package/types/lib/evinser/evinser.d.ts +88 -8
  210. package/types/lib/evinser/evinser.d.ts.map +1 -1
  211. package/types/lib/evinser/swiftsem.d.ts +10 -0
  212. package/types/lib/evinser/swiftsem.d.ts.map +1 -1
  213. package/types/lib/inventory/agentSkillSpec.d.ts +13 -0
  214. package/types/lib/inventory/agentSkillSpec.d.ts.map +1 -0
  215. package/types/lib/inventory/apkInstalledDb.d.ts +42 -0
  216. package/types/lib/inventory/apkInstalledDb.d.ts.map +1 -0
  217. package/types/lib/inventory/atomUtils.d.ts +33 -0
  218. package/types/lib/inventory/atomUtils.d.ts.map +1 -1
  219. package/types/lib/inventory/brotliDecoder.d.ts +58 -0
  220. package/types/lib/inventory/brotliDecoder.d.ts.map +1 -0
  221. package/types/lib/inventory/cacheDir.d.ts +12 -4
  222. package/types/lib/inventory/cacheDir.d.ts.map +1 -1
  223. package/types/lib/inventory/cbomutils.d.ts +17 -0
  224. package/types/lib/inventory/cbomutils.d.ts.map +1 -1
  225. package/types/lib/inventory/certificates.d.ts +205 -0
  226. package/types/lib/inventory/certificates.d.ts.map +1 -0
  227. package/types/lib/inventory/communityAiConfigParser.d.ts.map +1 -1
  228. package/types/lib/inventory/deps.d.ts +2 -1
  229. package/types/lib/inventory/deps.d.ts.map +1 -1
  230. package/types/lib/inventory/display.d.ts +18 -0
  231. package/types/lib/inventory/display.d.ts.map +1 -1
  232. package/types/lib/inventory/dosai.d.ts +162 -7
  233. package/types/lib/inventory/dosai.d.ts.map +1 -1
  234. package/types/lib/inventory/dotnetRuntime.d.ts +44 -0
  235. package/types/lib/inventory/dotnetRuntime.d.ts.map +1 -0
  236. package/types/lib/inventory/dynamic.d.ts +6 -1
  237. package/types/lib/inventory/dynamic.d.ts.map +1 -1
  238. package/types/lib/inventory/envcontext.d.ts +92 -8
  239. package/types/lib/inventory/envcontext.d.ts.map +1 -1
  240. package/types/lib/inventory/evidenceUtils.d.ts +29 -10
  241. package/types/lib/inventory/evidenceUtils.d.ts.map +1 -1
  242. package/types/lib/inventory/formulationParsers.d.ts.map +1 -1
  243. package/types/lib/inventory/goBuildInfo.d.ts +10 -0
  244. package/types/lib/inventory/goBuildInfo.d.ts.map +1 -0
  245. package/types/lib/inventory/gtfobins.d.ts.map +1 -1
  246. package/types/lib/inventory/hbom.d.ts.map +1 -1
  247. package/types/lib/inventory/hbomAnalysis.d.ts +10 -0
  248. package/types/lib/inventory/hbomAnalysis.d.ts.map +1 -1
  249. package/types/lib/inventory/hostRuntimeProbe.d.ts +44 -0
  250. package/types/lib/inventory/hostRuntimeProbe.d.ts.map +1 -0
  251. package/types/lib/inventory/hostTopology.d.ts.map +1 -1
  252. package/types/lib/inventory/jvmToolEnv.d.ts +12 -0
  253. package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -1
  254. package/types/lib/inventory/licenseEnhancer.d.ts.map +1 -1
  255. package/types/lib/inventory/mavenCentralMirror.d.ts +170 -0
  256. package/types/lib/inventory/mavenCentralMirror.d.ts.map +1 -0
  257. package/types/lib/inventory/nodeBuildInfo.d.ts +95 -0
  258. package/types/lib/inventory/nodeBuildInfo.d.ts.map +1 -0
  259. package/types/lib/inventory/nodeHostProbe.d.ts +15 -0
  260. package/types/lib/inventory/nodeHostProbe.d.ts.map +1 -0
  261. package/types/lib/inventory/osPackagePurl.d.ts +99 -0
  262. package/types/lib/inventory/osPackagePurl.d.ts.map +1 -0
  263. package/types/lib/inventory/osPackageResolver.d.ts +44 -2
  264. package/types/lib/inventory/osPackageResolver.d.ts.map +1 -1
  265. package/types/lib/inventory/osinfo.d.ts +31 -6
  266. package/types/lib/inventory/osinfo.d.ts.map +1 -1
  267. package/types/lib/inventory/osqueryTransform.d.ts +51 -0
  268. package/types/lib/inventory/osqueryTransform.d.ts.map +1 -1
  269. package/types/lib/inventory/phpBuildInfo.d.ts +142 -0
  270. package/types/lib/inventory/phpBuildInfo.d.ts.map +1 -0
  271. package/types/lib/inventory/phpHostProbe.d.ts +35 -0
  272. package/types/lib/inventory/phpHostProbe.d.ts.map +1 -0
  273. package/types/lib/inventory/rootfsCerts.d.ts +27 -0
  274. package/types/lib/inventory/rootfsCerts.d.ts.map +1 -0
  275. package/types/lib/inventory/rubyBuildInfo.d.ts +271 -0
  276. package/types/lib/inventory/rubyBuildInfo.d.ts.map +1 -0
  277. package/types/lib/inventory/rubyHostProbe.d.ts +47 -0
  278. package/types/lib/inventory/rubyHostProbe.d.ts.map +1 -0
  279. package/types/lib/inventory/source.d.ts +38 -0
  280. package/types/lib/inventory/source.d.ts.map +1 -1
  281. package/types/lib/inventory/spdx.d.ts +15 -0
  282. package/types/lib/inventory/spdx.d.ts.map +1 -1
  283. package/types/lib/inventory/traceRunner.d.ts +4 -1
  284. package/types/lib/inventory/traceRunner.d.ts.map +1 -1
  285. package/types/lib/inventory/tracedCrypto.d.ts +22 -0
  286. package/types/lib/inventory/tracedCrypto.d.ts.map +1 -1
  287. package/types/lib/managers/binary.d.ts +10 -0
  288. package/types/lib/managers/binary.d.ts.map +1 -1
  289. package/types/lib/managers/docker.d.ts +24 -0
  290. package/types/lib/managers/docker.d.ts.map +1 -1
  291. package/types/lib/parsers/email.d.ts +19 -0
  292. package/types/lib/parsers/email.d.ts.map +1 -0
  293. package/types/lib/parsers/largeJson.d.ts +17 -3
  294. package/types/lib/parsers/largeJson.d.ts.map +1 -1
  295. package/types/lib/server/server.d.ts +8 -0
  296. package/types/lib/server/server.d.ts.map +1 -1
  297. package/types/lib/stages/postgen/annotator.d.ts.map +1 -1
  298. package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -1
  299. package/types/lib/stages/postgen/ruleEngine.d.ts +7 -4
  300. package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
  301. package/types/lib/stages/postgen/specVersionCompat.d.ts.map +1 -1
  302. package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
  303. package/types/lib/stages/pregen/pregen.d.ts +26 -9
  304. package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
  305. package/types/lib/validator/bomValidator.d.ts.map +1 -1
@@ -1,9 +1,10 @@
1
1
  import { Buffer } from "node:buffer";
2
- import { readdirSync, readFileSync } from "node:fs";
2
+ import { readdirSync, readFileSync, realpathSync } from "node:fs";
3
3
  import { homedir } from "node:os";
4
- import { delimiter, dirname, join } from "node:path";
4
+ import { delimiter, dirname, join, resolve } from "node:path";
5
5
 
6
6
  import { Purl } from "@cdxgen/cdx-purl";
7
+ import { compare as compareSemver } from "semver";
7
8
 
8
9
  import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
9
10
  import { shouldFetchLicense, shouldFetchVCS } from "../core/env.js";
@@ -13,7 +14,7 @@ import {
13
14
  tryBuildPurl,
14
15
  tryParsePurl,
15
16
  } from "../inventory/purl.js";
16
- import { guessLicenseId } from "../inventory/spdx.js";
17
+ import { guessLicenseIds, isSpdxLicenseExpression } from "../inventory/spdx.js";
17
18
  import {
18
19
  getGoPkgLicense,
19
20
  getGoPkgVCSUrl,
@@ -49,17 +50,18 @@ function goModuleCacheDir() {
49
50
  }
50
51
 
51
52
  /**
52
- * Escape a module path the way Go's module cache directory names do: every
53
- * upper-case letter becomes `!` followed by the lower-case one, so the cache
54
- * stays case-preserving on case-insensitive file systems
55
- * (`github.com/Azure` is stored as `github.com/!azure`).
53
+ * Escape a module path or version the way Go's module cache directory names
54
+ * do (module.EscapePath and module.EscapeVersion): every upper-case letter
55
+ * becomes `!` followed by the lower-case one, so the cache stays
56
+ * case-preserving on case-insensitive file systems (`github.com/Azure` is
57
+ * stored as `github.com/!azure`, and `v2.10.17-RC.1` as `v2.10.17-!r!c.1`).
56
58
  *
57
- * @param {string} modulePath Full module path
58
- * @returns {string} The escaped path
59
+ * @param {string} value Module path or version
60
+ * @returns {string} The escaped value
59
61
  */
60
- function escapeGoModulePath(modulePath) {
62
+ function escapeGoCacheName(value) {
61
63
  let escaped = "";
62
- for (const ch of modulePath) {
64
+ for (const ch of value) {
63
65
  if (ch >= "A" && ch <= "Z") {
64
66
  escaped += `!${ch.toLowerCase()}`;
65
67
  } else {
@@ -69,122 +71,452 @@ function escapeGoModulePath(modulePath) {
69
71
  return escaped;
70
72
  }
71
73
 
74
+ // Path elements Windows reserves, which Go rejects in every module path.
75
+ const GO_WINDOWS_RESERVED_NAMES = new Set([
76
+ "CON",
77
+ "PRN",
78
+ "AUX",
79
+ "NUL",
80
+ ..."123456789".split("").flatMap((n) => [`COM${n}`, `LPT${n}`]),
81
+ ]);
82
+ // ASCII punctuation a file name element may hold, after module.fileNameOK.
83
+ const GO_FILE_NAME_PUNCTUATION = "!#$%&()+,-.=@[]^_{}~ ";
84
+
85
+ /**
86
+ * Whether one character may appear in a path element of the given kind, as
87
+ * module.modPathOK, module.importPathOK and module.fileNameOK decide.
88
+ *
89
+ * @param {string} ch One character
90
+ * @param {"module"|"import"|"file"} kind What the element belongs to
91
+ * @returns {boolean}
92
+ */
93
+ function isGoPathCharOK(ch, kind) {
94
+ if (/[A-Za-z0-9]/.test(ch)) {
95
+ return true;
96
+ }
97
+ if (kind === "file") {
98
+ return (
99
+ GO_FILE_NAME_PUNCTUATION.includes(ch) ||
100
+ (ch.charCodeAt(0) >= 0x80 && /\p{L}/u.test(ch))
101
+ );
102
+ }
103
+ return "-._~".includes(ch) || (kind === "import" && ch === "+");
104
+ }
105
+
106
+ /**
107
+ * Whether a path element is one the Go toolchain accepts, as module.checkElem
108
+ * decides. No accepted element is `.` or `..`, or holds a path separator, so a
109
+ * name that passes cannot leave the directory it is joined to.
110
+ *
111
+ * @param {string} elem Path element
112
+ * @param {"module"|"import"|"file"} kind What the element belongs to
113
+ * @returns {boolean}
114
+ */
115
+ function isGoPathElementOK(elem, kind) {
116
+ if (!elem || /^\.+$/.test(elem) || elem.endsWith(".")) {
117
+ return false;
118
+ }
119
+ if (kind === "module" && elem.startsWith(".")) {
120
+ return false;
121
+ }
122
+ for (const ch of elem) {
123
+ if (!isGoPathCharOK(ch, kind)) {
124
+ return false;
125
+ }
126
+ }
127
+ const dot = elem.indexOf(".");
128
+ const short = dot < 0 ? elem : elem.slice(0, dot);
129
+ if (GO_WINDOWS_RESERVED_NAMES.has(short.toUpperCase())) {
130
+ return false;
131
+ }
132
+ if (kind === "file") {
133
+ return true;
134
+ }
135
+ // Names that look like Windows short names, such as PROGRA~1.
136
+ const tilde = short.lastIndexOf("~");
137
+ return !(tilde >= 0 && /^\d+$/.test(short.slice(tilde + 1)));
138
+ }
139
+
140
+ /**
141
+ * Whether a slash-separated path is one the Go toolchain accepts, as
142
+ * module.checkPath decides.
143
+ *
144
+ * @param {string} path Module or import path
145
+ * @param {"module"|"import"} kind What the path is
146
+ * @returns {boolean}
147
+ */
148
+ function isGoPathOK(path, kind) {
149
+ return (
150
+ typeof path === "string" &&
151
+ path.length > 0 &&
152
+ !path.startsWith("-") &&
153
+ path.split("/").every((elem) => isGoPathElementOK(elem, kind))
154
+ );
155
+ }
156
+
157
+ /**
158
+ * Whether a module path passes module.CheckPath: a valid path whose first
159
+ * element is a lower-case host name with a dot in it. Go writes no module
160
+ * cache directory for any other path.
161
+ *
162
+ * @param {string} path Module path
163
+ * @returns {boolean}
164
+ */
165
+ function isValidGoModulePath(path) {
166
+ if (!isGoPathOK(path, "module")) {
167
+ return false;
168
+ }
169
+ const host = path.split("/")[0];
170
+ return host.includes(".") && /^[a-z0-9.-]+$/.test(host);
171
+ }
172
+
173
+ /**
174
+ * Whether a version passes module.EscapeVersion: a single valid file name
175
+ * element without `!`.
176
+ *
177
+ * @param {string} version Module version
178
+ * @returns {boolean}
179
+ */
180
+ function isValidGoVersion(version) {
181
+ return (
182
+ typeof version === "string" &&
183
+ !version.includes("!") &&
184
+ isGoPathElementOK(version, "file")
185
+ );
186
+ }
187
+
188
+ // The licence notice file names pkg.go.dev reads, compared without regard to
189
+ // case (golang/pkgsite internal/licenses FileNames).
190
+ const GO_LICENSE_FILE_NAMES = new Set(
191
+ [
192
+ "COPYING",
193
+ "COPYING.md",
194
+ "COPYING.markdown",
195
+ "COPYING.txt",
196
+ "COPYING.LESSER",
197
+ "COPYING.LESSER.md",
198
+ "COPYING.LESSER.markdown",
199
+ "COPYING.LESSER.txt",
200
+ "LICENCE",
201
+ "LICENCE.md",
202
+ "LICENCE.markdown",
203
+ "LICENCE.txt",
204
+ "LICENSE",
205
+ "LICENSE.md",
206
+ "LICENSE.markdown",
207
+ "LICENSE.txt",
208
+ "LICENSE-2.0.txt",
209
+ "LICENCE-2.0.txt",
210
+ "LICENSE-APACHE",
211
+ "LICENCE-APACHE",
212
+ "LICENSE-APACHE-2.0.txt",
213
+ "LICENCE-APACHE-2.0.txt",
214
+ "LICENSE-MIT",
215
+ "LICENCE-MIT",
216
+ "LICENSE.MIT",
217
+ "LICENCE.MIT",
218
+ "LICENSE.code",
219
+ "LICENCE.code",
220
+ "LICENSE.docs",
221
+ "LICENCE.docs",
222
+ "LICENSE.rst",
223
+ "LICENCE.rst",
224
+ "MIT-LICENSE",
225
+ "MIT-LICENCE",
226
+ "MIT-LICENSE.md",
227
+ "MIT-LICENCE.md",
228
+ "MIT-LICENSE.markdown",
229
+ "MIT-LICENCE.markdown",
230
+ "MIT-LICENSE.txt",
231
+ "MIT-LICENCE.txt",
232
+ "MIT_LICENSE",
233
+ "MIT_LICENCE",
234
+ "UNLICENSE",
235
+ "UNLICENCE",
236
+ ].map((f) => f.toLowerCase()),
237
+ );
238
+
72
239
  /**
73
- * Whether a file name is a licence or copying notice, whose text describes the
74
- * module's licence without a registry round.
240
+ * Whether a file name is one of the licence notices pkg.go.dev reads, whose
241
+ * text describes the module's licence without a registry round.
75
242
  *
76
243
  * @param {string} fileName Candidate file name
77
244
  * @returns {boolean}
78
245
  */
79
246
  function isGoLicenseFileName(fileName) {
80
- const lower = fileName.toLowerCase();
81
- return lower.startsWith("license") || lower.startsWith("copying");
247
+ return GO_LICENSE_FILE_NAMES.has(fileName.toLowerCase());
82
248
  }
83
249
 
84
250
  /**
85
- * The licence or copying notice of a module that is already on this machine,
86
- * from the directory `go list` reported, from the project's `vendor/` tree, or
87
- * from the module cache. Reading it costs no network round trip, so it answers
88
- * before pkg.go.dev is asked.
251
+ * The licence notices of a module that is already on this machine, from the
252
+ * directory `go list` reported, from the project's `vendor/` tree, or from the
253
+ * module cache. Reading them costs no network round trip, so they answer
254
+ * before pkg.go.dev is asked. Every notice of the first directory holding
255
+ * one is returned, as pkg.go.dev reads every notice of a module.
89
256
  *
90
257
  * The directory `go list` reported is always read. The vendor tree and the
91
258
  * module cache stand in for pkg.go.dev, so they are read only when licences
92
- * are being fetched.
259
+ * are being fetched. Their directories are built from a module path and
260
+ * version read from modules.txt or go.sum, so a path or version the Go
261
+ * toolchain would reject, such as one with a `..` element, is not looked up:
262
+ * Go never writes such a directory, and joining it could leave the tree.
93
263
  *
94
264
  * @param {string} name Full module path
95
265
  * @param {string} [version] Module version, for the module cache layout
96
266
  * @param {Object} [options] Options
97
267
  * @param {string} [options.dir] The module's directory, as `go list` reports it
98
268
  * @param {string} [options.projectRoot] Project root holding a `vendor/` tree
99
- * @returns {string|undefined} The notice text, or undefined when none is held
100
- * locally
269
+ * @param {string} [options.vendorPath] The path the module is vendored under,
270
+ * when it differs from `name`: Go vendors a replaced module under the path
271
+ * it replaces
272
+ * @returns {{fileName: string, text: string}[]} The notices, in file name
273
+ * order; empty when none is held locally
101
274
  */
102
- export function readLocalGoLicense(name, version, options = {}) {
275
+ export function readLocalGoLicenses(name, version, options = {}) {
103
276
  if (!name) {
104
- return undefined;
277
+ return [];
105
278
  }
106
279
  const candidateDirs = [];
107
280
  if (options.dir) {
108
281
  candidateDirs.push(options.dir);
109
282
  }
110
283
  const fetchingLicenses = shouldFetchLicense();
111
- if (fetchingLicenses && options.projectRoot) {
112
- candidateDirs.push(join(options.projectRoot, "vendor", name));
284
+ const vendorPath = options.vendorPath || name;
285
+ if (
286
+ fetchingLicenses &&
287
+ options.projectRoot &&
288
+ isGoPathOK(vendorPath, "import")
289
+ ) {
290
+ candidateDirs.push(
291
+ join(options.projectRoot, "vendor", ...vendorPath.split("/")),
292
+ );
113
293
  }
114
- if (fetchingLicenses && version) {
294
+ if (
295
+ fetchingLicenses &&
296
+ version &&
297
+ isValidGoModulePath(name) &&
298
+ isValidGoVersion(version)
299
+ ) {
115
300
  const cacheRoot = goModuleCacheDir();
116
301
  if (cacheRoot) {
117
302
  candidateDirs.push(
118
- join(cacheRoot, `${escapeGoModulePath(name)}@${version}`),
303
+ join(
304
+ cacheRoot,
305
+ `${escapeGoCacheName(name)}@${escapeGoCacheName(version)}`,
306
+ ),
119
307
  );
120
308
  }
121
309
  }
122
310
  for (const dir of candidateDirs) {
123
311
  let entries;
124
312
  try {
125
- entries = readdirSync(dir);
313
+ entries = readdirSync(dir, { withFileTypes: true });
126
314
  } catch {
127
315
  continue;
128
316
  }
129
- const names = entries
130
- .filter((e) => isGoLicenseFileName(e))
317
+ // Only regular files: a module zip holds no links, so a link named like
318
+ // a notice did not come from the module.
319
+ const fileNames = entries
320
+ .filter((e) => e.isFile() && isGoLicenseFileName(e.name))
321
+ .map((e) => e.name)
131
322
  .sort((a, b) => a.localeCompare(b));
132
- // A directory can hold both a licence and a copying notice; the licence
133
- // notice wins when both are present.
134
- const chosen =
135
- names.find((f) => f.toLowerCase().startsWith("license")) ||
136
- names.find((f) => f.toLowerCase().startsWith("copying"));
137
- if (!chosen) {
323
+ const notices = [];
324
+ for (const fileName of fileNames) {
325
+ try {
326
+ const text = readFileSync(join(dir, fileName), {
327
+ encoding: "utf-8",
328
+ });
329
+ if (text?.length) {
330
+ notices.push({ fileName, text });
331
+ }
332
+ } catch {
333
+ // An unreadable notice is the same as none.
334
+ }
335
+ }
336
+ if (notices.length) {
337
+ return notices;
338
+ }
339
+ }
340
+ return [];
341
+ }
342
+
343
+ /**
344
+ * The text of the first licence notice {@link readLocalGoLicenses} finds.
345
+ *
346
+ * @param {string} name Full module path
347
+ * @param {string} [version] Module version, for the module cache layout
348
+ * @param {Object} [options] The options {@link readLocalGoLicenses} takes
349
+ * @returns {string|undefined} The notice text, or undefined when none is held
350
+ * locally
351
+ */
352
+ export function readLocalGoLicense(name, version, options = {}) {
353
+ return readLocalGoLicenses(name, version, options)[0]?.text;
354
+ }
355
+
356
+ /**
357
+ * Whether a registry can hold this module: a module path and version the Go
358
+ * toolchain accepts. Any other module was never published, and its path in a
359
+ * pkg.go.dev URL could resolve to a different module (`../x` would ask for
360
+ * `x`).
361
+ *
362
+ * @param {string} name Full module path
363
+ * @param {string} [version] Module version
364
+ * @returns {boolean}
365
+ */
366
+ function isGoRegistryModule(name, version) {
367
+ return isValidGoModulePath(name) && (!version || isValidGoVersion(version));
368
+ }
369
+
370
+ /**
371
+ * The modules the pkg.go.dev batch asks about, each marked when its licence
372
+ * notice is held on this machine, so the batch skips its licence tab but
373
+ * still fetches its module page for the repository URL. A module no registry
374
+ * holds (the main module, local code, or a path Go rejects) is left out, as
375
+ * getGoPkgComponent never looks it up.
376
+ *
377
+ * @param {Object[]} modules Modules with `name`, `version`,
378
+ * `skipRegistryLookups` and the options {@link readLocalGoLicenses} takes
379
+ * @returns {Object[]} Copies of the modules to ask about, with
380
+ * `licenseKnown` set
381
+ */
382
+ function goModulesToPrefetch(modules) {
383
+ return modules
384
+ .filter(
385
+ (m) => !m.skipRegistryLookups && isGoRegistryModule(m.name, m.version),
386
+ )
387
+ .map((m) => ({
388
+ ...m,
389
+ licenseKnown: readLocalGoLicenses(m.name, m.version, m).length > 0,
390
+ }));
391
+ }
392
+
393
+ /**
394
+ * The CycloneDX licences of notices read from disk. Every licence each notice
395
+ * grants is listed with the notice as its text, and a notice that names none
396
+ * is a CUSTOM licence. A licence with an exception is an SPDX expression,
397
+ * which CycloneDX only carries as the sole entry, so then every licence found
398
+ * is joined into one expression.
399
+ *
400
+ * @param {{text: string}[]} notices Notices read by {@link readLocalGoLicenses}
401
+ * @returns {Object[]} CycloneDX licence choices
402
+ */
403
+ function localGoLicenses(notices) {
404
+ const licenses = [];
405
+ const ids = [];
406
+ for (const { text } of notices) {
407
+ const textObject = { contentType: "text/plain", content: text };
408
+ const noticeIds = guessLicenseIds(text);
409
+ if (!noticeIds.length) {
410
+ licenses.push({ license: { name: "CUSTOM", text: textObject } });
138
411
  continue;
139
412
  }
140
- try {
141
- const text = readFileSync(join(dir, chosen), {
142
- encoding: "utf-8",
143
- });
144
- if (text?.length) {
145
- return text;
413
+ for (const id of noticeIds) {
414
+ if (ids.includes(id)) {
415
+ continue;
146
416
  }
147
- } catch {
148
- // An unreadable notice is the same as none: the registry path answers.
417
+ ids.push(id);
418
+ licenses.push({ license: { id, text: textObject } });
149
419
  }
150
420
  }
151
- return undefined;
421
+ if (ids.some((id) => isSpdxLicenseExpression(id))) {
422
+ return [
423
+ {
424
+ expression: ids
425
+ .map((id) => (ids.length > 1 && id.includes(" ") ? `(${id})` : id))
426
+ .join(" AND "),
427
+ },
428
+ ];
429
+ }
430
+ return licenses;
152
431
  }
153
432
 
154
433
  /**
155
- * Mark the modules whose licence notice is held on this machine, so the
156
- * pkg.go.dev batch skips their licence tab but still fetches their module page
157
- * for the repository URL.
434
+ * What a replace directive makes of one module, in the shape every Go parser
435
+ * reports it. The Go toolchain builds the replacement, so a module replacement
436
+ * is reported under the replacement's path and version, as govulncheck and
437
+ * syft report it. A directory replacement is local code that no registry
438
+ * holds: it keeps the module path it replaces and has no version, so its purl
439
+ * names no registry release.
158
440
  *
159
- * @param {Object[]} modules Modules with `name`, `version` and the options
160
- * {@link readLocalGoLicense} takes
161
- * @returns {Object[]} Copies of the modules with `licenseKnown` set
441
+ * @param {string} path The module path being replaced
442
+ * @param {string} [version] The version being replaced, absent for a
443
+ * replacement of every version
444
+ * @param {string} newPath The replacement module path or directory
445
+ * @param {string} [newVersion] The replacement version, absent (or `(devel)`
446
+ * in build info) for a directory
447
+ * @returns {Object|undefined} `name` and `version` to report, `local` for a
448
+ * directory, `localDir` the directory as written, and `replaces` the
449
+ * `path@version` it stands in for
162
450
  */
163
- function withLocalLicenseFlags(modules) {
164
- return modules.map((m) => ({
165
- ...m,
166
- licenseKnown: !!readLocalGoLicense(m.name, m.version, m),
167
- }));
451
+ function goReplacement(path, version, newPath, newVersion) {
452
+ if (!path || !newPath) {
453
+ return undefined;
454
+ }
455
+ const replaces = version ? `${path}@${version}` : path;
456
+ if (!newVersion || newVersion === "(devel)") {
457
+ return {
458
+ name: path,
459
+ version: undefined,
460
+ local: true,
461
+ localDir: newPath,
462
+ replaces,
463
+ };
464
+ }
465
+ return { name: newPath, version: newVersion, local: false, replaces };
168
466
  }
169
467
 
170
468
  /**
171
- * The licence object for a notice read from disk: its SPDX id when the text
172
- * names one, else a CUSTOM licence, with the notice as its text either way.
469
+ * Record on a component the module a replace directive put it in place of.
173
470
  *
174
- * @param {string} text Notice text
175
- * @returns {Object} CycloneDX licence object
471
+ * @param {Object} component Component built for the replacement
472
+ * @param {Object} [replacement] Result of {@link goReplacement}
473
+ * @returns {Object} The same component
176
474
  */
177
- function localGoLicenseObject(text) {
178
- const licenseId = guessLicenseId(text);
179
- const identity = !licenseId
180
- ? { name: "CUSTOM" }
181
- : licenseId.includes(" ")
182
- ? { name: licenseId }
183
- : { id: licenseId };
184
- return {
185
- ...identity,
186
- text: { contentType: "text/plain", content: text },
187
- };
475
+ function addGoReplacementProperties(component, replacement) {
476
+ if (!replacement) {
477
+ return component;
478
+ }
479
+ component.properties = component.properties || [];
480
+ component.properties.push({
481
+ name: "cdx:go:replaces",
482
+ value: replacement.replaces,
483
+ });
484
+ if (replacement.localDir) {
485
+ component.properties.push({
486
+ name: "cdx:go:local_dir",
487
+ value: replacement.localDir,
488
+ });
489
+ }
490
+ return component;
491
+ }
492
+
493
+ /**
494
+ * Order two Go module versions by semantic version precedence, as Go's semver
495
+ * package does: a pre-release, pseudo-versions included, sorts before its
496
+ * release, and build metadata such as `+incompatible` does not count.
497
+ *
498
+ * @param {string} a A module version
499
+ * @param {string} b Another module version
500
+ * @returns {number} Negative, zero or positive as `a` sorts before, with or
501
+ * after `b`
502
+ */
503
+ export function compareGoVersions(a, b) {
504
+ try {
505
+ return compareSemver(a, b);
506
+ } catch {
507
+ return (a || "").localeCompare(b || "");
508
+ }
509
+ }
510
+
511
+ /**
512
+ * Convert a Go checksum (`h1:` followed by the base64 SHA-256) to the
513
+ * integrity string components carry.
514
+ *
515
+ * @param {string} [sum] Go checksum
516
+ * @returns {string|undefined} `sha256-…`, or undefined without a checksum
517
+ */
518
+ function goSumIntegrity(sum) {
519
+ return sum?.startsWith("h1:") ? `sha256-${sum.slice(3)}` : undefined;
188
520
  }
189
521
 
190
522
  /**
@@ -203,6 +535,8 @@ function localGoLicenseObject(text) {
203
535
  * it, where a local licence notice is read first
204
536
  * @param {string} [options.projectRoot] Project root holding a `vendor/` tree
205
537
  * the module may be vendored in
538
+ * @param {string} [options.vendorPath] The path the module is vendored under,
539
+ * when it differs from `name`
206
540
  * @returns {Promise<Object>} Component object ready for inclusion in a BOM package list
207
541
  */
208
542
  export async function getGoPkgComponent(
@@ -214,12 +548,15 @@ export async function getGoPkgComponent(
214
548
  ) {
215
549
  let license;
216
550
  let licenses;
217
- // A notice held on this machine answers before pkg.go.dev is asked, and the
551
+ const registryLookups =
552
+ !options.skipRegistryLookups &&
553
+ isGoRegistryModule(group ? `${group}/${name}` : name, version);
554
+ // Notices held on this machine answer before pkg.go.dev is asked, and the
218
555
  // registry is consulted only when there is none.
219
- const localLicense = readLocalGoLicense(name, version, options);
220
- if (localLicense) {
221
- licenses = [{ license: localGoLicenseObject(localLicense) }];
222
- } else if (shouldFetchLicense() && !options.skipRegistryLookups) {
556
+ const notices = readLocalGoLicenses(name, version, options);
557
+ if (notices.length) {
558
+ licenses = localGoLicenses(notices);
559
+ } else if (shouldFetchLicense() && registryLookups) {
223
560
  if (DEBUG_MODE) {
224
561
  console.log(
225
562
  `About to fetch go package license information for ${group}:${name}`,
@@ -254,7 +591,7 @@ export async function getGoPkgComponent(
254
591
  version: version || null,
255
592
  })?.replace(/%2F/g, "/") || null;
256
593
  let vcs;
257
- if (shouldFetchVCS() && !options.skipRegistryLookups) {
594
+ if (shouldFetchVCS() && registryLookups) {
258
595
  vcs = await getGoPkgVCSUrl(group, name, version);
259
596
  }
260
597
  const packageInfo = {
@@ -280,7 +617,8 @@ export async function getGoPkgComponent(
280
617
  * @param {String} goModData Contents of go.mod file
281
618
  * @param {Object} gosumMap Data from go.sum files
282
619
  *
283
- * @returns {Object} Object containing parent component, rootList and packages list
620
+ * @returns {Object} Object containing parent component, rootList, packages
621
+ * list, and the replace directives as `replacements`
284
622
  */
285
623
  export async function parseGoModData(goModData, gosumMap) {
286
624
  const pkgComponentsList = [];
@@ -289,10 +627,13 @@ export async function parseGoModData(goModData, gosumMap) {
289
627
  if (!goModData) {
290
628
  return {};
291
629
  }
292
- const { parentComponent, entries } = decodeGoModData(goModData, gosumMap);
630
+ const { parentComponent, entries, replacements } = decodeGoModData(
631
+ goModData,
632
+ gosumMap,
633
+ );
293
634
  // A module whose licence notice is read locally is not asked for its
294
635
  // licence; getGoPkgComponent reads the same notice for each entry below.
295
- await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
636
+ await prefetchGoPkgMetadata(goModulesToPrefetch(entries));
296
637
  for (const entry of entries) {
297
638
  const component = await getGoPkgComponent(
298
639
  "",
@@ -322,6 +663,7 @@ export async function parseGoModData(goModData, gosumMap) {
322
663
  (a["bom-ref"] || "").localeCompare(b["bom-ref"] || ""),
323
664
  ),
324
665
  rootList,
666
+ replacements,
325
667
  };
326
668
  }
327
669
 
@@ -334,107 +676,251 @@ export async function parseGoModData(goModData, gosumMap) {
334
676
  *
335
677
  * @param {String} goModData Contents of go.mod file
336
678
  * @param {Object} gosumMap Data from go.sum files
337
- * @returns {{parentComponent: Object, entries: Array<Object>}} Parent component
338
- * and module entries in file order
679
+ * @returns {{parentComponent: Object, entries: Array<Object>, replacements: Array<Object>}}
680
+ * Parent component, module entries in file order, and every replace
681
+ * directive as `path`, `version`, `newPath` and `newVersion`
339
682
  */
340
683
  function decodeGoModData(goModData, gosumMap) {
341
684
  const parentComponent = {};
342
685
  const entries = [];
343
- let isModReplacement = false;
344
- let isTool = false;
345
-
346
- const pkgs = goModData.split("\n");
347
- for (let l of pkgs) {
348
- // Windows of course
349
- l = l.replaceAll("\r", "").replace(/[\t ]+/g, " ");
350
- // Capture the parent component name from the module
351
- if (l.startsWith("module ")) {
352
- parentComponent.name = l.split(" ").pop().trim();
353
- parentComponent.type = "application";
354
- // Single-segment module paths are valid golang purls as of cdx-purl 0.0.4.
355
- applyPurl(
356
- parentComponent,
357
- tryParsePurl(`pkg:golang/${parentComponent.name}`),
358
- );
359
- continue;
360
- }
686
+ const replacements = [];
687
+ // The verb of the parenthesised block being read: undefined outside a block,
688
+ // and "" inside one whose header is not a single verb, which Go rejects.
689
+ let blockVerb;
361
690
 
362
- // The `tool` block dependency relations will be recorded into `require` block(need run `go mod tidy`), just ignore that
363
- if (l.includes("tool (")) {
364
- isTool = !l.includes(")");
691
+ for (const line of goModData.split("\n")) {
692
+ const { tokens, comment } = tokenizeGoModLine(line);
693
+ if (!tokens.length) {
365
694
  continue;
366
695
  }
367
- if (l.includes(")")) {
368
- isTool = false;
696
+ let verb;
697
+ let args;
698
+ if (blockVerb !== undefined) {
699
+ if (tokens.length === 1 && tokens[0] === ")") {
700
+ blockVerb = undefined;
701
+ continue;
702
+ }
703
+ verb = blockVerb;
704
+ args = tokens;
705
+ } else if (tokens.at(-1) === "(") {
706
+ // `require (` opens a block whose lines all carry the verb.
707
+ blockVerb = tokens.length === 2 ? tokens[0] : "";
369
708
  continue;
370
- }
371
- if (l.includes("tool ") || isTool) {
372
- continue;
373
- }
374
- if (l.startsWith("toolchain ")) {
375
- const toolchainVer = l.split(" ").pop().trim();
376
- parentComponent.properties = [
377
- { name: "cdx:go:toolchain", value: toolchainVer },
378
- ];
709
+ } else if (tokens.at(-2) === "(" && tokens.at(-1) === ")") {
710
+ // `require ( )` is an empty block.
379
711
  continue;
712
+ } else {
713
+ verb = tokens[0];
714
+ args = tokens.slice(1);
380
715
  }
381
- // Skip go.mod file headers, whitespace, and/or comments
382
- if (
383
- l.startsWith("go ") ||
384
- //TODO: should toolchain be considered as a dependency
385
- l.includes(")") ||
386
- l.trim() === "" ||
387
- l.trim().startsWith("//")
388
- ) {
389
- continue;
716
+ switch (verb) {
717
+ case "module":
718
+ if (args.length) {
719
+ parentComponent.name = unquoteGoModToken(args[0]);
720
+ parentComponent.type = "application";
721
+ // Single-segment module paths are valid golang purls as of cdx-purl 0.0.4.
722
+ applyPurl(
723
+ parentComponent,
724
+ tryParsePurl(`pkg:golang/${parentComponent.name}`),
725
+ );
726
+ }
727
+ break;
728
+ case "toolchain":
729
+ if (args.length) {
730
+ parentComponent.properties = [
731
+ { name: "cdx:go:toolchain", value: args[0] },
732
+ ];
733
+ }
734
+ break;
735
+ case "require": {
736
+ // module/path v1.2.3
737
+ const name = unquoteGoModToken(args[0]);
738
+ const version = args[1];
739
+ if (!name || !version) {
740
+ break;
741
+ }
742
+ entries.push({
743
+ group: "",
744
+ name,
745
+ version,
746
+ hash: gosumMap[`${name}@${version}`],
747
+ indirect: isGoModIndirectComment(comment),
748
+ replacement: false,
749
+ });
750
+ break;
751
+ }
752
+ case "replace": {
753
+ // module/path [v1.2.3] => other/module v1.4.5
754
+ // module/path [v1.2.3] => ../local/directory
755
+ // Only the first form names a module version; a directory
756
+ // replacement leaves the required module as the component.
757
+ const arrow = args[1] === "=>" ? 1 : 2;
758
+ if (
759
+ args[arrow] !== "=>" ||
760
+ (args.length !== arrow + 2 && args.length !== arrow + 3)
761
+ ) {
762
+ break;
763
+ }
764
+ const oldPath = unquoteGoModToken(args[0]);
765
+ const name = unquoteGoModToken(args[arrow + 1]);
766
+ const version = args[arrow + 2];
767
+ if (oldPath && name) {
768
+ replacements.push({
769
+ path: oldPath,
770
+ version: arrow === 2 ? args[1] : undefined,
771
+ newPath: name,
772
+ newVersion: version,
773
+ });
774
+ }
775
+ if (!name || args.length !== arrow + 3) {
776
+ break;
777
+ }
778
+ entries.push({
779
+ group: "",
780
+ name,
781
+ version,
782
+ hash: gosumMap[`${name}@${version}`],
783
+ indirect: false,
784
+ replacement: true,
785
+ });
786
+ break;
787
+ }
788
+ // `go`, `godebug`, `exclude`, `retract`, `ignore` and `tool` name no
789
+ // dependency: an excluded or retracted version is kept out of the build,
790
+ // and a tool's module is listed under `require` once the module is tidy.
791
+ default:
792
+ break;
390
793
  }
794
+ }
795
+ return { parentComponent, entries, replacements };
796
+ }
391
797
 
392
- // Handle required modules separately from replacement modules to ensure accuracy when parsing component data.
393
- if (l.includes("require (")) {
394
- isModReplacement = false;
395
- continue;
396
- }
397
- if (l.includes("replace (")) {
398
- isModReplacement = true;
399
- continue;
400
- }
401
- if (l.includes("replace ")) {
402
- // If this is an inline replacement, drop the word replace
403
- // (eg; "replace google.golang.org/grpc => google.golang.org/grpc v1.21.0" becomes " google.golang.org/grpc => google.golang.org/grpc v1.21.0")
404
- l = l.replace("replace", "");
405
- isModReplacement = true;
406
- }
407
- // require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17
408
- if (l.startsWith("require ")) {
409
- l = l.replace("require ", "");
410
- isModReplacement = false;
411
- }
412
- const tmpA = l.trim().split(" ");
413
- if (!isModReplacement) {
414
- // Add group, name and version component properties for required modules
415
- const version = tmpA[1];
416
- entries.push({
417
- group: "",
418
- name: tmpA[0],
419
- version,
420
- hash: gosumMap[`${tmpA[0]}@${version}`],
421
- indirect: l.endsWith("// indirect"),
422
- replacement: false,
423
- });
798
+ // Characters the go.mod lexer always reads as a token of their own.
799
+ const GO_MOD_PUNCTUATION = "()[]{},";
800
+
801
+ /**
802
+ * Split one go.mod line into tokens the way the Go toolchain's modfile lexer
803
+ * does: whitespace separates tokens, `(`, `)`, `[`, `]`, `{`, `}` and `,`
804
+ * stand alone, a quoted string is one token, and `//` starts a comment that
805
+ * runs to the end of the line.
806
+ *
807
+ * @param {string} line A single line of a go.mod file
808
+ * @returns {{tokens: string[], comment: string}} The tokens, quoted strings
809
+ * still quoted, and the trailing comment text without its `//`
810
+ */
811
+ function tokenizeGoModLine(line) {
812
+ const tokens = [];
813
+ let i = 0;
814
+ while (i < line.length) {
815
+ const c = line[i];
816
+ if (/\s/.test(c)) {
817
+ i++;
818
+ } else if (line.startsWith("//", i)) {
819
+ return { tokens, comment: line.slice(i + 2).trim() };
820
+ } else if (GO_MOD_PUNCTUATION.includes(c)) {
821
+ tokens.push(c);
822
+ i++;
823
+ } else if (c === '"' || c === "`") {
824
+ let end = i + 1;
825
+ while (end < line.length && line[end] !== c) {
826
+ // A double-quoted string escapes its quote with a backslash.
827
+ end += c === '"' && line[end] === "\\" ? 2 : 1;
828
+ }
829
+ tokens.push(line.slice(i, end + 1));
830
+ i = end + 1;
424
831
  } else {
425
- // Add group, name and version component properties for replacement modules
426
- const version = tmpA[3];
427
- entries.push({
428
- group: "",
429
- name: tmpA[2],
430
- version,
431
- hash: gosumMap[`${tmpA[2]}@${version}`],
432
- indirect: false,
433
- replacement: true,
434
- });
832
+ let end = i;
833
+ while (
834
+ end < line.length &&
835
+ !/\s/.test(line[end]) &&
836
+ !GO_MOD_PUNCTUATION.includes(line[end]) &&
837
+ !line.startsWith("//", end)
838
+ ) {
839
+ end++;
840
+ }
841
+ tokens.push(line.slice(i, end));
842
+ i = end;
435
843
  }
436
844
  }
437
- return { parentComponent, entries };
845
+ return { tokens, comment: "" };
846
+ }
847
+
848
+ /**
849
+ * Read a module path or directory token, which go.mod allows in double
850
+ * quotes. Like the Go toolchain, any other quote character makes it invalid.
851
+ *
852
+ * @param {string|undefined} token A go.mod token
853
+ * @returns {string|undefined} The unquoted value, or undefined when invalid
854
+ */
855
+ function unquoteGoModToken(token) {
856
+ if (token?.startsWith('"')) {
857
+ return token.length >= 2 && token.endsWith('"')
858
+ ? token.slice(1, -1)
859
+ : undefined;
860
+ }
861
+ if (/["'`]/.test(token || "")) {
862
+ return undefined;
863
+ }
864
+ return token;
865
+ }
866
+
867
+ /**
868
+ * Whether a require line's comment marks it indirect, as the Go toolchain
869
+ * reads it: `// indirect`, or `// indirect; ` followed by a reason.
870
+ *
871
+ * @param {string} comment Comment text without its `//`
872
+ * @returns {boolean} True for an indirect requirement
873
+ */
874
+ function isGoModIndirectComment(comment) {
875
+ const fields = comment.split(/\s+/).filter(Boolean);
876
+ return (
877
+ (fields.length === 1 && fields[0] === "indirect") ||
878
+ (fields.length > 1 && fields[0] === "indirect;")
879
+ );
880
+ }
881
+
882
+ // A semantic version as golang.org/x/mod/semver accepts it, used to tell a
883
+ // module line's version from the `=>` of a replacement.
884
+ const GO_SEMVER_PATTERN =
885
+ /^v(0|[1-9]\d*)(\.(0|[1-9]\d*)){0,2}(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$/;
886
+
887
+ /**
888
+ * Read one `# ` module line of a vendor/modules.txt file the way the Go
889
+ * toolchain does (modload.readVendorList):
890
+ *
891
+ * # module/path v1.2.3
892
+ * # module/path v1.2.3 => other/module v1.4.5
893
+ * # module/path v1.2.3 => ../local/directory
894
+ * # module/path => other/module v1.4.5 (a replacement of every version)
895
+ *
896
+ * @param {string} line A line starting with `# `
897
+ * @returns {Object|undefined} `path`, `version` and, for a replaced module,
898
+ * `newPath` and `newVersion`; undefined for a line Go ignores
899
+ */
900
+ function parseModulesTxtModuleLine(line) {
901
+ const fields = line.split(/\s+/).filter(Boolean);
902
+ if (fields.length < 3) {
903
+ return undefined;
904
+ }
905
+ let rest;
906
+ const module = { path: fields[1] };
907
+ if (GO_SEMVER_PATTERN.test(fields[2])) {
908
+ module.version = fields[2];
909
+ rest = fields.slice(3);
910
+ } else if (fields[2] === "=>") {
911
+ rest = fields.slice(2);
912
+ } else {
913
+ return undefined;
914
+ }
915
+ if (rest[0] === "=>") {
916
+ if (rest.length === 2) {
917
+ module.newPath = rest[1];
918
+ } else if (rest.length === 3 && GO_SEMVER_PATTERN.test(rest[2])) {
919
+ module.newPath = rest[1];
920
+ module.newVersion = rest[2];
921
+ }
922
+ }
923
+ return module;
438
924
  }
439
925
 
440
926
  /**
@@ -442,6 +928,12 @@ function decodeGoModData(goModData, gosumMap) {
442
928
  * Go package components. Cross-references the go.sum map for integrity hashes and
443
929
  * sets scope and confidence based on hash availability.
444
930
  *
931
+ * Only the modules the vendor tree holds become components: a module line
932
+ * followed by its `##` annotations or its packages. `go mod vendor` also
933
+ * records every replace directive of go.mod at the end of the file, as a
934
+ * module line with nothing after it, so the consumer can tell that the
935
+ * replacement had no effect; those lines name no vendored module.
936
+ *
445
937
  * @param {string} txtFile Path to the modules.txt file
446
938
  * @param {Object} gosumMap Map of "module@version" keys to sha256 hash values from go.sum
447
939
  * @returns {Promise<Object[]>} List of Go package component objects with evidence
@@ -449,33 +941,66 @@ function decodeGoModData(goModData, gosumMap) {
449
941
  export async function parseGoModulesTxt(txtFile, gosumMap) {
450
942
  const pkgList = [];
451
943
  const txtData = readFileSync(txtFile, { encoding: "utf-8" });
452
- const pkgs = txtData
453
- .split("\n")
454
- .filter((p) => p.trim().replace(/["']/g, "").startsWith("# "));
455
944
  // A modules.txt file lives in the vendor/ tree, so the modules it lists are
456
945
  // vendored one directory up from it.
457
946
  const projectRoot = dirname(dirname(txtFile));
458
- const modules = pkgs.map((l) => {
459
- const tmpA = l.split(" ");
460
- return {
947
+ const records = [];
948
+ let current;
949
+ for (const rawLine of txtData.split("\n")) {
950
+ // A checkout with CRLF line endings must read the same as the LF file
951
+ // go mod vendor wrote.
952
+ const line = rawLine.replaceAll("\r", "");
953
+ if (line.startsWith("# ")) {
954
+ current = parseModulesTxtModuleLine(line);
955
+ if (current) {
956
+ records.push(current);
957
+ }
958
+ continue;
959
+ }
960
+ if (current && line.trim()) {
961
+ current.vendored = true;
962
+ }
963
+ }
964
+ const modules = [];
965
+ for (const record of records) {
966
+ // A replacement record of every version names no module version, and a
967
+ // record with nothing after it was not vendored.
968
+ if (!record.version || !record.vendored) {
969
+ continue;
970
+ }
971
+ const replacement = goReplacement(
972
+ record.path,
973
+ record.version,
974
+ record.newPath,
975
+ record.newVersion,
976
+ );
977
+ modules.push({
461
978
  group: "",
462
- name: tmpA[1],
463
- version: tmpA[2],
979
+ name: replacement?.name ?? record.path,
980
+ version: replacement ? replacement.version : record.version,
981
+ replacement,
464
982
  projectRoot,
465
- };
466
- });
983
+ // Go vendors a replaced module under the path it replaces.
984
+ vendorPath: record.path,
985
+ skipRegistryLookups: !!replacement?.local,
986
+ });
987
+ }
467
988
  // A module whose vendored licence notice is read is not asked for its
468
- // licence; getGoPkgComponent reads the same file.
469
- await prefetchGoPkgMetadata(withLocalLicenseFlags(modules));
989
+ // licence; getGoPkgComponent reads the same file. Local code is never
990
+ // asked about.
991
+ await prefetchGoPkgMetadata(goModulesToPrefetch(modules));
470
992
  for (const m of modules) {
471
- const gosumHash = gosumMap[`${m.name}@${m.version}`];
993
+ const gosumHash = m.version
994
+ ? gosumMap[`${m.name}@${m.version}`]
995
+ : undefined;
472
996
  const component = await getGoPkgComponent(
473
997
  "",
474
998
  m.name,
475
999
  m.version,
476
1000
  gosumHash,
477
- { projectRoot },
1001
+ m,
478
1002
  );
1003
+ addGoReplacementProperties(component, m.replacement);
479
1004
  let confidence = 0.7;
480
1005
  if (gosumHash) {
481
1006
  component.scope = "required";
@@ -487,135 +1012,160 @@ export async function parseGoModulesTxt(txtFile, gosumMap) {
487
1012
  return pkgList;
488
1013
  }
489
1014
 
1015
+ /**
1016
+ * Whether two paths name the same directory. `go list` prints the real path
1017
+ * of a directory, which differs from the one cdxgen was given when that runs
1018
+ * through a symbolic link (macOS /tmp is /private/tmp).
1019
+ *
1020
+ * @param {string} [a] A directory
1021
+ * @param {string} [b] Another directory
1022
+ * @returns {boolean}
1023
+ */
1024
+ function isSameGoDir(a, b) {
1025
+ if (!a || !b) {
1026
+ return false;
1027
+ }
1028
+ if (resolve(a) === resolve(b)) {
1029
+ return true;
1030
+ }
1031
+ try {
1032
+ return realpathSync(a) === realpathSync(b);
1033
+ } catch {
1034
+ return false;
1035
+ }
1036
+ }
1037
+
490
1038
  /**
491
1039
  * Parse go list output
492
1040
  *
1041
+ * Each row is printed by the template in createGoBom:
1042
+ * `Path|Version|Indirect|GoMod|GoVersion|Main|Time|Deprecated|GoModSum|Dir|Sum|
1043
+ * Replace.Path|Replace.Version|Replace.Sum`. Rows from older templates end
1044
+ * after `Dir`.
1045
+ *
1046
+ * A replaced module is reported as the module Go builds (see
1047
+ * {@link goReplacement}), with the checksum of the replacement. A go.work
1048
+ * workspace makes every member a main module, so the parent is the main
1049
+ * module in `options.mainModuleDir`, and the other members are returned as
1050
+ * `workspaceModules`.
1051
+ *
493
1052
  * @param {string} rawOutput Output from go list invocation
494
1053
  * @param {Object} gosumMap go.sum data
495
- * @returns Object with parent component and List of packages
1054
+ * @param {Object} [options] Options
1055
+ * @param {string} [options.mainModuleDir] Directory `go list` ran in, whose
1056
+ * main module is the parent component
1057
+ * @returns Object with parent component, list of packages, and the other main
1058
+ * modules of a workspace
496
1059
  */
497
- export async function parseGoListDep(rawOutput, gosumMap) {
1060
+ export async function parseGoListDep(rawOutput, gosumMap, options = {}) {
498
1061
  let parentComponent = {};
499
1062
  const deps = [];
1063
+ const workspaceModules = [];
500
1064
  if (typeof rawOutput === "string") {
501
- const keys_cache = {};
502
- const pkgs = rawOutput
503
- .split("\n")
504
- .filter((p) => p.trim().replace(/["']/g, "").length);
505
- // Same shape and same duplicate filter as the loop below, so the batch
506
- // covers every module the loop resolves and nothing else.
507
- const prefetchSeen = new Set();
508
- const prefetchModules = [];
509
- // The directory of the main module row, which is the project root, under
510
- // which the vendored copies of the dependencies live. `go list` may print
511
- // the main module last, so its row is found before the batch is built.
512
- let mainModuleDir;
513
- for (const l of pkgs) {
1065
+ const rows = [];
1066
+ const seen = new Set();
1067
+ for (const l of rawOutput.split("\n")) {
514
1068
  const verArr = l.trim().replace(/["']/g, "").split("|");
515
- if (verArr?.length >= 10 && verArr[5] === "true" && verArr[9]?.length) {
516
- mainModuleDir = verArr[9];
517
- }
518
- }
519
- for (const l of pkgs) {
520
- const verArr = l.trim().replace(/["']/g, "").split("|");
521
- if (!verArr || verArr.length < 5) {
522
- continue;
523
- }
524
- // The row whose fifth field is true is the main module: the project
525
- // itself, which no registry holds and which the loop below turns into
526
- // the parent component.
527
- if (verArr[5] === "true") {
1069
+ if (verArr.length < 5) {
528
1070
  continue;
529
1071
  }
1072
+ // Every package of a module repeats the module's row.
530
1073
  const key = `${verArr[0]}-${verArr[1]}`;
531
- if (prefetchSeen.has(key)) {
1074
+ if (seen.has(key)) {
532
1075
  continue;
533
1076
  }
534
- prefetchSeen.add(key);
535
- prefetchModules.push({
1077
+ seen.add(key);
1078
+ rows.push(verArr);
1079
+ }
1080
+ // The main module is the project itself, which no registry holds. In a
1081
+ // go.work workspace every member is a main module, and the one go list
1082
+ // ran in is the parent.
1083
+ const mainRows = rows.filter((verArr) => verArr[5] === "true");
1084
+ const mainRow =
1085
+ mainRows.find((verArr) =>
1086
+ isSameGoDir(verArr[9], options.mainModuleDir),
1087
+ ) || mainRows[0];
1088
+ // The directory of the main module, which is the project root, under
1089
+ // which the vendored copies of the dependencies live.
1090
+ const mainModuleDir = mainRow?.[9] || undefined;
1091
+ const modules = rows.map((verArr) => {
1092
+ const isMainModule = verArr[5] === "true";
1093
+ const replacement = isMainModule
1094
+ ? undefined
1095
+ : goReplacement(verArr[0], verArr[1], verArr[11], verArr[12]);
1096
+ const name = replacement?.name ?? verArr[0];
1097
+ const version = replacement ? replacement.version : verArr[1];
1098
+ return {
1099
+ verArr,
1100
+ isMainModule,
1101
+ replacement,
536
1102
  group: "",
537
- name: verArr[0],
538
- version: verArr[1],
1103
+ name,
1104
+ version,
1105
+ // Module.Sum is the checksum of the module content, the hash of what
1106
+ // is built. GoModSum hashes only its go.mod file.
1107
+ hash:
1108
+ (version && gosumMap[`${name}@${version}`]) ||
1109
+ goSumIntegrity(replacement ? verArr[13] : verArr[10]),
539
1110
  dir: verArr[9]?.length ? verArr[9] : undefined,
540
1111
  projectRoot: mainModuleDir,
541
- });
542
- }
1112
+ // Go vendors a replaced module under the path it replaces.
1113
+ vendorPath: verArr[0],
1114
+ skipRegistryLookups: isMainModule || !!replacement?.local,
1115
+ };
1116
+ });
543
1117
  // A module whose licence notice is held on this machine is not asked for
544
1118
  // its licence: getGoPkgComponent reads the same notices, so the batch
545
1119
  // covers exactly what the loop will look up.
546
- await prefetchGoPkgMetadata(withLocalLicenseFlags(prefetchModules));
547
- for (const l of pkgs) {
548
- const verArr = l.trim().replace(/["']/g, "").split("|");
549
- if (verArr && verArr.length >= 5) {
550
- const key = `${verArr[0]}-${verArr[1]}`;
551
- // Filter duplicates
552
- if (!keys_cache[key]) {
553
- keys_cache[key] = key;
554
- const version = verArr[1];
555
- let gosumHash = gosumMap[`${verArr[0]}@${version}`];
556
- if (!gosumHash && verArr.length >= 8 && verArr[8]?.length) {
557
- gosumHash = `sha256-${verArr[8].replace("h1:", "")}`;
558
- }
559
- const isMainModule = verArr[5] === "true";
560
- const component = await getGoPkgComponent(
561
- "",
562
- verArr[0],
563
- version,
564
- gosumHash,
565
- {
566
- skipRegistryLookups: isMainModule,
567
- dir: verArr[9]?.length ? verArr[9] : undefined,
568
- projectRoot: mainModuleDir,
569
- },
570
- );
571
- // This is misusing the scope attribute to represent direct vs indirect
572
- if (verArr[2] === "false") {
573
- component.scope = "required";
574
- } else if (verArr[2] === "true") {
575
- component.scope = "optional";
576
- }
577
- component.properties = [
578
- {
579
- name: "internal:SrcGoMod",
580
- value: verArr[3] || "",
581
- },
582
- {
583
- name: "internal:ModuleGoVersion",
584
- value: verArr[4] || "",
585
- },
586
- {
587
- name: "cdx:go:indirect",
588
- value: verArr[2],
589
- },
590
- ];
591
- if (
592
- verArr.length >= 6 &&
593
- verArr[6]?.length &&
594
- verArr[6] !== "<nil>"
595
- ) {
596
- component.properties.push({
597
- name: "cdx:go:creation_time",
598
- value: verArr[6],
599
- });
600
- }
601
- if (verArr.length >= 7 && verArr[7]?.length) {
602
- component.properties.push({
603
- name: "cdx:go:deprecated",
604
- value: verArr[7],
605
- });
606
- }
607
- if (verArr.length >= 9 && verArr[9]?.length) {
608
- component.properties.push({
609
- name: "cdx:go:local_dir",
610
- value: verArr[9],
611
- });
612
- }
613
- if (verArr.length > 5 && verArr[5] === "true") {
614
- parentComponent = component;
615
- } else {
616
- deps.push(component);
617
- }
618
- }
1120
+ await prefetchGoPkgMetadata(goModulesToPrefetch(modules));
1121
+ for (const m of modules) {
1122
+ const { verArr } = m;
1123
+ const component = await getGoPkgComponent(
1124
+ "",
1125
+ m.name,
1126
+ m.version,
1127
+ m.hash,
1128
+ m,
1129
+ );
1130
+ // This is misusing the scope attribute to represent direct vs indirect
1131
+ if (verArr[2] === "false") {
1132
+ component.scope = "required";
1133
+ } else if (verArr[2] === "true") {
1134
+ component.scope = "optional";
1135
+ }
1136
+ component.properties = [
1137
+ {
1138
+ name: "internal:SrcGoMod",
1139
+ value: verArr[3] || "",
1140
+ },
1141
+ {
1142
+ name: "internal:ModuleGoVersion",
1143
+ value: verArr[4] || "",
1144
+ },
1145
+ {
1146
+ name: "cdx:go:indirect",
1147
+ value: verArr[2],
1148
+ },
1149
+ ];
1150
+ if (verArr.length >= 6 && verArr[6]?.length && verArr[6] !== "<nil>") {
1151
+ component.properties.push({
1152
+ name: "cdx:go:creation_time",
1153
+ value: verArr[6],
1154
+ });
1155
+ }
1156
+ if (verArr.length >= 7 && verArr[7]?.length) {
1157
+ component.properties.push({
1158
+ name: "cdx:go:deprecated",
1159
+ value: verArr[7],
1160
+ });
1161
+ }
1162
+ addGoReplacementProperties(component, m.replacement);
1163
+ if (verArr === mainRow) {
1164
+ parentComponent = component;
1165
+ } else if (m.isMainModule) {
1166
+ workspaceModules.push(component);
1167
+ } else {
1168
+ deps.push(component);
619
1169
  }
620
1170
  }
621
1171
  }
@@ -628,6 +1178,7 @@ export async function parseGoListDep(rawOutput, gosumMap) {
628
1178
  // be built silently reorders the output.
629
1179
  (a["bom-ref"] || "").localeCompare(b["bom-ref"] || ""),
630
1180
  ),
1181
+ workspaceModules,
631
1182
  };
632
1183
  }
633
1184
 
@@ -674,9 +1225,39 @@ function goModGraphTokenPurl(token) {
674
1225
  return canonical ? Purl.parse(canonical) : null;
675
1226
  }
676
1227
 
1228
+ /**
1229
+ * Split a `go mod graph` token into its module path and version, exactly as
1230
+ * Go printed them. A main module is printed without a version.
1231
+ *
1232
+ * @param {string} token `module/path@version` or `module/path`
1233
+ * @returns {{path: string, version: string|undefined}}
1234
+ */
1235
+ function splitGoModGraphToken(token) {
1236
+ const at = token.indexOf("@");
1237
+ return at < 0
1238
+ ? { path: token, version: undefined }
1239
+ : { path: token.slice(0, at), version: token.slice(at + 1) };
1240
+ }
1241
+
1242
+ /**
1243
+ * Value of the first property with the given name.
1244
+ *
1245
+ * @param {Object} component Component
1246
+ * @param {string} name Property name
1247
+ * @returns {string|undefined}
1248
+ */
1249
+ function goComponentProperty(component, name) {
1250
+ return component?.properties?.find((p) => p.name === name)?.value;
1251
+ }
1252
+
677
1253
  /**
678
1254
  * Parse go mod graph
679
1255
  *
1256
+ * `go mod graph` names every module by the path and version its requirement
1257
+ * names, whatever replaces it. A replaced module is mapped to the module Go
1258
+ * builds: from the components `go list` reported, which record what they
1259
+ * replace, or else from the replace directives of go.mod.
1260
+ *
680
1261
  * @param {string} rawOutput Output from go mod graph invocation
681
1262
  * @param {string} goModFile go.mod file
682
1263
  * @param {Object} gosumMap Hashes from gosum for lookups
@@ -702,9 +1283,41 @@ export async function parseGoModGraph(
702
1283
  let goModPkgMap = {};
703
1284
  // Direct dependencies by manually parsing the go.mod data
704
1285
  const goModDirectDepsMap = {};
1286
+ // The same direct dependencies by module path, which stays correct when
1287
+ // the version the go.mod text names has drifted from the selected one.
1288
+ const goModDirectDepNames = new Set();
705
1289
  // Indirect dependencies by manually parsing the go.mod data
706
1290
  const goModOptionalDepsMap = {};
707
1291
  const excludedRefs = [];
1292
+ // The module versions `go list` reported are the ones MVS selected, and the
1293
+ // only ones the BOM carries components for. `go mod graph` instead prints
1294
+ // the requirement graph, where each edge names the version the source's
1295
+ // go.mod asks for, so the two routinely disagree for the same module. They
1296
+ // are keyed by the module path exactly as Go prints it, which keeps its
1297
+ // upper-case letters; a replaced module is keyed by the path it replaces,
1298
+ // which is the one the graph prints.
1299
+ const selectedVersions = {};
1300
+ // `path@version` as the graph prints it, for every module a replace
1301
+ // directive put another module in place of.
1302
+ const replacedModules = new Map();
1303
+ for (const epkg of epkgList) {
1304
+ const replaces = goComponentProperty(epkg, "cdx:go:replaces");
1305
+ if (replaces) {
1306
+ const { path, version } = splitGoModGraphToken(replaces);
1307
+ replacedModules.set(replaces, {
1308
+ name: epkg.name,
1309
+ version: epkg.version,
1310
+ local: !epkg.version,
1311
+ localDir: goComponentProperty(epkg, "cdx:go:local_dir"),
1312
+ replaces,
1313
+ });
1314
+ if (version) {
1315
+ selectedVersions[path] = version;
1316
+ }
1317
+ } else if (epkg?.name && epkg?.version) {
1318
+ selectedVersions[epkg.name] = epkg.version;
1319
+ }
1320
+ }
708
1321
  if (goModFile) {
709
1322
  goModPkgMap = await parseGoModData(
710
1323
  readFileSync(goModFile, { encoding: "utf-8" }),
@@ -713,6 +1326,11 @@ export async function parseGoModGraph(
713
1326
  if (goModPkgMap?.rootList) {
714
1327
  for (const epkg of goModPkgMap.rootList) {
715
1328
  goModDirectDepsMap[epkg["bom-ref"]] = true;
1329
+ // The bom-ref above names the version the go.mod text requires,
1330
+ // which an untidy module can leave older than the version MVS
1331
+ // selected. Root-edge validation below also asks by module path,
1332
+ // so a drifted direct require still counts as direct (#4497).
1333
+ goModDirectDepNames.add(epkg.name);
716
1334
  }
717
1335
  }
718
1336
  if (goModPkgMap?.pkgList) {
@@ -723,68 +1341,117 @@ export async function parseGoModGraph(
723
1341
  }
724
1342
  }
725
1343
  }
1344
+ // What the replace directives of go.mod make of a module the graph names. A
1345
+ // directive for one version wins over one for every version, as in Go.
1346
+ const goModReplacement = (path, version) => {
1347
+ let wildcard;
1348
+ for (const r of goModPkgMap?.replacements || []) {
1349
+ if (r.path !== path) {
1350
+ continue;
1351
+ }
1352
+ if (r.version === undefined) {
1353
+ wildcard = r;
1354
+ } else if (r.version === version) {
1355
+ return goReplacement(path, version, r.newPath, r.newVersion);
1356
+ }
1357
+ }
1358
+ return wildcard
1359
+ ? goReplacement(path, version, wildcard.newPath, wildcard.newVersion)
1360
+ : undefined;
1361
+ };
1362
+ // The module Go builds for one graph token, with the reference of the
1363
+ // component that stands for it.
1364
+ const graphModule = (path, version) => {
1365
+ const replacement = version
1366
+ ? replacedModules.get(`${path}@${version}`) ||
1367
+ goModReplacement(path, version)
1368
+ : undefined;
1369
+ const name = replacement ? replacement.name : path;
1370
+ const moduleVersion = replacement ? replacement.version : version;
1371
+ const token = moduleVersion ? `${name}@${moduleVersion}` : name;
1372
+ // Some golang modules (e.g. go4.org) have no path separator and
1373
+ // cdx-purl requires a namespace. Use the raw ref when parsing fails.
1374
+ const purl = goModGraphTokenPurl(token);
1375
+ return {
1376
+ path,
1377
+ name,
1378
+ version: moduleVersion,
1379
+ token,
1380
+ replacement,
1381
+ ref: purl ? decodeURIComponent(purl.toString()) : `pkg:golang/${token}`,
1382
+ };
1383
+ };
726
1384
  for (const epkg of epkgList) {
727
1385
  existingPkgMap[epkg["bom-ref"]] = true;
728
1386
  }
729
1387
  if (parentComponent && Object.keys(parentComponent).length) {
730
1388
  existingPkgMap[parentComponent["bom-ref"]] = true;
731
1389
  }
1390
+ const filterByExisting = Object.keys(existingPkgMap).length > 0;
732
1391
  if (typeof rawOutput === "string") {
733
1392
  const lines = rawOutput.split("\n");
734
1393
  // Each line is of the form ref dependsOn
735
1394
  // github.com/spf13/afero@v1.2.2 golang.org/x/text@v0.3.0
736
1395
  for (const l of lines) {
737
- // To keep the parsing logic simple we prefix pkg:golang/
738
- // and let packageurl work out the rest
739
1396
  const tmpA = l.replaceAll("\r", "").split(" ");
740
1397
  if (tmpA && tmpA.length === 2) {
741
1398
  try {
742
- // Some golang modules (e.g. go4.org) have no path separator and
743
- // cdx-purl requires a namespace. Use the raw ref when parsing fails.
744
- let sourceRefString;
745
- let sourceName = tmpA[0].split("@")[0];
746
- const sourcePurl = goModGraphTokenPurl(tmpA[0]);
747
- if (sourcePurl) {
748
- sourceRefString = decodeURIComponent(sourcePurl.toString());
749
- sourceName = `${sourcePurl.namespace ? `${sourcePurl.namespace}/` : ""}${sourcePurl.name}`;
750
- } else {
751
- sourceRefString = `pkg:golang/${tmpA[0]}`;
752
- }
753
- let dependsRefString;
754
- let dependsName = tmpA[1].split("@")[0];
755
- const dependsPurl = goModGraphTokenPurl(tmpA[1]);
756
- if (dependsPurl) {
757
- dependsRefString = decodeURIComponent(dependsPurl.toString());
758
- dependsName = `${dependsPurl.namespace ? `${dependsPurl.namespace}/` : ""}${dependsPurl.name}`;
759
- } else {
760
- dependsRefString = `pkg:golang/${tmpA[1]}`;
1399
+ const sourceToken = splitGoModGraphToken(tmpA[0]);
1400
+ const dependsToken = splitGoModGraphToken(tmpA[1]);
1401
+ const source = graphModule(sourceToken.path, sourceToken.version);
1402
+ let depends = graphModule(dependsToken.path, dependsToken.version);
1403
+ // An edge whose target names a version the build did not select is
1404
+ // rewritten to the selected one rather than dropped: the selected
1405
+ // module is the one actually linked, and dropping the edge silently
1406
+ // disconnects it from every dependent that asked for an older
1407
+ // version (#4497). Only the target is rewritten; an edge from a
1408
+ // version that was not selected describes the requirements of a
1409
+ // module the build never used, so it stays filtered out.
1410
+ if (filterByExisting && !existingPkgMap[depends.ref]) {
1411
+ const selectedVersion = selectedVersions[dependsToken.path];
1412
+ if (selectedVersion && selectedVersion !== dependsToken.version) {
1413
+ const rewritten = graphModule(dependsToken.path, selectedVersion);
1414
+ if (existingPkgMap[rewritten.ref]) {
1415
+ depends = rewritten;
1416
+ }
1417
+ }
761
1418
  }
1419
+ const sourceRefString = source.ref;
1420
+ const dependsRefString = depends.ref;
762
1421
  // Since go mod graph over-reports direct dependencies we use the existing list
763
1422
  // from go deps to filter the result
764
1423
  if (
765
- existingPkgMap &&
766
- Object.keys(existingPkgMap).length &&
1424
+ filterByExisting &&
767
1425
  (!existingPkgMap[sourceRefString] ||
768
1426
  !existingPkgMap[dependsRefString])
769
1427
  ) {
770
1428
  continue;
771
1429
  }
772
1430
  // Add the source and depends to the pkgList
773
- if (!addedPkgs[tmpA[0]] && !excludedRefs.includes(sourceRefString)) {
1431
+ if (
1432
+ !addedPkgs[sourceRefString] &&
1433
+ !excludedRefs.includes(sourceRefString)
1434
+ ) {
774
1435
  // go mod graph prints a main module without a version. A main
775
1436
  // module is the project itself, or a go.work member, which no
776
1437
  // registry holds, so it is never looked up. It still becomes a
777
1438
  // component, because the graph's edges hang off it.
778
1439
  const isParentSource =
779
- !tmpA[0].includes("@") ||
780
- sourceName === goModPkgMap?.parentComponent?.name ||
1440
+ !sourceToken.version ||
1441
+ source.name === goModPkgMap?.parentComponent?.name ||
781
1442
  sourceRefString === parentComponent?.["bom-ref"];
782
- const component = await getGoPkgComponent(
783
- "",
784
- sourceName,
785
- sourcePurl?.version || tmpA[0].split("@")[1],
786
- gosumMap[tmpA[0]],
787
- { skipRegistryLookups: isParentSource },
1443
+ const component = addGoReplacementProperties(
1444
+ await getGoPkgComponent(
1445
+ "",
1446
+ source.name,
1447
+ source.version,
1448
+ gosumMap[source.token],
1449
+ {
1450
+ skipRegistryLookups:
1451
+ isParentSource || !!source.replacement?.local,
1452
+ },
1453
+ ),
1454
+ source.replacement,
788
1455
  );
789
1456
  let confidence = 0.7;
790
1457
  if (goModOptionalDepsMap[component["bom-ref"]]) {
@@ -796,26 +1463,35 @@ export async function parseGoModGraph(
796
1463
  // These are likely false positives
797
1464
  if (
798
1465
  goModFile &&
799
- !Object.keys(existingPkgMap).length &&
1466
+ !filterByExisting &&
800
1467
  goModPkgMap?.parentComponent?.["bom-ref"] !== sourceRefString &&
801
1468
  !component.scope
802
1469
  ) {
803
1470
  continue;
804
1471
  }
805
- // Don't add the parent component to the package list
806
- if (goModPkgMap?.parentComponent?.["bom-ref"] !== sourceRefString) {
1472
+ // Don't add the parent component to the package list: neither
1473
+ // this go.mod's module nor the caller's parent, which in a go.work
1474
+ // workspace is another member that the graph also prints.
1475
+ if (
1476
+ goModPkgMap?.parentComponent?.["bom-ref"] !== sourceRefString &&
1477
+ parentComponent?.["bom-ref"] !== sourceRefString
1478
+ ) {
807
1479
  pkgList.push(
808
1480
  _addGoComponentEvidence(component, goModFile, confidence),
809
1481
  );
810
1482
  }
811
- addedPkgs[tmpA[0]] = true;
1483
+ addedPkgs[sourceRefString] = true;
812
1484
  }
813
- if (!addedPkgs[tmpA[1]]) {
814
- const component = await getGoPkgComponent(
815
- "",
816
- dependsName,
817
- dependsPurl?.version || tmpA[1].split("@")[1],
818
- gosumMap[tmpA[1]],
1485
+ if (!addedPkgs[dependsRefString]) {
1486
+ const component = addGoReplacementProperties(
1487
+ await getGoPkgComponent(
1488
+ "",
1489
+ depends.name,
1490
+ depends.version,
1491
+ gosumMap[depends.token],
1492
+ { skipRegistryLookups: !!depends.replacement?.local },
1493
+ ),
1494
+ depends.replacement,
819
1495
  );
820
1496
  let confidence = 0.7;
821
1497
  if (goModDirectDepsMap[component["bom-ref"]]) {
@@ -845,7 +1521,7 @@ export async function parseGoModGraph(
845
1521
  if (
846
1522
  goModFile &&
847
1523
  goModPkgMap?.parentComponent?.["bom-ref"] !== sourceRefString &&
848
- !Object.keys(existingPkgMap).length &&
1524
+ !filterByExisting &&
849
1525
  !component.scope
850
1526
  ) {
851
1527
  excludedRefs.push(dependsRefString);
@@ -857,7 +1533,7 @@ export async function parseGoModGraph(
857
1533
  pkgList.push(
858
1534
  _addGoComponentEvidence(component, goModFile, confidence),
859
1535
  );
860
- addedPkgs[tmpA[1]] = true;
1536
+ addedPkgs[dependsRefString] = true;
861
1537
  }
862
1538
  if (!depsMap[sourceRefString]) {
863
1539
  depsMap[sourceRefString] = new Set();
@@ -865,11 +1541,18 @@ export async function parseGoModGraph(
865
1541
  if (!depsMap[dependsRefString]) {
866
1542
  depsMap[dependsRefString] = new Set();
867
1543
  }
868
- // Check if the root is really dependent on this component
1544
+ // Check if the root is really dependent on this component. The
1545
+ // by-path check accepts a direct require whose go.mod version
1546
+ // drifted from the selected one: the edge was already rewritten
1547
+ // to the selected version above, and the by-ref map only knows
1548
+ // the stale bom-ref (#4497). It also accepts a directly required
1549
+ // module that a directory replaces, whose component has no
1550
+ // version.
869
1551
  if (
870
1552
  goModPkgMap?.parentComponent?.["bom-ref"] === sourceRefString &&
871
1553
  Object.keys(goModDirectDepsMap).length &&
872
- !goModDirectDepsMap[dependsRefString]
1554
+ !goModDirectDepsMap[dependsRefString] &&
1555
+ !goModDirectDepNames.has(dependsToken.path)
873
1556
  ) {
874
1557
  // ignore
875
1558
  } else if (!excludedRefs.includes(dependsRefString)) {
@@ -944,41 +1627,56 @@ export function isGoModWhyNotNeeded(rawOutput) {
944
1627
  * building components or touching the network.
945
1628
  *
946
1629
  * @param {string} gosumData Content of go.sum
947
- * @returns {Object} Map of `module@version` keys to `sha256-…` values
1630
+ * @returns {Object} Map of `module@version` keys to the `sha256-…` checksum of
1631
+ * the module content, for every version go.sum holds one for
948
1632
  */
949
1633
  export function parseGosumHashes(gosumData) {
950
1634
  const gosumMap = {};
951
1635
  for (const entry of decodeGosumData(gosumData)) {
952
- gosumMap[`${entry.name}@${entry.version}`] = entry.hash;
1636
+ if (entry.hash) {
1637
+ gosumMap[`${entry.name}@${entry.version}`] = entry.hash;
1638
+ }
953
1639
  }
954
1640
  return gosumMap;
955
1641
  }
956
1642
 
957
1643
  /**
958
- * Decode the go.mod lines of a go.sum file into module entries.
1644
+ * Decode a go.sum file into one entry per module version it names.
1645
+ *
1646
+ * go.sum holds up to two lines per module version: `path version h1:…`, the
1647
+ * checksum of the module's content, and `path version/go.mod h1:…`, the
1648
+ * checksum of its go.mod file alone. The content checksum is the hash of the
1649
+ * component. A version with only a go.mod line was needed to resolve the
1650
+ * module graph but never downloaded, so it has no content hash.
959
1651
  *
960
1652
  * @param {string} gosumData Content of go.sum
961
- * @returns {Object[]} Entries with `name`, `version` and `hash`
1653
+ * @returns {Object[]} Entries with `name`, `version` and `hash`, in the order
1654
+ * go.sum first names them
962
1655
  */
963
1656
  function decodeGosumData(gosumData) {
964
- const entries = [];
1657
+ const entries = new Map();
965
1658
  if (!gosumData) {
966
- return entries;
1659
+ return [];
967
1660
  }
968
1661
  for (const l of gosumData.split("\n")) {
969
- const m = l.replaceAll("\r", "");
970
- // look for lines containing go.mod
971
- if (m.indexOf("go.mod") > -1) {
972
- const tmpA = m.split(" ");
973
- entries.push({
974
- group: "",
975
- name: tmpA[0],
976
- version: tmpA[1].replace("/go.mod", ""),
977
- hash: tmpA[tmpA.length - 1].replace("h1:", "sha256-"),
978
- });
1662
+ const fields = l.replaceAll("\r", "").trim().split(/\s+/);
1663
+ if (fields.length !== 3 || !fields[2].startsWith("h1:")) {
1664
+ continue;
1665
+ }
1666
+ const isGoModLine = fields[1].endsWith("/go.mod");
1667
+ const name = fields[0];
1668
+ const version = isGoModLine
1669
+ ? fields[1].slice(0, -"/go.mod".length)
1670
+ : fields[1];
1671
+ const key = `${name}@${version}`;
1672
+ if (!entries.has(key)) {
1673
+ entries.set(key, { group: "", name, version, hash: undefined });
1674
+ }
1675
+ if (!isGoModLine) {
1676
+ entries.get(key).hash = goSumIntegrity(fields[2]);
979
1677
  }
980
1678
  }
981
- return entries;
1679
+ return [...entries.values()];
982
1680
  }
983
1681
 
984
1682
  /**
@@ -991,7 +1689,7 @@ export async function parseGosumData(gosumData) {
991
1689
  const entries = decodeGosumData(gosumData);
992
1690
  // A module whose licence notice is read locally is not asked for its
993
1691
  // licence; getGoPkgComponent reads the same notice for each entry below.
994
- await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
1692
+ await prefetchGoPkgMetadata(goModulesToPrefetch(entries));
995
1693
  for (const entry of entries) {
996
1694
  pkgList.push(
997
1695
  await getGoPkgComponent("", entry.name, entry.version, entry.hash),
@@ -1081,6 +1779,12 @@ async function finalizeGopkgProject(pkg) {
1081
1779
  * Parses the output of `go version -m` (build info) and returns a list of Go
1082
1780
  * package components for each "dep" line, including name, version, and integrity hash.
1083
1781
  *
1782
+ * A dependency that a replace directive swapped out is printed as its `dep`
1783
+ * line followed by a `=>` line naming the replacement (runtime/debug
1784
+ * BuildInfo.String). The replacement is what the binary holds, so it is
1785
+ * reported as {@link goReplacement} describes, with the replacement's
1786
+ * checksum.
1787
+ *
1084
1788
  * @param {string} buildInfoData Raw string output from `go version -m`
1085
1789
  * @returns {Promise<Object[]>} List of Go package component objects
1086
1790
  */
@@ -1090,29 +1794,48 @@ export async function parseGoVersionData(buildInfoData) {
1090
1794
  return pkgList;
1091
1795
  }
1092
1796
  const entries = [];
1797
+ let lastDep;
1093
1798
  for (const line of buildInfoData.split("\n")) {
1094
- const l = line.trim().replace(/\t/g, " ");
1095
- if (!l.startsWith("dep")) {
1799
+ const fields = line.trim().split(/\s+/);
1800
+ if (fields[0] === "=>" && lastDep && fields.length >= 2) {
1801
+ lastDep.replacement = goReplacement(
1802
+ lastDep.name,
1803
+ lastDep.version,
1804
+ fields[1],
1805
+ fields[2],
1806
+ );
1807
+ lastDep.name = lastDep.replacement.name;
1808
+ lastDep.version = lastDep.replacement.version;
1809
+ lastDep.skipRegistryLookups = lastDep.replacement.local;
1810
+ lastDep.hash = goSumIntegrity(fields[3]) || "";
1811
+ lastDep = undefined;
1096
1812
  continue;
1097
1813
  }
1098
- const tmpA = l.split(" ");
1099
- if (!tmpA || tmpA.length < 3) {
1814
+ lastDep = undefined;
1815
+ if (fields[0] !== "dep" || fields.length < 3) {
1100
1816
  continue;
1101
1817
  }
1102
- entries.push({
1818
+ lastDep = {
1103
1819
  group: "",
1104
- name: tmpA[1].trim(),
1105
- version: tmpA[2].trim(),
1106
- hash:
1107
- tmpA.length === 4
1108
- ? tmpA[tmpA.length - 1].replace("h1:", "sha256-")
1109
- : "",
1110
- });
1820
+ name: fields[1],
1821
+ version: fields[2],
1822
+ hash: goSumIntegrity(fields[3]) || "",
1823
+ };
1824
+ entries.push(lastDep);
1111
1825
  }
1112
- await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
1826
+ await prefetchGoPkgMetadata(goModulesToPrefetch(entries));
1113
1827
  for (const entry of entries) {
1114
1828
  pkgList.push(
1115
- await getGoPkgComponent("", entry.name, entry.version, entry.hash),
1829
+ addGoReplacementProperties(
1830
+ await getGoPkgComponent(
1831
+ "",
1832
+ entry.name,
1833
+ entry.version,
1834
+ entry.hash,
1835
+ entry,
1836
+ ),
1837
+ entry.replacement,
1838
+ ),
1116
1839
  );
1117
1840
  }
1118
1841
  return pkgList;