@cdxgen/cdxgen 13.1.0 → 13.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -15
- package/bin/cdxgen.js +243 -122
- package/bin/evinse.js +7 -0
- package/bin/sign.js +88 -14
- package/bin/tracebom.js +64 -4
- package/bin/validate.js +29 -7
- package/bin/verify.js +79 -39
- package/data/crypto-oid.json +88 -12
- package/data/remediations.json +47 -0
- package/data/rules/ai-bom.yaml +78 -0
- package/data/rules/cargo-rust.yaml +132 -0
- package/data/rules/mcp-servers.yaml +122 -0
- package/lib/audit/targets.js +6 -1
- package/lib/cli/bomAssembly.js +117 -15
- package/lib/cli/cliOptions.js +93 -1
- package/lib/cli/index.js +235 -5
- package/lib/cli/jsBom.js +108 -26
- package/lib/cli/jvmBom.js +68 -12
- package/lib/cli/managedBom.js +264 -41
- package/lib/cli/nativeBom.js +923 -15
- package/lib/core/activity.js +120 -2
- package/lib/core/env.js +44 -7
- package/lib/core/fs.js +174 -6
- package/lib/core/metadataProperties.js +97 -0
- package/lib/core/projectConfig.js +31 -4
- package/lib/core/userAgent.js +151 -0
- package/lib/ecosystems/bunutils.js +269 -101
- package/lib/ecosystems/cargoCache.js +364 -0
- package/lib/ecosystems/caxa.js +447 -7
- package/lib/ecosystems/cmakeResolver.js +66 -0
- package/lib/ecosystems/cppEvidence.js +104 -33
- package/lib/ecosystems/ecosystems.js +260 -99
- package/lib/ecosystems/golem.js +11 -1
- package/lib/ecosystems/gradleutils.js +185 -9
- package/lib/ecosystems/jsEvidence.js +19 -20
- package/lib/ecosystems/kosi.js +1263 -0
- package/lib/ecosystems/npmutils.js +110 -9
- package/lib/ecosystems/parsers-crystal.js +179 -0
- package/lib/ecosystems/parsers-dotnet.js +120 -49
- package/lib/ecosystems/parsers-elm.js +816 -0
- package/lib/ecosystems/parsers-gleam.js +1 -1
- package/lib/ecosystems/parsers-go.js +47 -9
- package/lib/ecosystems/parsers-js.js +171 -50
- package/lib/ecosystems/parsers-julia.js +268 -0
- package/lib/ecosystems/parsers-lua.js +218 -0
- package/lib/ecosystems/parsers-misc.js +919 -161
- package/lib/ecosystems/parsers-nim.js +221 -0
- package/lib/ecosystems/parsers-opam.js +180 -0
- package/lib/ecosystems/parsers-perl.js +277 -0
- package/lib/ecosystems/parsers-python.js +27 -15
- package/lib/ecosystems/parsers-r.js +148 -0
- package/lib/ecosystems/parsers-rebar.js +411 -0
- package/lib/ecosystems/parsers-rust.js +954 -33
- package/lib/ecosystems/parsers-spack.js +204 -0
- package/lib/ecosystems/parsers-stack.js +264 -0
- package/lib/ecosystems/parsers-terraform.js +194 -0
- package/lib/ecosystems/rubyutils.js +1426 -166
- package/lib/ecosystems/rusi.js +158 -7
- package/lib/evinser/evinser.js +547 -129
- package/lib/evinser/swiftsem.js +1914 -302
- package/lib/helpers/agenticToolutils.js +925 -0
- package/lib/helpers/bomSigner.js +652 -221
- package/lib/inventory/agentFormulationParser.js +57 -4
- package/lib/inventory/aiCollector.js +504 -10
- package/lib/inventory/aiInventory.js +12 -0
- package/lib/inventory/analyzer.js +488 -30
- package/lib/inventory/astWorker.js +6 -0
- package/lib/inventory/atomUtils.js +568 -31
- package/lib/inventory/auditCategories.js +8 -0
- package/lib/inventory/cbomutils.js +30 -0
- package/lib/inventory/cdxrs.js +110 -14
- package/lib/inventory/communityAiConfigParser.js +215 -2
- package/lib/inventory/depsUtils.js +183 -11
- package/lib/inventory/display.js +4 -5
- package/lib/inventory/dosai.js +627 -13
- package/lib/inventory/dynamic.js +39 -1
- package/lib/inventory/envcontext.js +9 -6
- package/lib/inventory/evidenceUtils.js +26 -5
- package/lib/inventory/fetchBatch.js +190 -19
- package/lib/inventory/fetchRate.js +74 -11
- package/lib/inventory/forgeEnricher.js +3 -2
- package/lib/inventory/jvmToolEnv.js +6 -5
- package/lib/inventory/mcpConfigParser.js +166 -2
- package/lib/inventory/mcpDiscovery.js +11 -0
- package/lib/inventory/mcpServerManifest.js +389 -0
- package/lib/inventory/mcpbBundle.js +419 -0
- package/lib/inventory/plugins.js +3 -0
- package/lib/inventory/protobom.js +62 -89
- package/lib/inventory/protobomLoader.js +4 -0
- package/lib/inventory/purl.js +152 -3
- package/lib/inventory/rustFormulationParser.js +1 -1
- package/lib/inventory/sbomDocument.js +2 -70
- package/lib/inventory/source.js +47 -23
- package/lib/inventory/stylePackageRefs.js +41 -0
- package/lib/inventory/svelteUtils.js +524 -0
- package/lib/inventory/traceRunner.js +210 -21
- package/lib/managers/binary.js +115 -24
- package/lib/managers/docker.js +440 -20
- package/lib/managers/oci.js +39 -1
- package/lib/parsers/cmakeLists.js +452 -0
- package/lib/parsers/huggingfaceUtils.js +15 -7
- package/lib/parsers/largeJson.js +454 -0
- package/lib/parsers/npmrc.js +19 -1
- package/lib/parsers/onnx.js +294 -0
- package/lib/parsers/safetensors.js +114 -0
- package/lib/stages/postgen/introspection/markers.js +29 -6
- package/lib/stages/postgen/postgen.js +237 -16
- package/lib/third-party/arborist/snapshots/calc-dep-flags.test.cjs +462 -0
- package/lib/validator/bomValidator.js +6 -16
- package/lib/validator/index.js +48 -10
- package/lib/validator/reporters/console.js +3 -0
- package/lib/validator/reporters/json.js +1 -0
- package/lib/validator/reporters/sarif.js +7 -0
- package/package.json +27 -27
- package/types/lib/audit/targets.d.ts.map +1 -1
- package/types/lib/cli/bomAssembly.d.ts +2 -1
- package/types/lib/cli/bomAssembly.d.ts.map +1 -1
- package/types/lib/cli/cliOptions.d.ts +33 -1
- package/types/lib/cli/cliOptions.d.ts.map +1 -1
- package/types/lib/cli/index.d.ts +2 -2
- package/types/lib/cli/index.d.ts.map +1 -1
- package/types/lib/cli/jsBom.d.ts +23 -0
- package/types/lib/cli/jsBom.d.ts.map +1 -1
- package/types/lib/cli/jvmBom.d.ts.map +1 -1
- package/types/lib/cli/managedBom.d.ts +33 -0
- package/types/lib/cli/managedBom.d.ts.map +1 -1
- package/types/lib/cli/nativeBom.d.ts +88 -8
- package/types/lib/cli/nativeBom.d.ts.map +1 -1
- package/types/lib/core/activity.d.ts.map +1 -1
- package/types/lib/core/env.d.ts +21 -0
- package/types/lib/core/env.d.ts.map +1 -1
- package/types/lib/core/fs.d.ts +55 -0
- package/types/lib/core/fs.d.ts.map +1 -1
- package/types/lib/core/metadataProperties.d.ts +39 -0
- package/types/lib/core/metadataProperties.d.ts.map +1 -0
- package/types/lib/core/projectConfig.d.ts +6 -2
- package/types/lib/core/projectConfig.d.ts.map +1 -1
- package/types/lib/core/userAgent.d.ts +56 -0
- package/types/lib/core/userAgent.d.ts.map +1 -0
- package/types/lib/ecosystems/bunutils.d.ts +12 -5
- package/types/lib/ecosystems/bunutils.d.ts.map +1 -1
- package/types/lib/ecosystems/cargoCache.d.ts +75 -0
- package/types/lib/ecosystems/cargoCache.d.ts.map +1 -0
- package/types/lib/ecosystems/caxa.d.ts +14 -0
- package/types/lib/ecosystems/caxa.d.ts.map +1 -1
- package/types/lib/ecosystems/cmakeResolver.d.ts +23 -0
- package/types/lib/ecosystems/cmakeResolver.d.ts.map +1 -1
- package/types/lib/ecosystems/cppEvidence.d.ts +2 -2
- package/types/lib/ecosystems/cppEvidence.d.ts.map +1 -1
- package/types/lib/ecosystems/ecosystems.d.ts +15 -1
- package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
- package/types/lib/ecosystems/golem.d.ts.map +1 -1
- package/types/lib/ecosystems/gradleutils.d.ts +40 -4
- package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
- package/types/lib/ecosystems/jsEvidence.d.ts +5 -2
- package/types/lib/ecosystems/jsEvidence.d.ts.map +1 -1
- package/types/lib/ecosystems/kosi.d.ts +86 -0
- package/types/lib/ecosystems/kosi.d.ts.map +1 -0
- package/types/lib/ecosystems/npmutils.d.ts +42 -0
- package/types/lib/ecosystems/npmutils.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-crystal.d.ts +27 -0
- package/types/lib/ecosystems/parsers-crystal.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-dotnet.d.ts +19 -0
- package/types/lib/ecosystems/parsers-dotnet.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-elm.d.ts +79 -0
- package/types/lib/ecosystems/parsers-elm.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-go.d.ts +10 -0
- package/types/lib/ecosystems/parsers-go.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-js.d.ts +10 -4
- package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-julia.d.ts +57 -0
- package/types/lib/ecosystems/parsers-julia.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-lua.d.ts +50 -0
- package/types/lib/ecosystems/parsers-lua.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-misc.d.ts +90 -3
- package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-nim.d.ts +43 -0
- package/types/lib/ecosystems/parsers-nim.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-opam.d.ts +31 -0
- package/types/lib/ecosystems/parsers-opam.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-perl.d.ts +36 -0
- package/types/lib/ecosystems/parsers-perl.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-r.d.ts +35 -0
- package/types/lib/ecosystems/parsers-r.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-rebar.d.ts +42 -0
- package/types/lib/ecosystems/parsers-rebar.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-rust.d.ts +199 -0
- package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-spack.d.ts +33 -0
- package/types/lib/ecosystems/parsers-spack.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-stack.d.ts +35 -0
- package/types/lib/ecosystems/parsers-stack.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-terraform.d.ts +34 -0
- package/types/lib/ecosystems/parsers-terraform.d.ts.map +1 -0
- package/types/lib/ecosystems/pythonutils.d.ts +1 -1
- package/types/lib/ecosystems/rubyutils.d.ts +143 -2
- package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
- package/types/lib/ecosystems/rusi.d.ts +13 -0
- package/types/lib/ecosystems/rusi.d.ts.map +1 -1
- package/types/lib/evinser/evinser.d.ts +81 -2
- package/types/lib/evinser/evinser.d.ts.map +1 -1
- package/types/lib/evinser/swiftsem.d.ts +280 -15
- package/types/lib/evinser/swiftsem.d.ts.map +1 -1
- package/types/lib/helpers/agenticToolutils.d.ts +87 -0
- package/types/lib/helpers/agenticToolutils.d.ts.map +1 -0
- package/types/lib/helpers/bomSigner.d.ts +106 -13
- package/types/lib/helpers/bomSigner.d.ts.map +1 -1
- package/types/lib/inventory/agentFormulationParser.d.ts.map +1 -1
- package/types/lib/inventory/aiCollector.d.ts.map +1 -1
- package/types/lib/inventory/aiInventory.d.ts.map +1 -1
- package/types/lib/inventory/analyzer.d.ts.map +1 -1
- package/types/lib/inventory/atomUtils.d.ts +145 -3
- package/types/lib/inventory/atomUtils.d.ts.map +1 -1
- package/types/lib/inventory/auditCategories.d.ts +4 -0
- package/types/lib/inventory/auditCategories.d.ts.map +1 -1
- package/types/lib/inventory/cbomutils.d.ts.map +1 -1
- package/types/lib/inventory/cdxrs.d.ts +8 -3
- package/types/lib/inventory/cdxrs.d.ts.map +1 -1
- package/types/lib/inventory/communityAiConfigParser.d.ts.map +1 -1
- package/types/lib/inventory/depsUtils.d.ts +15 -0
- package/types/lib/inventory/depsUtils.d.ts.map +1 -1
- package/types/lib/inventory/display.d.ts.map +1 -1
- package/types/lib/inventory/dosai.d.ts +87 -1
- package/types/lib/inventory/dosai.d.ts.map +1 -1
- package/types/lib/inventory/dynamic.d.ts.map +1 -1
- package/types/lib/inventory/envcontext.d.ts.map +1 -1
- package/types/lib/inventory/evidenceUtils.d.ts.map +1 -1
- package/types/lib/inventory/fetchBatch.d.ts +34 -0
- package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
- package/types/lib/inventory/fetchRate.d.ts +9 -6
- package/types/lib/inventory/fetchRate.d.ts.map +1 -1
- package/types/lib/inventory/forgeEnricher.d.ts.map +1 -1
- package/types/lib/inventory/jvmToolEnv.d.ts +5 -4
- package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -1
- package/types/lib/inventory/mcpConfigParser.d.ts.map +1 -1
- package/types/lib/inventory/mcpDiscovery.d.ts.map +1 -1
- package/types/lib/inventory/mcpServerManifest.d.ts +19 -0
- package/types/lib/inventory/mcpServerManifest.d.ts.map +1 -0
- package/types/lib/inventory/mcpbBundle.d.ts +19 -0
- package/types/lib/inventory/mcpbBundle.d.ts.map +1 -0
- package/types/lib/inventory/plugins.d.ts.map +1 -1
- package/types/lib/inventory/protobom.d.ts +18 -13
- package/types/lib/inventory/protobom.d.ts.map +1 -1
- package/types/lib/inventory/protobomLoader.d.ts.map +1 -1
- package/types/lib/inventory/purl.d.ts +71 -2
- package/types/lib/inventory/purl.d.ts.map +1 -1
- package/types/lib/inventory/sbomDocument.d.ts.map +1 -1
- package/types/lib/inventory/source.d.ts.map +1 -1
- package/types/lib/inventory/stylePackageRefs.d.ts +17 -0
- package/types/lib/inventory/stylePackageRefs.d.ts.map +1 -0
- package/types/lib/inventory/svelteUtils.d.ts +124 -0
- package/types/lib/inventory/svelteUtils.d.ts.map +1 -0
- package/types/lib/inventory/traceRunner.d.ts +47 -1
- package/types/lib/inventory/traceRunner.d.ts.map +1 -1
- package/types/lib/managers/binary.d.ts +30 -1
- package/types/lib/managers/binary.d.ts.map +1 -1
- package/types/lib/managers/docker.d.ts +29 -0
- package/types/lib/managers/docker.d.ts.map +1 -1
- package/types/lib/managers/oci.d.ts +14 -0
- package/types/lib/managers/oci.d.ts.map +1 -1
- package/types/lib/parsers/cmakeLists.d.ts +76 -0
- package/types/lib/parsers/cmakeLists.d.ts.map +1 -0
- package/types/lib/parsers/huggingfaceUtils.d.ts +8 -1
- package/types/lib/parsers/huggingfaceUtils.d.ts.map +1 -1
- package/types/lib/parsers/largeJson.d.ts +52 -0
- package/types/lib/parsers/largeJson.d.ts.map +1 -0
- package/types/lib/parsers/npmrc.d.ts +5 -0
- package/types/lib/parsers/npmrc.d.ts.map +1 -1
- package/types/lib/parsers/onnx.d.ts +28 -0
- package/types/lib/parsers/onnx.d.ts.map +1 -0
- package/types/lib/parsers/safetensors.d.ts +22 -0
- package/types/lib/parsers/safetensors.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts +3 -2
- package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -1
- package/types/lib/stages/postgen/postgen.d.ts +33 -0
- package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
- package/types/lib/validator/bomValidator.d.ts.map +1 -1
- package/types/lib/validator/index.d.ts +10 -1
- package/types/lib/validator/index.d.ts.map +1 -1
- package/types/lib/validator/reporters/console.d.ts.map +1 -1
- package/types/lib/validator/reporters/json.d.ts.map +1 -1
- package/types/lib/validator/reporters/sarif.d.ts.map +1 -1
package/lib/inventory/dynamic.js
CHANGED
|
@@ -34,6 +34,7 @@ export async function buildDynamicComponents(
|
|
|
34
34
|
const result = await executeAndTrace(commandStr, workingDir, traceOptions);
|
|
35
35
|
const libPaths = result.libPaths || [];
|
|
36
36
|
const httpAccessEntries = result.httpAccessEntries || [];
|
|
37
|
+
const egressEntries = result.egressEntries || [];
|
|
37
38
|
const cryptoComponents = result.cryptoComponents || [];
|
|
38
39
|
const components = [];
|
|
39
40
|
|
|
@@ -141,5 +142,42 @@ export async function buildDynamicComponents(
|
|
|
141
142
|
};
|
|
142
143
|
});
|
|
143
144
|
|
|
144
|
-
|
|
145
|
+
// Egress seen by the safer-exec proxy: hostnames the command connected to
|
|
146
|
+
// (allowed) or tried to reach (denied). This works on every platform,
|
|
147
|
+
// including macOS where eBPF URL tracing is unavailable.
|
|
148
|
+
const servicesByName = new Map(services.map((svc) => [svc.name, svc]));
|
|
149
|
+
for (const egress of egressEntries) {
|
|
150
|
+
const name = `dynamic-${egress.host}-${egress.port}`;
|
|
151
|
+
const defaultPort = egress.protocol === "http" ? 80 : 443;
|
|
152
|
+
const endpoint = `${egress.protocol}://${egress.host}${egress.port !== defaultPort ? `:${egress.port}` : ""}`;
|
|
153
|
+
let svc = servicesByName.get(name);
|
|
154
|
+
if (!svc) {
|
|
155
|
+
svc = {
|
|
156
|
+
name,
|
|
157
|
+
bomRef: `urn:service:dynamic:${name}`,
|
|
158
|
+
endpoints: [],
|
|
159
|
+
properties: [],
|
|
160
|
+
};
|
|
161
|
+
servicesByName.set(name, svc);
|
|
162
|
+
services.push(svc);
|
|
163
|
+
}
|
|
164
|
+
if (!svc.endpoints.some((e) => e.startsWith(endpoint))) {
|
|
165
|
+
svc.endpoints.push(endpoint);
|
|
166
|
+
svc.endpoints.sort();
|
|
167
|
+
}
|
|
168
|
+
if (
|
|
169
|
+
!svc.properties.some(
|
|
170
|
+
(p) =>
|
|
171
|
+
p.name === "cdx:dynamic:egressDecision" &&
|
|
172
|
+
p.value === egress.decision,
|
|
173
|
+
)
|
|
174
|
+
) {
|
|
175
|
+
svc.properties.push({
|
|
176
|
+
name: "cdx:dynamic:egressDecision",
|
|
177
|
+
value: egress.decision,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return { components, services, dryRunSummary: result.dryRunSummary };
|
|
145
183
|
}
|
|
@@ -101,15 +101,18 @@ function hardenedGitEnv() {
|
|
|
101
101
|
* @type {Object<string, string>}
|
|
102
102
|
*/
|
|
103
103
|
export const SDKMAN_JAVA_TOOL_ALIASES = {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
104
|
+
// Identifiers must be downloadable from the sdkman broker: the `sdk list`
|
|
105
|
+
// output advertises Temurin respin ids such as 21.0.12+1.1-tem, but the
|
|
106
|
+
// broker cannot serve ids containing '+', so the plain ids are pinned.
|
|
107
|
+
java8: readEnvironmentVariable("JAVA8_TOOL") || "8.0.504-zulu", // Temurin no longer offers java8 :(
|
|
108
|
+
java11: readEnvironmentVariable("JAVA11_TOOL") || "11.0.32-tem",
|
|
109
|
+
java17: readEnvironmentVariable("JAVA17_TOOL") || "17.0.20-tem",
|
|
110
|
+
java21: readEnvironmentVariable("JAVA21_TOOL") || "21.0.12-tem",
|
|
108
111
|
java22: readEnvironmentVariable("JAVA22_TOOL") || "22.0.2-tem",
|
|
109
112
|
java23: readEnvironmentVariable("JAVA23_TOOL") || "23.0.2-tem",
|
|
110
113
|
java24: readEnvironmentVariable("JAVA24_TOOL") || "24.0.2-tem",
|
|
111
|
-
java25: readEnvironmentVariable("JAVA25_TOOL") || "25.0.
|
|
112
|
-
java26: readEnvironmentVariable("JAVA26_TOOL") || "26.0.
|
|
114
|
+
java25: readEnvironmentVariable("JAVA25_TOOL") || "25.0.4-tem",
|
|
115
|
+
java26: readEnvironmentVariable("JAVA26_TOOL") || "26.0.2-tem",
|
|
113
116
|
};
|
|
114
117
|
|
|
115
118
|
/**
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
1
|
import { platform } from "node:os";
|
|
3
2
|
import { extname } from "node:path";
|
|
4
3
|
|
|
@@ -6,6 +5,10 @@ import { build } from "@cdxgen/cdx-purl";
|
|
|
6
5
|
|
|
7
6
|
import { safeExistsSync } from "../core/fs.js";
|
|
8
7
|
import { thoughtLog } from "../core/logger.js";
|
|
8
|
+
import {
|
|
9
|
+
addDosaiReachabilityProperties,
|
|
10
|
+
readDosaiMethodsReport,
|
|
11
|
+
} from "./dosai.js";
|
|
9
12
|
import {
|
|
10
13
|
addDosaiSetValue,
|
|
11
14
|
buildDosaiPurlAliasMap,
|
|
@@ -256,6 +259,7 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
|
|
|
256
259
|
const purlLocationMap = {};
|
|
257
260
|
const purlModulesMap = {};
|
|
258
261
|
const purlMethodsMap = {};
|
|
262
|
+
const purlReachabilityMap = {};
|
|
259
263
|
const purlAliasMap = buildDosaiPurlAliasMap(pkgList);
|
|
260
264
|
for (const apkg of pkgList) {
|
|
261
265
|
if (apkg.properties && Array.isArray(apkg.properties)) {
|
|
@@ -273,10 +277,16 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
|
|
|
273
277
|
});
|
|
274
278
|
}
|
|
275
279
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
+
// Only MethodCalls into a package .dll produce evidence below, so a report
|
|
281
|
+
// too large to parse whole keeps just those.
|
|
282
|
+
const slicesData = readDosaiMethodsReport(slicesFile, {
|
|
283
|
+
keepMethodCall: (methodCall) =>
|
|
284
|
+
Boolean(
|
|
285
|
+
methodCall?.Module?.endsWith(".dll") &&
|
|
286
|
+
pkgFilePurlMap[methodCall.Module],
|
|
287
|
+
),
|
|
288
|
+
});
|
|
289
|
+
if (!slicesData) {
|
|
280
290
|
return pkgList;
|
|
281
291
|
}
|
|
282
292
|
if (slicesData && Object.keys(slicesData)) {
|
|
@@ -340,6 +350,9 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
|
|
|
340
350
|
if (!modPurl) {
|
|
341
351
|
continue;
|
|
342
352
|
}
|
|
353
|
+
// Confidence/evidence facts for this purl, applied to the component below so an
|
|
354
|
+
// unbuilt tree stays distinguishable from an only-imported package.
|
|
355
|
+
purlReachabilityMap[modPurl] ??= reachability;
|
|
343
356
|
let hasExplicitSourceLocations = false;
|
|
344
357
|
for (const sourceLocation of reachability.SourceLocations || []) {
|
|
345
358
|
const location = dosaiSourceLocation(sourceLocation);
|
|
@@ -456,6 +469,14 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
|
|
|
456
469
|
"I didn't find any occurrence evidence or detailed imported modules, even though there is good dependency slice data from dosai. This is surprising.",
|
|
457
470
|
);
|
|
458
471
|
}
|
|
472
|
+
// Reachability confidence facts (kind, confidence, evidence, reasons) as properties.
|
|
473
|
+
// Applied outside the location guard: a Low-confidence unbuilt tree must stay
|
|
474
|
+
// distinguishable from an only-imported package even when no locations were found.
|
|
475
|
+
for (const apkg of pkgList) {
|
|
476
|
+
if (purlReachabilityMap[apkg.purl]) {
|
|
477
|
+
addDosaiReachabilityProperties(apkg, purlReachabilityMap[apkg.purl]);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
459
480
|
return pkgList;
|
|
460
481
|
}
|
|
461
482
|
|
|
@@ -248,33 +248,178 @@ export async function prefetchJson(requests, options = {}) {
|
|
|
248
248
|
* - a non-JSON `responseType`, since the envelope has nowhere to put HTML;
|
|
249
249
|
* - request-specific `headers`, since cdxrs derives its own Authorization from
|
|
250
250
|
* `GITHUB_TOKEN` and would silently drop any other credential, turning an
|
|
251
|
-
* authenticated lookup into an anonymous one
|
|
251
|
+
* authenticated lookup into an anonymous one;
|
|
252
|
+
* - anything addressed to a host whose rate policy only one transport knows.
|
|
253
|
+
* crates.io's crawler policy allows one request per second with no
|
|
254
|
+
* parallelism, and package.elm-lang.org has a JS-only allowance because
|
|
255
|
+
* rate.rs carries no row for it. Both transports keep separate gates, so a
|
|
256
|
+
* batch split across the two would issue twice the agreed rate; the JS pool
|
|
257
|
+
* owns these hosts so there is one gate to reason about.
|
|
252
258
|
*
|
|
253
259
|
* The pool applies the same global and per-host concurrency and rate policy, so
|
|
254
260
|
* the batching win is unaffected; only the on-disk cache is skipped.
|
|
255
261
|
*
|
|
256
|
-
* @param {{responseType?: string, headers?: Object}} request
|
|
262
|
+
* @param {{url?: string, responseType?: string, headers?: Object}} request
|
|
257
263
|
* @returns {boolean}
|
|
258
264
|
*/
|
|
259
265
|
function cdxrsCanServe(request) {
|
|
260
266
|
if (request.responseType && request.responseType !== "json") {
|
|
261
267
|
return false;
|
|
262
268
|
}
|
|
269
|
+
if (isSingleGateHost(extractHost(request.url))) {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
263
272
|
return !request.headers || !Object.keys(request.headers).length;
|
|
264
273
|
}
|
|
265
274
|
|
|
275
|
+
/**
|
|
276
|
+
* Whether a hostname belongs to crates.io, including its index and CDN hosts.
|
|
277
|
+
*
|
|
278
|
+
* @param {string|null} host Hostname, or null when the URL did not parse.
|
|
279
|
+
* @returns {boolean}
|
|
280
|
+
*/
|
|
281
|
+
export function isCratesHost(host) {
|
|
282
|
+
if (!host) {
|
|
283
|
+
return false;
|
|
284
|
+
}
|
|
285
|
+
const lower = host.toLowerCase();
|
|
286
|
+
return lower === "crates.io" || lower.endsWith(".crates.io");
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Whether a host's rate policy lives in the JS table alone and therefore needs
|
|
291
|
+
* every request to pass through the one gate the JS pool holds.
|
|
292
|
+
*
|
|
293
|
+
* @param {string|null} host Hostname, or null when the URL did not parse.
|
|
294
|
+
* @returns {boolean}
|
|
295
|
+
*/
|
|
296
|
+
export function isSingleGateHost(host) {
|
|
297
|
+
if (!host) {
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
return isCratesHost(host) || host.toLowerCase() === "package.elm-lang.org";
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Most URLs sent to one `cdxrs fetch` run. The envelope comes back as a single
|
|
305
|
+
* JSON string and V8 caps strings at ~512 MB, while a full npm packument is a
|
|
306
|
+
* few hundred KB on average and tens of MB for the largest packages. One run
|
|
307
|
+
* per workspace therefore outgrew the limit on large pnpm workspaces (issue
|
|
308
|
+
* 4393); runs of this size stay far below it for any realistic mix.
|
|
309
|
+
*/
|
|
310
|
+
const RUST_FETCH_CHUNK_SIZE = 250;
|
|
311
|
+
|
|
266
312
|
/**
|
|
267
313
|
* Run the batch through the `cdxrs fetch` subprocess and translate its
|
|
268
314
|
* envelope into the shared result shape.
|
|
269
315
|
*
|
|
316
|
+
* The URLs go out in chunks of {@link RUST_FETCH_CHUNK_SIZE}, one run at a time
|
|
317
|
+
* so each host still sees a single rate gate. A chunk whose envelope is still
|
|
318
|
+
* too large to deliver is split in half and retried, which is cheap because
|
|
319
|
+
* the first attempt filled the cdxrs disk cache. A chunk that fails for any
|
|
320
|
+
* other reason falls back to the JS pool on its own, so one bad run never
|
|
321
|
+
* costs the rest of the batch its Rust fetch.
|
|
322
|
+
*
|
|
270
323
|
* @param {Array<{url: string, accept?: string, authRealm?: string}>} unique
|
|
271
324
|
* Already deduplicated by URL.
|
|
272
325
|
* @param {Object} options Options with `timeoutMs`.
|
|
273
326
|
* @returns {Promise<Map<string, BatchEntry>>}
|
|
274
327
|
*/
|
|
275
328
|
async function rustBatchFetch(unique, options) {
|
|
329
|
+
const pending = [];
|
|
330
|
+
for (let i = 0; i < unique.length; i += RUST_FETCH_CHUNK_SIZE) {
|
|
331
|
+
pending.push(unique.slice(i, i + RUST_FETCH_CHUNK_SIZE));
|
|
332
|
+
}
|
|
333
|
+
const results = new Map();
|
|
334
|
+
const chunkStats = [];
|
|
335
|
+
while (pending.length) {
|
|
336
|
+
const chunk = pending.shift();
|
|
337
|
+
const outcome = await rustFetchChunk(chunk, options);
|
|
338
|
+
if (outcome.reason === "stdout-too-large" && chunk.length > 1) {
|
|
339
|
+
const middle = Math.ceil(chunk.length / 2);
|
|
340
|
+
pending.unshift(chunk.slice(0, middle), chunk.slice(middle));
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (!outcome.results) {
|
|
344
|
+
// Not "binary absent" (that never reaches here), so fall through to the
|
|
345
|
+
// JS pool rather than returning nothing: a transient cdxrs crash must
|
|
346
|
+
// not take this chunk's concurrency with it.
|
|
347
|
+
for (const [url, entry] of await jsBatchFetch(chunk)) {
|
|
348
|
+
results.set(url, entry);
|
|
349
|
+
}
|
|
350
|
+
chunkStats.push(_lastStats);
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
for (const [url, entry] of outcome.results) {
|
|
354
|
+
results.set(url, entry);
|
|
355
|
+
}
|
|
356
|
+
chunkStats.push(outcome.stats);
|
|
357
|
+
}
|
|
358
|
+
_lastStats = combineBatchStats(chunkStats);
|
|
359
|
+
if (DEBUG_MODE && _lastStats) {
|
|
360
|
+
const s = _lastStats;
|
|
361
|
+
console.log(
|
|
362
|
+
`cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
return results;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Add up the stats of the runs that served one batch. Numeric fields sum
|
|
370
|
+
* because the runs are sequential, except peak concurrency, which is the
|
|
371
|
+
* widest run. Per-host entries under `hosts` are combined the same way.
|
|
372
|
+
*
|
|
373
|
+
* @param {Array<Object|null|undefined>} parts Stats from each run.
|
|
374
|
+
* @returns {Object|null} Combined stats, or null when no run reported any.
|
|
375
|
+
*/
|
|
376
|
+
function combineBatchStats(parts) {
|
|
377
|
+
const reported = parts.filter(Boolean);
|
|
378
|
+
if (!reported.length) {
|
|
379
|
+
return null;
|
|
380
|
+
}
|
|
381
|
+
if (reported.length === 1) {
|
|
382
|
+
return reported[0];
|
|
383
|
+
}
|
|
384
|
+
const combined = addStats({}, reported);
|
|
385
|
+
const hosts = {};
|
|
386
|
+
for (const part of reported) {
|
|
387
|
+
for (const [host, entry] of Object.entries(part.hosts || {})) {
|
|
388
|
+
hosts[host] = addStats(hosts[host] || {}, [entry]);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (Object.keys(hosts).length) {
|
|
392
|
+
combined.hosts = hosts;
|
|
393
|
+
}
|
|
394
|
+
return combined;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function addStats(target, parts) {
|
|
398
|
+
for (const part of parts) {
|
|
399
|
+
for (const [key, value] of Object.entries(part)) {
|
|
400
|
+
if (typeof value !== "number") {
|
|
401
|
+
continue;
|
|
402
|
+
}
|
|
403
|
+
target[key] =
|
|
404
|
+
key === "peakConcurrency"
|
|
405
|
+
? Math.max(target[key] || 0, value)
|
|
406
|
+
: (target[key] || 0) + value;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return target;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Run one chunk through `cdxrs fetch`.
|
|
414
|
+
*
|
|
415
|
+
* @param {Array<{url: string, accept?: string, authRealm?: string}>} chunk
|
|
416
|
+
* @param {Object} options Options with `timeoutMs`.
|
|
417
|
+
* @returns {Promise<{results?: Map<string, BatchEntry>, stats?: Object, reason?: string}>}
|
|
418
|
+
* `results` when the run succeeded; otherwise the reason it did not.
|
|
419
|
+
*/
|
|
420
|
+
async function rustFetchChunk(chunk, options) {
|
|
276
421
|
const payload = { requests: [] };
|
|
277
|
-
for (const request of
|
|
422
|
+
for (const request of chunk) {
|
|
278
423
|
payload.requests.push({
|
|
279
424
|
id: request.url,
|
|
280
425
|
url: request.url,
|
|
@@ -291,20 +436,19 @@ async function rustBatchFetch(unique, options) {
|
|
|
291
436
|
if (!ok) {
|
|
292
437
|
if (DEBUG_MODE && reason !== "disabled" && reason !== "binary-not-found") {
|
|
293
438
|
console.log(
|
|
294
|
-
|
|
439
|
+
reason === "stdout-too-large" && chunk.length > 1
|
|
440
|
+
? `cdxrs fetch output for ${chunk.length} url(s) is too large; retrying in two halves.`
|
|
441
|
+
: `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
|
|
295
442
|
);
|
|
296
443
|
}
|
|
297
|
-
|
|
298
|
-
// to the JS pool rather than returning an empty map, so a transient cdxrs
|
|
299
|
-
// crash does not take the whole batch's concurrency with it.
|
|
300
|
-
return await jsBatchFetch(unique);
|
|
444
|
+
return { reason };
|
|
301
445
|
}
|
|
302
446
|
|
|
303
447
|
let envelope;
|
|
304
448
|
try {
|
|
305
449
|
envelope = JSON.parse(stdout);
|
|
306
450
|
} catch (_err) {
|
|
307
|
-
return
|
|
451
|
+
return { reason: "malformed-envelope" };
|
|
308
452
|
}
|
|
309
453
|
if (envelope?.schemaVersion !== BATCH_SCHEMA_VERSION) {
|
|
310
454
|
// A binary from a different generation of this protocol is not something
|
|
@@ -314,10 +458,9 @@ async function rustBatchFetch(unique, options) {
|
|
|
314
458
|
`cdxrs fetch envelope version ${envelope?.schemaVersion} != ${BATCH_SCHEMA_VERSION}; falling back to the JS batch pool.`,
|
|
315
459
|
);
|
|
316
460
|
}
|
|
317
|
-
return
|
|
461
|
+
return { reason: "version-mismatch" };
|
|
318
462
|
}
|
|
319
463
|
|
|
320
|
-
_lastStats = envelope.stats || null;
|
|
321
464
|
const results = new Map();
|
|
322
465
|
for (const result of envelope.results || []) {
|
|
323
466
|
if (!result?.id) {
|
|
@@ -335,14 +478,7 @@ async function rustBatchFetch(unique, options) {
|
|
|
335
478
|
const definite = status >= 400 && status < 500 && status !== 429;
|
|
336
479
|
results.set(result.id, { ok: false, status, definite });
|
|
337
480
|
}
|
|
338
|
-
|
|
339
|
-
if (DEBUG_MODE && envelope.stats) {
|
|
340
|
-
const s = envelope.stats;
|
|
341
|
-
console.log(
|
|
342
|
-
`cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
|
|
343
|
-
);
|
|
344
|
-
}
|
|
345
|
-
return results;
|
|
481
|
+
return { results, stats: envelope.stats || null };
|
|
346
482
|
}
|
|
347
483
|
|
|
348
484
|
/**
|
|
@@ -700,6 +836,41 @@ async function fetchOne(request, gauge) {
|
|
|
700
836
|
}
|
|
701
837
|
}
|
|
702
838
|
|
|
839
|
+
/**
|
|
840
|
+
* Run a caller's own request under the same per-host gate the batch pool uses.
|
|
841
|
+
*
|
|
842
|
+
* Not every registry lookup goes through the pool: a caller that misses the
|
|
843
|
+
* prefetch, or runs with prefetching disabled, issues `cdxgenAgent.get`
|
|
844
|
+
* directly. Those requests reach the same host and count against the same
|
|
845
|
+
* published budget, so they have to queue behind the same limiter rather than
|
|
846
|
+
* a second one — crates.io's one-request-per-second policy is not honoured by
|
|
847
|
+
* two independent gates that each allow one per second.
|
|
848
|
+
*
|
|
849
|
+
* The per-host semaphore and limiter are module state, so this shares one gate
|
|
850
|
+
* per host for the life of the process.
|
|
851
|
+
*
|
|
852
|
+
* @template T
|
|
853
|
+
* @param {string} url The URL about to be requested.
|
|
854
|
+
* @param {() => Promise<T>} issue Issues the request and resolves its result.
|
|
855
|
+
* @returns {Promise<T>} Whatever `issue` resolves to.
|
|
856
|
+
*/
|
|
857
|
+
export async function withHostRateLimit(url, issue) {
|
|
858
|
+
const host = extractHost(url);
|
|
859
|
+
if (!host) {
|
|
860
|
+
return await issue();
|
|
861
|
+
}
|
|
862
|
+
const controls = getHostControls(host);
|
|
863
|
+
const releaseGlobal = await _globalSemaphore.acquire();
|
|
864
|
+
const releaseHost = await controls.sem.acquire();
|
|
865
|
+
try {
|
|
866
|
+
await controls.limiter.wait();
|
|
867
|
+
return await issue();
|
|
868
|
+
} finally {
|
|
869
|
+
releaseHost();
|
|
870
|
+
releaseGlobal();
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
703
874
|
/**
|
|
704
875
|
* Dedupe key for an in-flight request: the URL, plus everything that changes
|
|
705
876
|
* which response the caller gets back. That is the headers which change what
|
|
@@ -2,18 +2,21 @@
|
|
|
2
2
|
* Per-host rate policy for batched registry fetches.
|
|
3
3
|
*
|
|
4
4
|
* This is the single source of truth for the JS transport's concurrency and
|
|
5
|
-
* minimum-interval decisions.
|
|
5
|
+
* minimum-interval decisions. Every number is shared with the Rust transport
|
|
6
6
|
* (`cdxgen-plugins-bin/thirdparty/cdxrs/src/fetch/rate.rs`); a change to one
|
|
7
7
|
* side must be reflected in the other, and `fetchRate.poku.js` pins them so a
|
|
8
|
-
* drift is visible.
|
|
8
|
+
* drift is visible. crates.io and package.elm-lang.org are routed so that their
|
|
9
|
+
* requests never reach the Rust transport, which makes this table the one that
|
|
10
|
+
* governs them in practice — their rows are mirrored there regardless, because
|
|
11
|
+
* two tables that disagree are a bug waiting for the routing to change.
|
|
9
12
|
*
|
|
10
13
|
* Why a per-host table rather than a single global interval: a single-registry
|
|
11
14
|
* project — which is to say, almost every npm project — has exactly one host,
|
|
12
15
|
* so a conservative per-host cap *is* the global cap. Applying crates.io's
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
16
|
+
* one-per-second policy to every host would cap an npm run at one request a
|
|
17
|
+
* second, orders of magnitude slower than that registry asks for. Hosts that
|
|
18
|
+
* publish a limit get an interval; hosts that do not are bounded by the
|
|
19
|
+
* per-host concurrency cap alone, and by whatever `Retry-After` they send.
|
|
17
20
|
*/
|
|
18
21
|
|
|
19
22
|
import { readEnvironmentVariable } from "../core/activity.js";
|
|
@@ -48,13 +51,18 @@ import { readEnvironmentVariable } from "../core/activity.js";
|
|
|
48
51
|
* Suffix match is used so `index.crates.io` inherits `crates.io`.
|
|
49
52
|
*/
|
|
50
53
|
const HOST_POLICIES = [
|
|
51
|
-
// crates.io's
|
|
54
|
+
// crates.io's crawler policy asks for at most one request per second, made
|
|
55
|
+
// serially. It is a condition of use rather than a throughput hint, so the
|
|
56
|
+
// interval is a full second and the concurrency is 1: with more than one
|
|
57
|
+
// request in flight the gate would still average a second apiece but would
|
|
58
|
+
// deliver them in bursts, which is the thing the policy rules out. The
|
|
59
|
+
// budget is per IP and does not improve with a token.
|
|
52
60
|
{
|
|
53
61
|
host: "crates.io",
|
|
54
|
-
anonInterval:
|
|
55
|
-
anonConcurrency:
|
|
56
|
-
authInterval:
|
|
57
|
-
authConcurrency:
|
|
62
|
+
anonInterval: 1000,
|
|
63
|
+
anonConcurrency: 1,
|
|
64
|
+
authInterval: 1000,
|
|
65
|
+
authConcurrency: 1,
|
|
58
66
|
},
|
|
59
67
|
// GitHub: 60 req/h anonymous, 5000 req/h authenticated. The limit is an
|
|
60
68
|
// hourly budget rather than a rate, so a batch of a few hundred lookups is
|
|
@@ -88,6 +96,61 @@ const HOST_POLICIES = [
|
|
|
88
96
|
authInterval: 250,
|
|
89
97
|
authConcurrency: 4,
|
|
90
98
|
},
|
|
99
|
+
// package.elm-lang.org publishes no limit: its robots.txt carries no
|
|
100
|
+
// `Crawl-delay` and the registry has no terms of use or documented API. What
|
|
101
|
+
// it does have is a single community-run origin rather than a CDN, so it is
|
|
102
|
+
// held to the same gentle allowance as the other unpublished host in this
|
|
103
|
+
// table instead of the CDN default. Like crates.io, its requests are kept off
|
|
104
|
+
// the Rust transport to leave one gate in charge; rate.rs still carries the
|
|
105
|
+
// same row so the two tables state one policy.
|
|
106
|
+
// Maven Central publishes no numeric limit. What it publishes instead is a
|
|
107
|
+
// policy: consumption limits are enforced against aggregate traffic, the
|
|
108
|
+
// thresholds are unpublished and moving downward, and the named offenders are
|
|
109
|
+
// "CI/CD platforms, security scanners, and other services that generate
|
|
110
|
+
// outsized traffic" — which is cdxgen's exact job description. A block is an
|
|
111
|
+
// escalating ban of up to 30 days, not a per-request 429, so the cost of
|
|
112
|
+
// being slightly too fast is losing Maven resolution for a month rather than
|
|
113
|
+
// retrying a second later. That asymmetry, not a published number, is what
|
|
114
|
+
// sets these rows.
|
|
115
|
+
//
|
|
116
|
+
// repo1 is the artifact store: Cloudflare-fronted (`cf-cache-status` on every
|
|
117
|
+
// response), and a POM fetch that hits the edge cache never reaches Sonatype.
|
|
118
|
+
// It gets a real allowance — 100 ms and 8 in flight, so ~10 req/s sustained —
|
|
119
|
+
// which is half the CDN default and orders of magnitude below anything that
|
|
120
|
+
// would show up in the top few percent of Central's traffic.
|
|
121
|
+
{
|
|
122
|
+
host: "repo1.maven.org",
|
|
123
|
+
anonInterval: 100,
|
|
124
|
+
anonConcurrency: 8,
|
|
125
|
+
authInterval: 100,
|
|
126
|
+
authConcurrency: 8,
|
|
127
|
+
},
|
|
128
|
+
// The search API is the opposite: a Solr query against the origin, one
|
|
129
|
+
// request per unknown jar hash, cacheable by nobody. cdxgen already treats it
|
|
130
|
+
// as fragile — a single timeout or 429 disables jar search for the rest of
|
|
131
|
+
// the run — so the gate is a serial 500 ms (~2 req/s). A jar sweep stays
|
|
132
|
+
// usable while staying far below anything worth throttling.
|
|
133
|
+
{
|
|
134
|
+
host: "search.maven.org",
|
|
135
|
+
anonInterval: 500,
|
|
136
|
+
anonConcurrency: 1,
|
|
137
|
+
authInterval: 500,
|
|
138
|
+
authConcurrency: 1,
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
host: "central.sonatype.com",
|
|
142
|
+
anonInterval: 500,
|
|
143
|
+
anonConcurrency: 1,
|
|
144
|
+
authInterval: 500,
|
|
145
|
+
authConcurrency: 1,
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
host: "package.elm-lang.org",
|
|
149
|
+
anonInterval: 250,
|
|
150
|
+
anonConcurrency: 4,
|
|
151
|
+
authInterval: 250,
|
|
152
|
+
authConcurrency: 4,
|
|
153
|
+
},
|
|
91
154
|
];
|
|
92
155
|
|
|
93
156
|
/**
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { cdxgenAgent, readEnvironmentVariable } from "../core/activity.js";
|
|
2
|
+
import { getUserAgent } from "../core/userAgent.js";
|
|
2
3
|
import { getOriginUrl } from "./envcontext.js";
|
|
3
4
|
import { prefetchJson } from "./fetchBatch.js";
|
|
4
5
|
|
|
@@ -123,7 +124,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
|
|
|
123
124
|
const githubHeaders = {
|
|
124
125
|
Accept: "application/vnd.github+json",
|
|
125
126
|
Authorization: `Bearer ${githubToken}`,
|
|
126
|
-
"User-Agent": "
|
|
127
|
+
"User-Agent": getUserAgent("api.github.com"),
|
|
127
128
|
};
|
|
128
129
|
const pullsUrlFor = (commit) =>
|
|
129
130
|
`https://api.github.com/repos/${owner}/${repo}/commits/${commit.hash}/pulls`;
|
|
@@ -214,7 +215,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
|
|
|
214
215
|
const commitsToQuery = aiCommits.slice(0, 5);
|
|
215
216
|
const gitlabHeaders = {
|
|
216
217
|
"PRIVATE-TOKEN": gitlabToken,
|
|
217
|
-
"User-Agent": "
|
|
218
|
+
"User-Agent": getUserAgent("gitlab.com"),
|
|
218
219
|
};
|
|
219
220
|
const mrUrlFor = (commit) =>
|
|
220
221
|
`https://gitlab.com/api/v4/projects/${projectEncoded}/repository/commits/${commit.hash}/merge_requests`;
|
|
@@ -45,10 +45,11 @@ const JVM_BUILD_TOOL_CANDIDATES = ["maven", "gradle", "sbt", "scala"];
|
|
|
45
45
|
|
|
46
46
|
/**
|
|
47
47
|
* Anchored validator for sdkman version identifiers such as `3.9.9`,
|
|
48
|
-
* `4.0.0-rc-5`, `8.14`,
|
|
49
|
-
* shell command by the sdkman installer, so every
|
|
50
|
-
* safe token: it must start with a digit and may
|
|
51
|
-
* letters, dots, and
|
|
48
|
+
* `4.0.0-rc-5`, `8.14`, `21.0.7-tem`, or `26.0.2+1.1-tem`. Versions are
|
|
49
|
+
* interpolated into a shell command by the sdkman installer, so every
|
|
50
|
+
* identifier must be a single safe token: it must start with a digit and may
|
|
51
|
+
* only contain digits, ascii letters, dots, hyphens, and plus signs (`+` has
|
|
52
|
+
* no shell meaning and sdkman uses it in Temurin respin identifiers).
|
|
52
53
|
*
|
|
53
54
|
* Every separator must be followed by at least one alphanumeric character, and
|
|
54
55
|
* a single repeated group covers both the dotted numeric part and the vendor
|
|
@@ -63,7 +64,7 @@ export function isValidSdkmanVersion(version) {
|
|
|
63
64
|
typeof version === "string" &&
|
|
64
65
|
version.length > 0 &&
|
|
65
66
|
version.length <= MAX_VERSION_TOKEN_LENGTH &&
|
|
66
|
-
/^\d+(?:[
|
|
67
|
+
/^\d+(?:[-.+][0-9A-Za-z]+)*$/.test(version)
|
|
67
68
|
);
|
|
68
69
|
}
|
|
69
70
|
|