@cdxgen/cdxgen 13.0.1 → 13.2.0

package/lib/inventory/fetchRate.js

@@ -2,18 +2,21 @@
  * Per-host rate policy for batched registry fetches.
  *
  * This is the single source of truth for the JS transport's concurrency and
- * minimum-interval decisions. The numbers are shared with the Rust transport
+ * minimum-interval decisions. Every number is shared with the Rust transport
  * (`cdxgen-plugins-bin/thirdparty/cdxrs/src/fetch/rate.rs`); a change to one
  * side must be reflected in the other, and `fetchRate.poku.js` pins them so a
- * drift is visible.
+ * drift is visible. crates.io and package.elm-lang.org are routed so that their
+ * requests never reach the Rust transport, which makes this table the one that
+ * governs them in practice — their rows are mirrored there regardless, because
+ * two tables that disagree are a bug waiting for the routing to change.
  *
  * Why a per-host table rather than a single global interval: a single-registry
  * project — which is to say, almost every npm project — has exactly one host,
  * so a conservative per-host cap *is* the global cap. Applying crates.io's
- * 250 ms to every host caps an npm run at 4 requests per second, which is
- * slower than the serial JS path once connection reuse is taken into account.
- * Hosts that publish a limit get an interval; hosts that do not are bounded by
- * the per-host concurrency cap alone, and by whatever `Retry-After` they send.
+ * one-per-second policy to every host would cap an npm run at one request a
+ * second, orders of magnitude slower than that registry asks for. Hosts that
+ * publish a limit get an interval; hosts that do not are bounded by the
+ * per-host concurrency cap alone, and by whatever `Retry-After` they send.
  */
 
 import { readEnvironmentVariable } from "../core/activity.js";
@@ -48,13 +51,18 @@ import { readEnvironmentVariable } from "../core/activity.js";
  * Suffix match is used so `index.crates.io` inherits `crates.io`.
  */
 const HOST_POLICIES = [
-  // crates.io's published budget is per IP and does not improve with a token.
+  // crates.io's crawler policy asks for at most one request per second, made
+  // serially. It is a condition of use rather than a throughput hint, so the
+  // interval is a full second and the concurrency is 1: with more than one
+  // request in flight the gate would still average a second apiece but would
+  // deliver them in bursts, which is the thing the policy rules out. The
+  // budget is per IP and does not improve with a token.
   {
     host: "crates.io",
-    anonInterval: 250,
-    anonConcurrency: 4,
-    authInterval: 250,
-    authConcurrency: 4,
+    anonInterval: 1000,
+    anonConcurrency: 1,
+    authInterval: 1000,
+    authConcurrency: 1,
   },
   // GitHub: 60 req/h anonymous, 5000 req/h authenticated. The limit is an
   // hourly budget rather than a rate, so a batch of a few hundred lookups is
@@ -88,6 +96,61 @@ const HOST_POLICIES = [
     authInterval: 250,
     authConcurrency: 4,
   },
+  // package.elm-lang.org publishes no limit: its robots.txt carries no
+  // `Crawl-delay` and the registry has no terms of use or documented API. What
+  // it does have is a single community-run origin rather than a CDN, so it is
+  // held to the same gentle allowance as the other unpublished host in this
+  // table instead of the CDN default. Like crates.io, its requests are kept off
+  // the Rust transport to leave one gate in charge; rate.rs still carries the
+  // same row so the two tables state one policy.
+  // Maven Central publishes no numeric limit. What it publishes instead is a
+  // policy: consumption limits are enforced against aggregate traffic, the
+  // thresholds are unpublished and moving downward, and the named offenders are
+  // "CI/CD platforms, security scanners, and other services that generate
+  // outsized traffic" — which is cdxgen's exact job description. A block is an
+  // escalating ban of up to 30 days, not a per-request 429, so the cost of
+  // being slightly too fast is losing Maven resolution for a month rather than
+  // retrying a second later. That asymmetry, not a published number, is what
+  // sets these rows.
+  //
+  // repo1 is the artifact store: Cloudflare-fronted (`cf-cache-status` on every
+  // response), and a POM fetch that hits the edge cache never reaches Sonatype.
+  // It gets a real allowance — 100 ms and 8 in flight, so ~10 req/s sustained —
+  // which is half the CDN default and orders of magnitude below anything that
+  // would show up in the top few percent of Central's traffic.
+  {
+    host: "repo1.maven.org",
+    anonInterval: 100,
+    anonConcurrency: 8,
+    authInterval: 100,
+    authConcurrency: 8,
+  },
+  // The search API is the opposite: a Solr query against the origin, one
+  // request per unknown jar hash, cacheable by nobody. cdxgen already treats it
+  // as fragile — a single timeout or 429 disables jar search for the rest of
+  // the run — so the gate is a serial 500 ms (~2 req/s). A jar sweep stays
+  // usable while staying far below anything worth throttling.
+  {
+    host: "search.maven.org",
+    anonInterval: 500,
+    anonConcurrency: 1,
+    authInterval: 500,
+    authConcurrency: 1,
+  },
+  {
+    host: "central.sonatype.com",
+    anonInterval: 500,
+    anonConcurrency: 1,
+    authInterval: 500,
+    authConcurrency: 1,
+  },
+  {
+    host: "package.elm-lang.org",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 250,
+    authConcurrency: 4,
+  },
 ];
 
 /**

package/lib/inventory/forgeEnricher.js

@@ -1,4 +1,5 @@
 import { cdxgenAgent, readEnvironmentVariable } from "../core/activity.js";
+import { getUserAgent } from "../core/userAgent.js";
 import { getOriginUrl } from "./envcontext.js";
 import { prefetchJson } from "./fetchBatch.js";
 
@@ -123,7 +124,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
     const githubHeaders = {
       Accept: "application/vnd.github+json",
       Authorization: `Bearer ${githubToken}`,
-      "User-Agent": "cdxgen-oversight",
+      "User-Agent": getUserAgent("api.github.com"),
     };
     const pullsUrlFor = (commit) =>
       `https://api.github.com/repos/${owner}/${repo}/commits/${commit.hash}/pulls`;
@@ -214,7 +215,7 @@ export async function enrichFromForge(dir, aiCommits, options = {}) {
     const commitsToQuery = aiCommits.slice(0, 5);
     const gitlabHeaders = {
       "PRIVATE-TOKEN": gitlabToken,
-      "User-Agent": "cdxgen-oversight",
+      "User-Agent": getUserAgent("gitlab.com"),
     };
     const mrUrlFor = (commit) =>
       `https://gitlab.com/api/v4/projects/${projectEncoded}/repository/commits/${commit.hash}/merge_requests`;