@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/inventory/fetchBatch.js

@@ -10,11 +10,13 @@
  * read instead of the network.
  *
  * There is one policy layer (`fetchRate.js`, mirrored by the Rust `rate.rs`)
- * and two transports. When the `cdxrs` binary is available it is preferred,
- * because it brings an on-disk conditional cache (D26) the JS pool does not
- * have. When it is absent, disabled, or unusable, the JS pool runs every
- * request through `cdxgenAgent` so the secure-mode host allowlist, the
- * activity recorder and the test cassette interceptor all still apply.
+ * and two transports. When the `cdxrs` binary is available it is preferred for
+ * the JSON it can carry. When it is absent, disabled, or unusable, and whenever
+ * a dry run, secure mode or a host allowlist is in force, the JS pool runs
+ * every request through `cdxgenAgent` so those policies, the activity recorder
+ * and the test cassette interceptor all still apply. The JS pool keeps the same
+ * on-disk conditional cache as cdxrs (`fetchDiskCache.js`), in the same
+ * directory and layout.
  *
  * Two properties are deliberate and load-bearing:
  *
@@ -27,22 +29,35 @@
  *    JS on all three registries it covered.
  *
  * 2. **Fallback is per URL, not per run.** A URL that the batch could not
- *    resolve for a transport reason is simply absent from the map, and the
- *    caller's own `cdxgenAgent.get` runs for it exactly as before. A URL that
- *    resolved to a definite HTTP error is recorded as such, so the caller does
- *    not re-request it only to get the same 404.
+ *    resolve for a transport reason is left to the caller, whose own request
+ *    goes through {@link gatedGet} and the same per-host gate. A URL the server
+ *    answered with an error status, after the batch's retries, is recorded as
+ *    such, so the caller does not ask again only to get the same answer, and a
+ *    404 or 410 is remembered for the rest of the run.
  */
 
+import { createHash } from "node:crypto";
+
 import {
   BLOCKED_HOST_ERROR_CODE,
   cdxgenAgent,
   DEBUG_MODE,
   DRY_RUN_ERROR_CODE,
+  isDryRun,
+  isSecureMode,
   readEnvironmentVariable,
 } from "../core/activity.js";
 import { isLedgerEnabled, recordDegradation } from "../core/buildLedger.js";
+import { registerRunReset } from "../core/runState.js";
 import { resolveCacheDir } from "./cacheDir.js";
 import { cdxrsAvailable, runCdxrs } from "./cdxrs.js";
+import {
+  diskCacheable,
+  diskCacheGet,
+  diskCachePut,
+  diskCachePutMiss,
+  diskCacheRefresh,
+} from "./fetchDiskCache.js";
 import {
   credentialsFor,
   DEFAULT_GLOBAL_CONCURRENCY,
@@ -86,7 +101,7 @@ let _available = false;
 // these guarantee one request per unique URL under concurrency, which the
 // serial path got for free from `await`-in-a-loop and which the in-memory
 // `responseCache` cannot provide because it is populated only after a response
-// arrives (and is disabled entirely under `CDXGEN_NO_CACHE`).
+// arrives.
 // ---------------------------------------------------------------------------
 
 /** @type {Map<string, Promise<BatchEntry>>} Keyed by `${url}|${accept}|${authRealm}`. */
@@ -100,6 +115,299 @@ const _hostLimiters = new Map();
 
 const _globalSemaphore = makeSemaphore(DEFAULT_GLOBAL_CONCURRENCY);
 
+/**
+ * Hosts that rate-limit by client IP and escalate the block when it keeps
+ * receiving requests. Maven Central's documented blocks grow to as long as 30
+ * days for sustained over-use, so a 429 from any of these pauses every further
+ * request to that host instead of being retried.
+ */
+const CIRCUIT_BREAKER_HOSTS = new Map([
+  ["repo1.maven.org", "maven-central"],
+  ["repo.maven.apache.org", "maven-central"],
+  ["search.maven.org", "maven-search"],
+  ["central.sonatype.com", "maven-search"],
+]);
+
+/** How long a paused host stays paused when the server sends no Retry-After. */
+const DEFAULT_CIRCUIT_OPEN_MS = 10 * 60 * 1000;
+
+/** Host to the time its pause ends. */
+const _openCircuits = new Map();
+
+/**
+ * Requests a host may answer with HTTP 429, after their retries, before it is
+ * paused. Maven Central and its search are paused on the first.
+ */
+const RATE_LIMITED_REQUESTS_BEFORE_PAUSE = 3;
+
+/** Host to the number of its requests that ended in HTTP 429 this run. */
+const _rateLimitedRequests = new Map();
+
+/**
+ * Requests that answered 404 or 410, so that no later pass of the same run asks
+ * again. Keyed by {@link missKey}, with the time the entry lapses, so that a
+ * process that never resets its run state still sees new releases.
+ */
+const _definiteMisses = new Map();
+const DEFINITE_MISS_TTL_MS = 60 * 60 * 1000;
+
+registerRunReset("fetch:misses-and-rate-limits", () => {
+  _definiteMisses.clear();
+  _rateLimitedRequests.clear();
+});
+
+/**
+ * Whether requests to a host are paused.
+ *
+ * @param {string|null} host Hostname.
+ * @returns {boolean}
+ */
+export function isHostCircuitOpen(host) {
+  if (!host) {
+    return false;
+  }
+  const until = _openCircuits.get(host.toLowerCase());
+  if (until === undefined) {
+    return false;
+  }
+  if (until > Date.now()) {
+    return true;
+  }
+  _openCircuits.delete(host.toLowerCase());
+  return false;
+}
+
+/**
+ * Pause requests to a host, for at least {@link DEFAULT_CIRCUIT_OPEN_MS} or for
+ * as long as the server asked. Warns once per pause.
+ *
+ * @param {string} host Hostname.
+ * @param {{delayMs?: number|null, cause?: string, quiet?: boolean, minimumMs?: number}} [details]
+ *   Server-supplied delay, a short description of what happened, whether to
+ *   pause without a warning or a degradation record (for failures that are not
+ *   the host's doing, such as a dry run or an allowlist block), and the
+ *   shortest pause, which a host that says exactly when it recovers sets to 0.
+ * @returns {void}
+ */
+export function openHostCircuit(
+  host,
+  { delayMs, cause, quiet, minimumMs = DEFAULT_CIRCUIT_OPEN_MS } = {},
+) {
+  if (!host) {
+    return;
+  }
+  const key = host.toLowerCase();
+  const wasOpen = isHostCircuitOpen(key);
+  // A host that said how long to wait is paused that long, but not for less
+  // than `minimumMs`. One that did not is paused for the default.
+  const pauseMs = delayMs
+    ? Math.max(delayMs, minimumMs)
+    : DEFAULT_CIRCUIT_OPEN_MS;
+  const until = Date.now() + pauseMs;
+  _openCircuits.set(key, Math.max(until, _openCircuits.get(key) || 0));
+  if (wasOpen || quiet) {
+    return;
+  }
+  const minutes = Math.ceil((_openCircuits.get(key) - Date.now()) / 60000);
+  const what = cause || "rate limited this machine";
+  const breaker = CIRCUIT_BREAKER_HOSTS.get(key);
+  if (breaker === "maven-search") {
+    console.warn(
+      `${host} ${what}, so jar identification by hash skips it for the next ${minutes} minute(s). Set SEARCH_MAVEN_ORG=false to turn these lookups off.`,
+    );
+  } else if (isGitHubApiHost(key)) {
+    console.warn(
+      `${host} ${what}, so cdxgen will not call it for the next ${minutes} minute(s).${readEnvironmentVariable("GITHUB_TOKEN") ? "" : " Set GITHUB_TOKEN to raise the limit: https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api"}`,
+    );
+  } else if (breaker === "maven-central") {
+    console.warn(
+      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s) and Maven metadata comes from local caches only. Set MAVEN_CENTRAL_URL to a mirror or repository manager, for example https://maven-central.storage-download.googleapis.com/maven2/, to keep license enrichment working.`,
+    );
+  } else {
+    console.warn(
+      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s). The components it would describe keep the metadata found locally.`,
+    );
+  }
+  recordPolicyDegradationOnce("policy.rate-limited", {
+    ecosystem: "generic",
+    impact: "licenses",
+    detail: `${host} ${what}, so further requests to it were skipped.`,
+  });
+}
+
+/**
+ * Clear every paused host. Tests only.
+ *
+ * @returns {void}
+ */
+export function resetHostCircuits() {
+  _openCircuits.clear();
+  _rateLimitedRequests.clear();
+}
+
+/**
+ * The error a request to a paused host fails with, shaped like an HTTP 429 so
+ * callers treat it the way they treat the response that opened the circuit.
+ *
+ * @param {string} host Hostname.
+ * @returns {Error}
+ */
+function hostCircuitOpenError(host) {
+  const err = new Error(
+    `Response code 429 (requests to ${host} are paused after it rate limited this run)`,
+  );
+  err.name = "HTTPError";
+  err.code = "CDXGEN_HOST_CIRCUIT_OPEN";
+  err.statusCode = 429;
+  err.response = { statusCode: 429, headers: {} };
+  return err;
+}
+
+/**
+ * Open the circuit for a breaker host that answered 429.
+ *
+ * @param {string|null} host Hostname.
+ * @param {number|undefined} status HTTP status.
+ * @param {Object} [headers] Response headers.
+ * @returns {boolean} True when the circuit was opened.
+ */
+function tripOnRateLimit(host, status, headers) {
+  if (
+    status !== 429 ||
+    !host ||
+    !CIRCUIT_BREAKER_HOSTS.has(host.toLowerCase())
+  ) {
+    return false;
+  }
+  openHostCircuit(host, {
+    delayMs: parseRetryAfter({ headers }),
+    cause: "answered HTTP 429 (rate limited)",
+  });
+  return true;
+}
+
+/**
+ * Whether a host is GitHub's REST API, or the GitHub Enterprise API that
+ * `GITHUB_API_URL` names. It reports its quota in response headers rather than
+ * with a 429.
+ *
+ * @param {string|null} host Hostname.
+ * @returns {boolean}
+ */
+function isGitHubApiHost(host) {
+  const lower = host?.toLowerCase();
+  if (!lower) {
+    return false;
+  }
+  if (lower === "api.github.com") {
+    return true;
+  }
+  const configured = readEnvironmentVariable("GITHUB_API_URL");
+  return !!configured && extractHost(configured)?.toLowerCase() === lower;
+}
+
+/**
+ * Pause the GitHub API when it says the quota is used up: a 403 or 429 with no
+ * requests remaining, or a secondary limit that sends Retry-After. GitHub says
+ * when the quota resets, so the pause lasts exactly that long.
+ *
+ * @param {string|null} host Hostname.
+ * @param {number|undefined} status HTTP status.
+ * @param {Object} [headers] Response headers.
+ * @returns {boolean} True when the API was paused.
+ */
+function tripOnGitHubQuota(host, status, headers) {
+  if (!isGitHubApiHost(host) || (status !== 403 && status !== 429)) {
+    return false;
+  }
+  const remaining = headers?.["x-ratelimit-remaining"];
+  if (String(remaining) !== "0" && headers?.["retry-after"] === undefined) {
+    return false;
+  }
+  openHostCircuit(host, {
+    delayMs: parseRetryAfter({ headers }),
+    cause: "reports that its API rate limit is used up",
+    minimumMs: 0,
+  });
+  return true;
+}
+
+/**
+ * Count a request to a host that ended in HTTP 429 once its retries ran out,
+ * and pause the host once it has refused several. Breaker hosts are paused on
+ * the first by {@link tripOnRateLimit}.
+ *
+ * @param {string|null} host Hostname.
+ * @param {Object} [headers] Response headers, for the server's back-off.
+ * @returns {void}
+ */
+function noteRateLimited(host, headers) {
+  if (!host || CIRCUIT_BREAKER_HOSTS.has(host.toLowerCase())) {
+    return;
+  }
+  const key = host.toLowerCase();
+  const count = (_rateLimitedRequests.get(key) || 0) + 1;
+  _rateLimitedRequests.set(key, count);
+  if (count >= RATE_LIMITED_REQUESTS_BEFORE_PAUSE) {
+    openHostCircuit(host, {
+      delayMs: parseRetryAfter({ headers }),
+      cause: `answered HTTP 429 (rate limited) to ${count} requests`,
+    });
+  }
+}
+
+/**
+ * The key a definite miss is remembered under: the URL, and the credential it
+ * was asked with, since a private document can be missing to an anonymous
+ * request only. A credential enters the key as a digest, never as itself.
+ *
+ * @param {string} url Request URL.
+ * @param {{authRealm?: string, headers?: Object}} [request] Request details.
+ * @returns {string}
+ */
+function missKey(url, { authRealm, headers } = {}) {
+  const authorization = headers?.Authorization || headers?.authorization;
+  const credential = authorization
+    ? createHash("sha256").update(String(authorization)).digest("hex")
+    : "";
+  return `${url}|${authRealm || ""}|${credential}`;
+}
+
+/**
+ * The status a request answered with when it was last asked this run, if that
+ * was 404 or 410.
+ *
+ * @param {string} key Key from {@link missKey}.
+ * @returns {number|undefined}
+ */
+function rememberedMiss(key) {
+  const miss = _definiteMisses.get(key);
+  if (!miss) {
+    return undefined;
+  }
+  if (miss.until > Date.now()) {
+    return miss.status;
+  }
+  _definiteMisses.delete(key);
+  return undefined;
+}
+
+/**
+ * Remember a 404 or 410 for the rest of the run.
+ *
+ * @param {string} key Key from {@link missKey}.
+ * @param {number|undefined} status HTTP status.
+ * @returns {void}
+ */
+function rememberIfMissing(key, status) {
+  if (status === 404 || status === 410) {
+    _definiteMisses.set(key, {
+      status,
+      until: Date.now() + DEFINITE_MISS_TTL_MS,
+    });
+  }
+}
+
 /**
  * Statistics from the most recent batch, or null when no batch has run.
  *
@@ -156,6 +464,8 @@ export function resetBatchFetchAvailability() {
  * @property {number} [status] HTTP status, when the server produced one.
  * @property {boolean} [definite] True when the status is a final answer (a 4xx
  *   other than 429) and re-requesting it in JS would be pointless.
+ * @property {Object} [headers] Response headers of a failed request, when the
+ *   JS pool made it. cdxrs reports no headers.
  */
 
 /**
@@ -232,10 +542,57 @@ export async function prefetchJson(requests, options = {}) {
     return empty;
   }
 
-  if (batchFetchAvailable() && unique.every(cdxrsCanServe)) {
-    return await rustBatchFetch(unique, options);
+  // A paused host is not asked again, and neither is a document that answered
+  // 404 or 410 earlier in the run. Those URLs fail at once and definitely, so
+  // callers do not retry them on their own either.
+  const answered = new Map();
+  const runnable = [];
+  for (const request of unique) {
+    const missing = rememberedMiss(missKey(request.url, request));
+    if (missing) {
+      answered.set(request.url, { ok: false, status: missing, definite: true });
+    } else if (isHostCircuitOpen(extractHost(request.url))) {
+      answered.set(request.url, { ok: false, status: 429, definite: true });
+    } else {
+      runnable.push(request);
+    }
+  }
+  if (!runnable.length) {
+    return answered;
+  }
+  const results =
+    cdxrsMayFetch() && batchFetchAvailable() && runnable.every(cdxrsCanServe)
+      ? await rustBatchFetch(runnable, options)
+      : await jsBatchFetch(runnable);
+  for (const request of runnable) {
+    rememberIfMissing(
+      missKey(request.url, request),
+      results.get(request.url)?.status,
+    );
+  }
+  for (const [url, entry] of answered) {
+    results.set(url, entry);
   }
-  return await jsBatchFetch(unique);
+  return results;
+}
+
+/**
+ * Whether cdxrs may make this run's requests at all.
+ *
+ * cdxrs opens its own connections, so the rules cdxgenAgent applies to every
+ * request would not reach them: a dry run makes no request, secure mode allows
+ * only HTTPS and follows no redirect, and `CDXGEN_ALLOWED_HOSTS` names the only
+ * hosts that may be contacted. While any of them is in force the JS pool makes
+ * every request, through cdxgenAgent.
+ *
+ * @returns {boolean}
+ */
+function cdxrsMayFetch() {
+  return (
+    !isDryRun &&
+    !isSecureMode &&
+    !readEnvironmentVariable("CDXGEN_ALLOWED_HOSTS")
+  );
 }
 
 /**
@@ -249,6 +606,8 @@ export async function prefetchJson(requests, options = {}) {
  * - request-specific `headers`, since cdxrs derives its own Authorization from
  *   `GITHUB_TOKEN` and would silently drop any other credential, turning an
  *   authenticated lookup into an anonymous one;
+ * - the GitHub API, whose quota is reported in response headers that the
+ *   envelope does not carry, so a spent quota could not pause it;
  * - anything addressed to a host whose rate policy only one transport knows.
  *   crates.io's crawler policy allows one request per second with no
  *   parallelism, and package.elm-lang.org has a JS-only allowance because
@@ -266,7 +625,8 @@ function cdxrsCanServe(request) {
   if (request.responseType && request.responseType !== "json") {
     return false;
   }
-  if (isSingleGateHost(extractHost(request.url))) {
+  const host = extractHost(request.url);
+  if (isSingleGateHost(host) || isGitHubApiHost(host)) {
     return false;
   }
   return !request.headers || !Object.keys(request.headers).length;
@@ -300,18 +660,126 @@ export function isSingleGateHost(host) {
   return isCratesHost(host) || host.toLowerCase() === "package.elm-lang.org";
 }
 
+/**
+ * Most URLs sent to one `cdxrs fetch` run. The envelope comes back as a single
+ * JSON string and V8 caps strings at ~512 MB, while a full npm packument is a
+ * few hundred KB on average and tens of MB for the largest packages. One run
+ * per workspace therefore outgrew the limit on large pnpm workspaces (issue
+ * 4393); runs of this size stay far below it for any realistic mix.
+ */
+const RUST_FETCH_CHUNK_SIZE = 250;
+
 /**
  * Run the batch through the `cdxrs fetch` subprocess and translate its
  * envelope into the shared result shape.
  *
+ * The URLs go out in chunks of {@link RUST_FETCH_CHUNK_SIZE}, one run at a time
+ * so each host still sees a single rate gate. A chunk whose envelope is still
+ * too large to deliver is split in half and retried, which is cheap because
+ * the first attempt filled the cdxrs disk cache. A chunk that fails for any
+ * other reason falls back to the JS pool on its own, so one bad run never
+ * costs the rest of the batch its Rust fetch.
+ *
  * @param {Array<{url: string, accept?: string, authRealm?: string}>} unique
  *   Already deduplicated by URL.
  * @param {Object} options Options with `timeoutMs`.
  * @returns {Promise<Map<string, BatchEntry>>}
  */
 async function rustBatchFetch(unique, options) {
+  const pending = [];
+  for (let i = 0; i < unique.length; i += RUST_FETCH_CHUNK_SIZE) {
+    pending.push(unique.slice(i, i + RUST_FETCH_CHUNK_SIZE));
+  }
+  const results = new Map();
+  const chunkStats = [];
+  while (pending.length) {
+    const chunk = pending.shift();
+    const outcome = await rustFetchChunk(chunk, options);
+    if (outcome.reason === "stdout-too-large" && chunk.length > 1) {
+      const middle = Math.ceil(chunk.length / 2);
+      pending.unshift(chunk.slice(0, middle), chunk.slice(middle));
+      continue;
+    }
+    if (!outcome.results) {
+      // Not "binary absent" (that never reaches here), so fall through to the
+      // JS pool rather than returning nothing: a transient cdxrs crash must
+      // not take this chunk's concurrency with it.
+      for (const [url, entry] of await jsBatchFetch(chunk)) {
+        results.set(url, entry);
+      }
+      chunkStats.push(_lastStats);
+      continue;
+    }
+    for (const [url, entry] of outcome.results) {
+      results.set(url, entry);
+    }
+    chunkStats.push(outcome.stats);
+  }
+  _lastStats = combineBatchStats(chunkStats);
+  if (DEBUG_MODE && _lastStats) {
+    const s = _lastStats;
+    console.log(
+      `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
+    );
+  }
+  return results;
+}
+
+/**
+ * Add up the stats of the runs that served one batch. Numeric fields sum
+ * because the runs are sequential, except peak concurrency, which is the
+ * widest run. Per-host entries under `hosts` are combined the same way.
+ *
+ * @param {Array<Object|null|undefined>} parts Stats from each run.
+ * @returns {Object|null} Combined stats, or null when no run reported any.
+ */
+function combineBatchStats(parts) {
+  const reported = parts.filter(Boolean);
+  if (!reported.length) {
+    return null;
+  }
+  if (reported.length === 1) {
+    return reported[0];
+  }
+  const combined = addStats({}, reported);
+  const hosts = {};
+  for (const part of reported) {
+    for (const [host, entry] of Object.entries(part.hosts || {})) {
+      hosts[host] = addStats(hosts[host] || {}, [entry]);
+    }
+  }
+  if (Object.keys(hosts).length) {
+    combined.hosts = hosts;
+  }
+  return combined;
+}
+
+function addStats(target, parts) {
+  for (const part of parts) {
+    for (const [key, value] of Object.entries(part)) {
+      if (typeof value !== "number") {
+        continue;
+      }
+      target[key] =
+        key === "peakConcurrency"
+          ? Math.max(target[key] || 0, value)
+          : (target[key] || 0) + value;
+    }
+  }
+  return target;
+}
+
+/**
+ * Run one chunk through `cdxrs fetch`.
+ *
+ * @param {Array<{url: string, accept?: string, authRealm?: string}>} chunk
+ * @param {Object} options Options with `timeoutMs`.
+ * @returns {Promise<{results?: Map<string, BatchEntry>, stats?: Object, reason?: string}>}
+ *   `results` when the run succeeded; otherwise the reason it did not.
+ */
+async function rustFetchChunk(chunk, options) {
   const payload = { requests: [] };
-  for (const request of unique) {
+  for (const request of chunk) {
     payload.requests.push({
       id: request.url,
       url: request.url,
@@ -328,20 +796,19 @@ async function rustBatchFetch(unique, options) {
   if (!ok) {
     if (DEBUG_MODE && reason !== "disabled" && reason !== "binary-not-found") {
       console.log(
-        `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
+        reason === "stdout-too-large" && chunk.length > 1
+          ? `cdxrs fetch output for ${chunk.length} url(s) is too large; retrying in two halves.`
+          : `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
       );
     }
-    // The Rust path failed in a way that is not "binary absent". Fall through
-    // to the JS pool rather than returning an empty map, so a transient cdxrs
-    // crash does not take the whole batch's concurrency with it.
-    return await jsBatchFetch(unique);
+    return { reason };
   }
 
   let envelope;
   try {
     envelope = JSON.parse(stdout);
   } catch (_err) {
-    return await jsBatchFetch(unique);
+    return { reason: "malformed-envelope" };
   }
   if (envelope?.schemaVersion !== BATCH_SCHEMA_VERSION) {
     // A binary from a different generation of this protocol is not something
@@ -351,10 +818,9 @@ async function rustBatchFetch(unique, options) {
         `cdxrs fetch envelope version ${envelope?.schemaVersion} != ${BATCH_SCHEMA_VERSION}; falling back to the JS batch pool.`,
       );
     }
-    return await jsBatchFetch(unique);
+    return { reason: "version-mismatch" };
   }
 
-  _lastStats = envelope.stats || null;
   const results = new Map();
   for (const result of envelope.results || []) {
     if (!result?.id) {
@@ -369,17 +835,17 @@ async function rustBatchFetch(unique, options) {
     // try through the JS agent, which has its own proxy and auth handling.
     const status =
       typeof result.status === "number" ? result.status : undefined;
-    const definite = status >= 400 && status < 500 && status !== 429;
+    // cdxrs has already honoured Retry-After. A breaker host that still says
+    // 429 is paused, and the caller must not ask it again.
+    const tripped = tripOnRateLimit(extractHost(result.id), status, {});
+    if (status === 429 && !tripped) {
+      noteRateLimited(extractHost(result.id), {});
+    }
+    const definite =
+      tripped || (status >= 400 && status < 500 && status !== 429);
     results.set(result.id, { ok: false, status, definite });
   }
-
-  if (DEBUG_MODE && envelope.stats) {
-    const s = envelope.stats;
-    console.log(
-      `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
-    );
-  }
-  return results;
+  return { results, stats: envelope.stats || null };
 }
 
 /**
@@ -548,16 +1014,42 @@ function exponentialBackoffMs(attempt) {
  * @param {RateLimiter} limiter The per-host rate limiter.
  * @returns {Promise<BatchEntry>}
  */
-async function issueWithRetries(request, limiter) {
+async function issueWithRetries(request, limiter, disk = undefined) {
   const host = extractHost(request.url);
-  const options = buildAgentOptions(request, host);
+  const options = {
+    ...buildAgentOptions(request, host),
+    // The limiter is consulted only when the request is about to reach the
+    // network; a response already in the cache does not wait for a slot.
+    hooks: { beforeNetwork: [() => limiter.wait()] },
+  };
+  // A stale disk entry with validators is revalidated, and a 304 keeps it.
+  const stale = disk?.stale;
+  if (stale) {
+    options.headers = {
+      ...options.headers,
+      ...(stale.etag ? { "If-None-Match": stale.etag } : {}),
+      ...(stale.last_modified
+        ? { "If-Modified-Since": stale.last_modified }
+        : {}),
+    };
+    options.cache = false;
+  }
   let attempt = 0;
   while (true) {
     attempt++;
-    await limiter.wait();
+    if (isHostCircuitOpen(host)) {
+      return { ok: false, status: 429, definite: true };
+    }
     let result;
     try {
       const res = await cdxgenAgent.get(request.url, options);
+      if (res.statusCode === 304 && stale) {
+        diskCacheRefresh(disk.key, stale, res.headers);
+        return { ok: true, body: stale.body, status: stale.status };
+      }
+      if (disk && res.statusCode === 200) {
+        diskCachePut(disk.key, res);
+      }
       result = {
         retryable: false,
         delay: null,
@@ -565,6 +1057,14 @@ async function issueWithRetries(request, limiter) {
       };
     } catch (err) {
       result = classifyAgentError(err);
+      const status = err?.response?.statusCode;
+      const headers = err?.response?.headers;
+      if (tripOnRateLimit(host, status, headers)) {
+        return { ok: false, status: 429, definite: true };
+      }
+      if (tripOnGitHubQuota(host, status, headers)) {
+        return { ok: false, status, definite: true, headers };
+      }
     }
     if (result.retryable && attempt <= MAX_RETRIES) {
       const delay =
@@ -581,12 +1081,26 @@ async function issueWithRetries(request, limiter) {
       }
       continue;
     }
+    if (result.entry.status === 429) {
+      noteRateLimited(host, result.headers);
+    }
+    const status = result.entry.status;
+    if (disk && (status === 404 || status === 410)) {
+      diskCachePutMiss(disk.key, status);
+    } else if (stale && !result.entry.ok && (!status || status >= 500)) {
+      // A stale copy is better than none while the server is failing, as in
+      // cdxrs, but a 404 means the document is gone.
+      return { ok: true, body: stale.body, status: stale.status };
+    }
     return result.entry;
   }
 }
 
 /** Policy and connectivity degradations already recorded this run; these describe the run, not one URL. */
 const recordedPolicyDegradations = new Set();
+registerRunReset("fetch:policy-records", () =>
+  recordedPolicyDegradations.clear(),
+);
 
 /**
  * Record a run-level policy or connectivity degradation once per process. A
@@ -660,25 +1174,28 @@ function classifyAgentError(err) {
   }
   const status = err?.response?.statusCode;
   const headers = err?.response?.headers;
+  // The response headers travel with the entry, so a caller can read what
+  // the server said about its limits, as it could from its own request.
   if (status === 429) {
     return {
       retryable: true,
       delay: parseRetryAfter({ headers }),
-      entry: { ok: false, status, definite: false },
+      headers,
+      entry: { ok: false, status, definite: false, headers },
     };
   }
   if (status >= 500 && status < 600) {
     return {
       retryable: true,
       delay: null,
-      entry: { ok: false, status, definite: false },
+      entry: { ok: false, status, definite: false, headers },
     };
   }
   if (status >= 400 && status < 500) {
     return {
       retryable: false,
       delay: null,
-      entry: { ok: false, status, definite: true },
+      entry: { ok: false, status, definite: true, headers },
     };
   }
   // Transport error (timeout, connection reset). The name check mirrors how
@@ -721,6 +1238,25 @@ async function fetchOne(request, gauge) {
   if (!host) {
     return { ok: false, status: undefined, definite: false };
   }
+  // The disk cache answers before any permit is taken, so a cached document
+  // never waits for the host's gate.
+  const diskKey = diskCacheable(request)
+    ? {
+        url: request.url,
+        accept: request.accept,
+        authRealm: request.authRealm,
+        responseType: request.responseType || "json",
+      }
+    : undefined;
+  const cached = diskKey ? diskCacheGet(diskKey) : undefined;
+  if (cached?.state === "fresh") {
+    gauge.cacheHits += 1;
+    return { ok: true, body: cached.entry.body, status: cached.entry.status };
+  }
+  if (cached?.state === "miss") {
+    gauge.cacheHits += 1;
+    return { ok: false, status: cached.entry.status, definite: true };
+  }
   const controls = getHostControls(host);
   const releaseGlobal = await _globalSemaphore.acquire();
   const releaseHost = await controls.sem.acquire();
@@ -729,7 +1265,14 @@ async function fetchOne(request, gauge) {
     gauge.peak = gauge.inFlight;
   }
   try {
-    return await issueWithRetries(request, controls.limiter);
+    return await issueWithRetries(
+      request,
+      controls.limiter,
+      diskKey && {
+        key: diskKey,
+        stale: cached?.state === "stale" ? cached.entry : undefined,
+      },
+    );
   } finally {
     gauge.inFlight -= 1;
     releaseHost();
@@ -750,28 +1293,100 @@ async function fetchOne(request, gauge) {
  * The per-host semaphore and limiter are module state, so this shares one gate
  * per host for the life of the process.
  *
+ * A host paused by its circuit breaker fails at once with a 429-shaped error,
+ * and a 429 from a breaker host pauses it.
+ *
+ * With `deferGate`, `issue` receives request options carrying a
+ * `beforeNetwork` hook and must pass them to `cdxgenAgent`. The rate limiter
+ * then runs only when the request misses the response cache, so a cached
+ * answer does not wait for a slot. Without it, the limiter runs before
+ * `issue`, as before.
+ *
  * @template T
  * @param {string} url The URL about to be requested.
- * @param {() => Promise<T>} issue Issues the request and resolves its result.
+ * @param {(gate: Object) => Promise<T>} issue Issues the request and resolves its result.
+ * @param {{deferGate?: boolean}} [opts]
  * @returns {Promise<T>} Whatever `issue` resolves to.
  */
-export async function withHostRateLimit(url, issue) {
+export async function withHostRateLimit(url, issue, { deferGate } = {}) {
   const host = extractHost(url);
   if (!host) {
-    return await issue();
+    return await issue({});
+  }
+  if (isHostCircuitOpen(host)) {
+    throw hostCircuitOpenError(host);
   }
   const controls = getHostControls(host);
   const releaseGlobal = await _globalSemaphore.acquire();
   const releaseHost = await controls.sem.acquire();
   try {
-    await controls.limiter.wait();
-    return await issue();
+    let gate = {};
+    if (deferGate) {
+      gate = { hooks: { beforeNetwork: [() => controls.limiter.wait()] } };
+    } else {
+      await controls.limiter.wait();
+    }
+    return await issue(gate);
+  } catch (err) {
+    const status = err?.response?.statusCode;
+    const headers = err?.response?.headers;
+    if (
+      !tripOnRateLimit(host, status, headers) &&
+      !tripOnGitHubQuota(host, status, headers) &&
+      status === 429
+    ) {
+      noteRateLimited(host, headers);
+    }
+    throw err;
   } finally {
     releaseHost();
     releaseGlobal();
   }
 }
 
+/**
+ * Request a URL the batch did not answer, the way the pool would have: behind
+ * the per-host gate, failing at once for a document that already answered 404
+ * or 410 this run, and remembering such an answer for the next pass.
+ *
+ * This is the fallback every metadata function uses after a prefetch. Without
+ * it, a URL the batch left open went straight to the network with no rate gate,
+ * and the same miss was asked again on every pass.
+ *
+ * @param {string} url URL to request.
+ * @param {Object} [options] `cdxgenAgent.get` options.
+ * @returns {Promise<Object>} The response.
+ * @throws {Error} The request's own error, or a 404-shaped error for a
+ *   remembered miss, so the caller's `catch` runs as it would for the network.
+ */
+export async function gatedGet(url, options = {}) {
+  const key = missKey(url, options);
+  const missing = rememberedMiss(key);
+  if (missing) {
+    throw httpStatusError(url, missing, {}, "already answered this run");
+  }
+  try {
+    return await withHostRateLimit(
+      url,
+      (gate) =>
+        cdxgenAgent.get(url, {
+          ...options,
+          hooks: {
+            ...options.hooks,
+            beforeNetwork: [
+              ...(options.hooks?.beforeNetwork || []),
+              ...(gate.hooks?.beforeNetwork || []),
+            ],
+          },
+        }),
+      { deferGate: true },
+    );
+  } catch (err) {
+    rememberIfMissing(key, err?.response?.statusCode ?? err?.statusCode);
+    throw err;
+  }
+}
+
 /**
  * Dedupe key for an in-flight request: the URL, plus everything that changes
  * which response the caller gets back. That is the headers which change what
@@ -808,14 +1423,14 @@ async function jsBatchFetch(unique) {
   const results = new Map();
   // Concurrency is counted per batch. A request served from another batch's
   // in-flight promise is counted against that batch, not this one.
-  const gauge = { inFlight: 0, peak: 0 };
+  const gauge = { inFlight: 0, peak: 0, cacheHits: 0 };
   const tasks = unique.map(async (request) => {
     const key = inFlightKey(request);
     // Cross-batch in-flight dedupe. If another batch already has this URL in
     // flight, await its promise rather than issuing a second request. The
     // entry is removed when the promise settles, so a later request for the
-    // same URL (after completion) goes through `responseCache` or, under
-    // `CDXGEN_NO_CACHE`, issues a fresh request — which is correct.
+    // same URL (after completion) goes through `responseCache`, which is
+    // correct.
     let promise = _inFlight.get(key);
     if (!promise) {
       promise = fetchOne(request, gauge);
@@ -846,7 +1461,7 @@ async function jsBatchFetch(unique) {
     unique: unique.length,
     ok,
     failures,
-    cacheHits: 0,
+    cacheHits: gauge.cacheHits,
     elapsedMs: Date.now() - started,
     peakConcurrency: gauge.peak,
   };
@@ -859,6 +1474,26 @@ async function jsBatchFetch(unique) {
   return results;
 }
 
+/**
+ * An error shaped like the agent's `HTTPError`, for an HTTP answer the batch
+ * or the miss memo already holds.
+ *
+ * @param {string} url Request URL.
+ * @param {number} status HTTP status.
+ * @param {Object} [headers] Response headers.
+ * @param {string} [detail] Why no request was made.
+ * @returns {Error}
+ */
+function httpStatusError(url, status, headers = {}, detail = undefined) {
+  const err = new Error(
+    `Request failed with status code ${status}${detail ? ` (${detail})` : ""}`,
+  );
+  err.name = "HTTPError";
+  err.statusCode = status;
+  err.response = { statusCode: status, headers, url };
+  return err;
+}
+
 /**
  * Read a prefetched response, or signal that the caller should fetch it itself.
  *
@@ -866,9 +1501,11 @@ async function jsBatchFetch(unique) {
  * @param {string} url The URL the caller is about to request.
  * @returns {{body: *}|undefined} A response-shaped object when the body is
  *   available, or `undefined` when the caller should issue its own request.
- * @throws {Error} When the batch established a definite HTTP error for this
- *   URL, so that the caller's existing `catch` treats it exactly as it treats a
- *   failed `cdxgenAgent.get`.
+ * @throws {Error} When the server answered with an error status, so that the
+ *   caller's existing `catch` treats it exactly as it treats a failed
+ *   `cdxgenAgent.get`. A 5xx or a 429 is the answer after the batch's own
+ *   retries, and asking once more at once would only add to the load. Only a
+ *   transport failure, with no status, is left to the caller to retry.
  */
 export function prefetchedResponse(prefetched, url) {
   const entry = prefetched?.get(url);
@@ -878,10 +1515,8 @@ export function prefetchedResponse(prefetched, url) {
   if (entry.ok) {
     return { body: entry.body };
   }
-  if (entry.definite) {
-    const err = new Error(`Request failed with status code ${entry.status}`);
-    err.statusCode = entry.status;
-    throw err;
+  if (entry.status !== undefined) {
+    throw httpStatusError(url, entry.status, entry.headers);
   }
   return undefined;
 }