@zeldrisho/pi-web-fetch 0.7.0 → 0.8.0

package/src/fetch.ts

@@ -1,6 +1,7 @@
 import type { IncomingMessage } from "node:http";
 import { awaitWithAbort } from "./abort";
 import { sliceCompleteDocument, type CompleteDocument, type FetchResult } from "./content";
+import { diagnoseExtraction, extractDocumentLinks, hasExtractionWarning } from "./evidence";
 import { extractHtmlToMarkdown } from "./extract";
 import { requestFollowingRedirects, type RedirectDependencies } from "./network-redirects";
 import type { ValidatedTarget } from "./network-policy";
@@ -27,8 +28,21 @@ export function normalizeGitHubRawUrl(rawUrl: string): string {
     const url = new URL(rawUrl);
     if (url.protocol !== "https:") return rawUrl;
     if (url.hostname === "github.com") {
-      if (!url.pathname.includes("/blob/")) return rawUrl;
-      return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
+      if (url.pathname.includes("/blob/")) {
+        return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
+      }
+      // Parse pathname segments: /owner/repo/tree/ref/path
+      const segments = url.pathname.split("/").filter(Boolean);
+      if (segments.length >= 4 && segments[2] === "tree") {
+        const lastSegment = segments[segments.length - 1];
+        if (lastSegment.includes(".")) {
+          // Reconstruct path: /owner/repo/ref/path (remove "tree" segment)
+          const newPath = `/${segments[0]}/${segments[1]}/${segments.slice(3).join("/")}`;
+          return `https://raw.githubusercontent.com${newPath}${url.search}`;
+        }
+        return rawUrl;
+      }
+      return rawUrl;
     }
     if (url.hostname === "gist.github.com") {
       const segments = url.pathname.split("/").filter(Boolean);
@@ -43,37 +57,9 @@ export function normalizeGitHubRawUrl(rawUrl: string): string {
   }
 }
 
-const APP_SHELL_MARKERS = [
-  /please\s+enable\s+javascript/i,
-  /enable\s+javascript/i,
-  // Consent is only treated as an interstitial signal when it appears in a
-  // cookie/consent-banner phrase. A bare "consent" matches ordinary prose
-  // (e.g. privacy articles) and must not flag readable content as a shell.
-  /manage\s+(your\s+)?consent/i,
-  /your\s+(privacy\s+)?consent/i,
-  /consent\s+to\s+(our\s+use\s+of\s+cookies|cookies)/i,
-  /accept\s+(all\s+)?cookies/i,
-  /we\s+use\s+cookies/i,
-  /are\s+you\s+a\s+robot/i,
-  /verify\s+you\s+are\s+human/i,
-  /checking\s+your\s+browser/i,
-  /<title>\s*just\s+a\s+moment/i,
-];
-
-/**
- * Detects pages that are likely app shells, bot walls, or consent interstitials rather than
- * readable content.
- *
- * @param raw - The raw response body
- * @param markdown - The extracted Markdown
- * @returns True when the extracted text is suspiciously sparse relative to the raw page
- */
+/** @deprecated Prefer diagnoseExtraction for explicit extraction-quality signals. */
 export function detectAppShell(raw: string, markdown: string): boolean {
-  if (APP_SHELL_MARKERS.some((marker) => marker.test(raw))) return true;
-  // Require the extracted text to be both absolutely tiny and a very small
-  // fraction of the raw payload, so content-rich pages (e.g. React/Next.js SPAs
-  // whose raw HTML is dominated by inline scripts) are not mistaken for shells.
-  return raw.length > 4000 && markdown.length < 1024 && markdown.length < raw.length * 0.008;
+  return hasExtractionWarning(diagnoseExtraction(raw, markdown));
 }
 
 const REQUEST_TIMEOUT_MS = 20_000;
@@ -225,10 +211,13 @@ async function documentFromResponse(
     }
   } else markdown = raw.trim();
 
-  const shellSuspected =
-    contentType === "text/html" || contentType === "application/xhtml+xml"
-      ? detectAppShell(raw, markdown)
-      : false;
+  const isHtml = contentType === "text/html" || contentType === "application/xhtml+xml";
+  const extractionDiagnostics = isHtml ? diagnoseExtraction(raw, markdown) : undefined;
+  const shellSuspected = extractionDiagnostics
+    ? hasExtractionWarning(extractionDiagnostics)
+    : false;
+  const etag = responseHeader(response, "etag");
+  const lastModified = responseHeader(response, "last-modified");
 
   return {
     url: target.url.toString(),
@@ -237,16 +226,51 @@ async function documentFromResponse(
     title,
     extractor,
     shellSuspected,
+    extractionDiagnostics,
+    links: isHtml ? extractDocumentLinks(raw, target.url) : undefined,
+    validators: etag || lastModified ? { etag, lastModified } : undefined,
+    cachedAt: Date.now(),
     llmsTxtDescribedBy: describedBy,
     markdownAlternateUrl,
   };
 }
 
-export async function fetchCompleteDocument(
+/**
+ * Validates that a URL string is an absolute http or https URL.
+ *
+ * @param value - The URL string to validate
+ * @returns The parsed URL object
+ * @throws If the URL is invalid or not http(s)
+ */
+function assertAbsoluteHttpUrlForFetch(value: string): URL {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`Invalid URL: ${value}`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`web_fetch only supports http(s) URLs: ${value}`);
+  }
+  return url;
+}
+
+/**
+ * Fetches a URL and returns the complete document with extracted content.
+ *
+ * @param rawUrl - The URL to fetch
+ * @param signal - Optional abort signal to cancel the request
+ * @param dependencies - Dependencies for network policy validation, transport, and HTML extraction
+ * @returns The complete document with extracted markdown content
+ * @throws If the request times out, is cancelled, or encounters an error
+ */
+async function fetchDocument(
   rawUrl: string,
   signal: AbortSignal | undefined,
   dependencies: FetchRemoteDependencies,
-): Promise<CompleteDocument> {
+  cached?: CompleteDocument,
+): Promise<{ document: CompleteDocument; revalidated: boolean }> {
+  assertAbsoluteHttpUrlForFetch(normalizeGitHubRawUrl(rawUrl));
   const controller = new AbortController();
   const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
   const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
@@ -257,17 +281,36 @@ export async function fetchCompleteDocument(
   }, timeoutMs);
   const cancel = () => controller.abort();
   signal?.addEventListener("abort", cancel, { once: true });
+  const conditionalHeaders: Record<string, string> = {};
+  if (cached?.validators?.etag) conditionalHeaders["If-None-Match"] = cached.validators.etag;
+  if (cached?.validators?.lastModified)
+    conditionalHeaders["If-Modified-Since"] = cached.validators.lastModified;
 
   try {
     const { target, response } = await requestFollowingRedirects(
       normalizeGitHubRawUrl(rawUrl),
       controller.signal,
-      {
-        validateUrl: dependencies.validateUrl,
-        request: dependencies.request,
-      },
+      dependencies,
+      conditionalHeaders,
     );
-    return await documentFromResponse(target, response, controller.signal, extractHtml);
+    if (response.statusCode === 304 && cached) {
+      response.resume();
+      const etag = responseHeader(response, "etag") ?? cached.validators?.etag;
+      const lastModified =
+        responseHeader(response, "last-modified") ?? cached.validators?.lastModified;
+      return {
+        document: {
+          ...cached,
+          validators: etag || lastModified ? { etag, lastModified } : undefined,
+          cachedAt: Date.now(),
+        },
+        revalidated: true,
+      };
+    }
+    return {
+      document: await documentFromResponse(target, response, controller.signal, extractHtml),
+      revalidated: false,
+    };
   } catch (error) {
     if (timedOut) throw new Error(`web_fetch timed out after ${timeoutMs / 1000} seconds.`);
     if (signal?.aborted) throw new Error("web_fetch was cancelled.");
@@ -278,6 +321,34 @@ export async function fetchCompleteDocument(
   }
 }
 
+export async function fetchCompleteDocument(
+  rawUrl: string,
+  signal: AbortSignal | undefined,
+  dependencies: FetchRemoteDependencies,
+): Promise<CompleteDocument> {
+  return (await fetchDocument(rawUrl, signal, dependencies)).document;
+}
+
+/** Revalidates a stale representation with its ETag and Last-Modified validators. */
+export async function revalidateCompleteDocument(
+  rawUrl: string,
+  cached: CompleteDocument,
+  signal: AbortSignal | undefined,
+  dependencies: FetchRemoteDependencies,
+): Promise<{ document: CompleteDocument; revalidated: boolean }> {
+  return fetchDocument(rawUrl, signal, dependencies, cached);
+}
+
+/**
+ * Fetches a URL and returns a slice of the content starting at the given offset.
+ *
+ * @param rawUrl - The URL to fetch
+ * @param offset - The character offset to start slicing from
+ * @param maxCharacters - The maximum number of characters to return
+ * @param signal - Optional abort signal to cancel the request
+ * @param dependencies - Dependencies for network policy validation, transport, and HTML extraction
+ * @returns The sliced fetch result with content and metadata
+ */
 export async function fetchRemoteContent(
   rawUrl: string,
   offset: number,

package/src/focus.ts

@@ -0,0 +1,125 @@
+const WORD_PATTERN = /[\p{L}\p{N}_+#]+/gu;
+const HEADING_PATTERN = /^#{1,6}\s+\S/;
+
+/** Metadata describing deterministic query-focused extraction. */
+export interface FocusDetails {
+  query: string;
+  matchedSections: number;
+  totalSections: number;
+  omittedSections: number;
+  originalCharacters: number;
+  focusedCharacters: number;
+}
+
+export interface FocusResult {
+  markdown: string;
+  details: FocusDetails;
+}
+
+interface Section {
+  markdown: string;
+  tokens: string[];
+}
+
+function tokens(value: string): string[] {
+  return (value.toLowerCase().match(WORD_PATTERN) ?? []).filter((token) => token.length > 1);
+}
+
+/**
+ * Splits Markdown into sections. Headings stay attached to their following content; a
+ * headingless document falls back to paragraphs so focused extraction remains useful for
+ * plain-text and basic-extractor responses.
+ */
+function splitSections(markdown: string): Section[] {
+  const blocks = markdown
+    .split(/\n{2,}/)
+    .map((block) => block.trim())
+    .filter(Boolean);
+  if (blocks.length === 0) return [];
+
+  const lines = markdown.split(/\r?\n/);
+  const hasHeadings = lines.some((line) => HEADING_PATTERN.test(line));
+  if (!hasHeadings) return blocks.map((block) => ({ markdown: block, tokens: tokens(block) }));
+
+  const sections: string[] = [];
+  let current: string[] = [];
+  const flush = () => {
+    const section = current.join("\n").trim();
+    if (section) sections.push(section);
+    current = [];
+  };
+  for (const line of lines) {
+    if (HEADING_PATTERN.test(line)) flush();
+    current.push(line);
+  }
+  flush();
+  return sections.map((section) => ({ markdown: section, tokens: tokens(section) }));
+}
+
+/**
+ * Produces a deterministic, query-focused view of Markdown using a compact BM25-style
+ * section ranker. Matching sections retain source order so the result remains readable
+ * and attributable to the fetched page. The complete document is never mutated.
+ */
+export function focusMarkdown(markdown: string, query: string): FocusResult {
+  const normalizedQuery = query.trim();
+  const queryTerms = [...new Set(tokens(normalizedQuery))];
+  const sections = splitSections(markdown);
+  const originalCharacters = markdown.length;
+
+  if (queryTerms.length === 0 || sections.length === 0) {
+    return {
+      markdown: "",
+      details: {
+        query: normalizedQuery,
+        matchedSections: 0,
+        totalSections: sections.length,
+        omittedSections: sections.length,
+        originalCharacters,
+        focusedCharacters: 0,
+      },
+    };
+  }
+
+  const averageLength =
+    sections.reduce((sum, section) => sum + section.tokens.length, 0) / sections.length || 1;
+  const documentFrequency = new Map<string, number>();
+  for (const term of queryTerms) {
+    documentFrequency.set(
+      term,
+      sections.reduce((count, section) => count + (section.tokens.includes(term) ? 1 : 0), 0),
+    );
+  }
+
+  const selected = sections.filter((section) => {
+    const frequencies = new Map<string, number>();
+    for (const token of section.tokens) frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
+    const lengthNormalization = 1 - 0.75 + 0.75 * (section.tokens.length / averageLength);
+    let score = 0;
+    for (const term of queryTerms) {
+      const frequency = frequencies.get(term) ?? 0;
+      if (frequency === 0) continue;
+      const frequencyInDocuments = documentFrequency.get(term) ?? 0;
+      const inverseDocumentFrequency = Math.log(
+        1 + (sections.length - frequencyInDocuments + 0.5) / (frequencyInDocuments + 0.5),
+      );
+      score +=
+        inverseDocumentFrequency *
+        ((frequency * (1.2 + 1)) / (frequency + 1.2 * lengthNormalization));
+    }
+    return score > 0;
+  });
+
+  const focused = selected.map((section) => section.markdown).join("\n\n");
+  return {
+    markdown: focused,
+    details: {
+      query: normalizedQuery,
+      matchedSections: selected.length,
+      totalSections: sections.length,
+      omittedSections: sections.length - selected.length,
+      originalCharacters,
+      focusedCharacters: focused.length,
+    },
+  };
+}

package/src/index.ts

@@ -6,6 +6,7 @@ import {
   FETCH_DEFAULT_OFFSET,
   FETCH_MAX_CHARACTERS,
   FETCH_MAX_OFFSET_CHARACTERS,
+  FETCH_MAX_QUERY_CHARACTERS,
   FETCH_MAX_URL_CHARACTERS,
   FETCH_MIN_MAX_CHARACTERS,
 } from "./limits";
@@ -14,9 +15,19 @@ import { executeWebFetch } from "./service";
 
 export { ExpiringLruCache } from "./cache";
 export type { FetchResult } from "./content";
+export {
+  diagnoseExtraction,
+  extractDocumentLinks,
+  type ExtractedLink,
+  type ExtractedLinks,
+  type ExtractionDiagnostics,
+} from "./evidence";
 export { fetchRemoteContent, type FetchRemoteDependencies } from "./fetch";
+export { focusMarkdown, type FocusDetails, type FocusResult } from "./focus";
+export { createDocumentOutline, type DocumentOutline, type OutlineHeading } from "./outline";
 export {
   executeWebFetch,
+  type CacheStatus,
   type WebFetchDetails,
   type WebFetchParameters,
   type WebFetchTruncationDetails,
@@ -35,6 +46,14 @@ export const webFetchParameters = Type.Object({
     maxLength: FETCH_MAX_URL_CHARACTERS,
     description: "Public HTTP or HTTPS URL to fetch",
   }),
+  query: Type.Optional(
+    Type.String({
+      minLength: 1,
+      maxLength: FETCH_MAX_QUERY_CHARACTERS,
+      description:
+        "Optional focus query; returns matching source sections in document order (offsets apply to the focused view)",
+    }),
+  ),
   offset: Type.Optional(
     Type.Integer({
       minimum: 0,
@@ -51,6 +70,14 @@ export const webFetchParameters = Type.Object({
   ),
 });
 
+/**
+ * Pi web fetch extension that registers the web_fetch tool.
+ *
+ * Provides a tool for fetching public HTTP(S) pages and converting them to
+ * Markdown with support for pagination via offset/nextOffset parameters.
+ *
+ * @param pi - The extension API instance
+ */
 export default function (pi: ExtensionAPI) {
   pi.registerTool({
     name: "web_fetch",

package/src/inflight.ts

@@ -5,6 +5,14 @@ interface InflightEntry<V> {
   waiters: number;
 }
 
+/**
+ * Wraps a promise to respect a caller's abort signal without cancelling the underlying operation.
+ *
+ * @param operation - Promise to wait for
+ * @param signal - Optional abort signal for this caller
+ * @param cancelledMessage - Error message to use when signal is aborted
+ * @returns Promise that rejects if signal aborts, otherwise resolves with operation result
+ */
 function waitForCaller<T>(
   operation: Promise<T>,
   signal: AbortSignal | undefined,
@@ -28,10 +36,24 @@ export class InflightCoalescer<K, V> {
 
   constructor(readonly maxEntries: number) {}
 
+  /**
+   * Gets the number of currently in-flight operations.
+   *
+   * @returns Count of active operations
+   */
   get size(): number {
     return this.#entries.size;
   }
 
+  /**
+   * Runs an operation, coalescing with any identical in-flight operation.
+   *
+   * @param key - Operation key for coalescing identical operations
+   * @param operation - Function that performs the operation
+   * @param signal - Optional abort signal for this caller
+   * @param cancelledMessage - Error message when signal is aborted
+   * @returns Promise resolving to the operation result
+   */
   async run(
     key: K,
     operation: (signal: AbortSignal | undefined) => Promise<V>,

package/src/limits.ts

@@ -1,6 +1,20 @@
+/** Default starting offset for content extraction (characters). */
 export const FETCH_DEFAULT_OFFSET = 0;
+
+/** Minimum allowed value for maxCharacters parameter. */
 export const FETCH_MIN_MAX_CHARACTERS = 1_000;
+
+/** Default maximum characters to return per fetch. */
 export const FETCH_DEFAULT_MAX_CHARACTERS = 6_000;
+
+/** Maximum allowed characters to return per fetch. */
 export const FETCH_MAX_CHARACTERS = 30_000;
+
+/** Maximum allowed offset for content extraction (characters). */
 export const FETCH_MAX_OFFSET_CHARACTERS = 20_000_000;
+
+/** Maximum allowed length for an optional focus query (characters). */
+export const FETCH_MAX_QUERY_CHARACTERS = 400;
+
+/** Maximum allowed length for input URLs (characters). */
 export const FETCH_MAX_URL_CHARACTERS = 2_048;

package/src/network-policy.ts

@@ -1,11 +1,13 @@
 import { lookup as dnsLookup } from "node:dns/promises";
 import { BlockList, isIP } from "node:net";
 
-// Reviewed 2026-08-03 against the IANA IPv4 and IPv6 Special-Purpose Address Registries:
-// https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
-// https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
-// Keep non-global, documentation, benchmarking, multicast, and reserved space blocked. The explicit
-// endpoints make registry reviews and boundary tests auditable without changing the pinning flow.
+/**
+ * IPv4 address ranges that are blocked from network requests.
+ *
+ * Reviewed 2026-08-03 against the IANA IPv4 Special-Purpose Address Registry:
+ * https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+ * Includes non-global, documentation, benchmarking, multicast, and reserved ranges.
+ */
 export const BLOCKED_IPV4_RANGES = [
   ["0.0.0.0", 8, "0.0.0.0", "0.255.255.255", "current network"],
   ["10.0.0.0", 8, "10.0.0.0", "10.255.255.255", "private use"],
@@ -27,6 +29,13 @@ export const BLOCKED_IPV4_RANGES = [
   ["240.0.0.0", 4, "240.0.0.0", "255.255.255.255", "reserved"],
 ] as const;
 
+/**
+ * IPv6 address ranges that are blocked from network requests.
+ *
+ * Reviewed 2026-08-03 against the IANA IPv6 Special-Purpose Address Registry:
+ * https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
+ * Includes non-global, documentation, benchmarking, multicast, and reserved ranges.
+ */
 export const BLOCKED_IPV6_RANGES = [
   ["::", 128, "::", "::", "unspecified"],
   ["::1", 128, "::1", "::1", "loopback"],
@@ -68,6 +77,12 @@ for (const [network, prefix] of GLOBALLY_REACHABLE_IPV6_EXCEPTIONS) {
   allowedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
 
+/**
+ * A validated and resolved network target ready for connection.
+ *
+ * Contains the target URL, preferred address, and all resolved addresses
+ * for the hostname.
+ */
 export interface ValidatedTarget {
   url: URL;
   /** The preferred address used for the first connection attempt. */
@@ -80,6 +95,7 @@ export interface ValidatedTarget {
   addresses?: string[];
 }
 
+/** Function type for resolving a hostname to IP addresses. */
 export type ResolveAddresses = (hostname: string) => Promise<string[]>;
 
 /**

package/src/network-redirects.ts

@@ -3,33 +3,148 @@ import { awaitWithAbort } from "./abort";
 import { validateRemoteUrl, type ValidatedTarget } from "./network-policy";
 import { requestPinned, responseHeader } from "./network-transport";
 
+/** Maximum number of HTTP redirects to follow before aborting. */
 export const FETCH_MAX_REDIRECTS = 5;
+const MAX_ORIGIN_CONCURRENCY = 4;
+const MAX_RETRIES = 2;
+const MAX_RETRY_DELAY_MS = 10_000;
+const BASE_RETRY_DELAY_MS = 250;
 
+interface OriginState {
+  active: number;
+  waiting: Array<() => void>;
+}
+const origins = new Map<string, OriginState>();
+
+/** Optional dependencies for redirect handling (used for testing). */
 export interface RedirectDependencies {
   validateUrl?: (value: string | URL) => Promise<ValidatedTarget>;
-  request?: (target: ValidatedTarget, signal: AbortSignal) => Promise<IncomingMessage>;
+  request?: (
+    target: ValidatedTarget,
+    signal: AbortSignal,
+    headers?: Readonly<Record<string, string>>,
+  ) => Promise<IncomingMessage>;
+  random?: () => number;
+  sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
+}
+
+function abortError(): Error {
+  const error = new Error("Operation aborted.");
+  error.name = "AbortError";
+  return error;
+}
+
+async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() => void> {
+  const state = origins.get(origin) ?? { active: 0, waiting: [] };
+  origins.set(origin, state);
+  let reserved = false;
+  const release = () => {
+    state.active -= 1;
+    state.waiting.shift()?.();
+    if (state.active === 0 && state.waiting.length === 0) origins.delete(origin);
+  };
+  if (state.active >= MAX_ORIGIN_CONCURRENCY) {
+    await new Promise<void>((resolve, reject) => {
+      const enter = () => {
+        signal.removeEventListener("abort", cancel);
+        state.active += 1;
+        reserved = true;
+        resolve();
+      };
+      const cancel = () => {
+        const index = state.waiting.indexOf(enter);
+        if (index >= 0) state.waiting.splice(index, 1);
+        reject(abortError());
+      };
+      if (signal.aborted) cancel();
+      else {
+        state.waiting.push(enter);
+        signal.addEventListener("abort", cancel, { once: true });
+      }
+    });
+  }
+  if (signal.aborted) {
+    if (reserved) release();
+    throw abortError();
+  }
+  if (!reserved) state.active += 1;
+  return release;
+}
+
+async function defaultSleep(milliseconds: number, signal: AbortSignal): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(done, milliseconds);
+    function done(): void {
+      signal.removeEventListener("abort", cancel);
+      resolve();
+    }
+    function cancel(): void {
+      clearTimeout(timer);
+      reject(abortError());
+    }
+    if (signal.aborted) cancel();
+    else signal.addEventListener("abort", cancel, { once: true });
+  });
+}
+
+function retryAfterMilliseconds(value: string | undefined, now = Date.now()): number | undefined {
+  if (!value) return undefined;
+  const seconds = Number(value.trim());
+  if (Number.isFinite(seconds) && seconds >= 0)
+    return Math.min(seconds * 1_000, MAX_RETRY_DELAY_MS);
+  const date = Date.parse(value);
+  if (!Number.isFinite(date)) return undefined;
+  return Math.min(Math.max(0, date - now), MAX_RETRY_DELAY_MS);
+}
+
+async function coordinatedRequest(
+  target: ValidatedTarget,
+  signal: AbortSignal,
+  headers: Readonly<Record<string, string>>,
+  dependencies: RedirectDependencies,
+): Promise<IncomingMessage> {
+  const request =
+    dependencies.request ??
+    ((value, requestSignal, requestHeaders) =>
+      requestPinned(value, requestSignal, { headers: requestHeaders }));
+  const random = dependencies.random ?? Math.random;
+  const sleep = dependencies.sleep ?? defaultSleep;
+  for (let attempt = 0; ; attempt += 1) {
+    const release = await acquireOrigin(target.url.origin, signal);
+    let response: IncomingMessage;
+    try {
+      response = await request(target, signal, headers);
+    } finally {
+      release();
+    }
+    const status = response.statusCode ?? 0;
+    if (![429, 503].includes(status) || attempt >= MAX_RETRIES) return response;
+    // The retry decision uses headers only. Drop an untrusted error body so it cannot
+    // consume unbounded bandwidth while the caller is backing off.
+    response.destroy();
+    const advertised = retryAfterMilliseconds(responseHeader(response, "retry-after"));
+    const exponential = BASE_RETRY_DELAY_MS * 2 ** attempt;
+    const jittered = Math.round(exponential * (0.75 + Math.min(1, Math.max(0, random())) * 0.5));
+    await sleep(advertised ?? Math.min(jittered, MAX_RETRY_DELAY_MS), signal);
+  }
 }
 
 /**
- * Requests a URL and follows supported HTTP redirects.
- *
- * @param value - The initial URL to request
- * @param signal - Signal used to cancel validation and requests
- * @param dependencies - Optional URL-validation and request implementations
- * @returns The final validated target and its HTTP response
- * @throws If a redirect lacks a `Location` header or the redirect limit is exceeded
+ * Requests a URL and follows supported HTTP redirects. Every attempt, redirect, and retry
+ * remains inside URL validation, DNS pinning, origin coordination, and caller cancellation.
  */
 export async function requestFollowingRedirects(
   value: string | URL,
   signal: AbortSignal,
   dependencies: RedirectDependencies = {},
+  headers: Readonly<Record<string, string>> = {},
 ): Promise<{ target: ValidatedTarget; response: IncomingMessage }> {
   const validateUrl = dependencies.validateUrl ?? validateRemoteUrl;
-  const request = dependencies.request ?? requestPinned;
   let target = await awaitWithAbort(validateUrl(value), signal);
+  let requestHeaders = headers;
 
   for (let redirects = 0; redirects <= FETCH_MAX_REDIRECTS; redirects += 1) {
-    const response = await request(target, signal);
+    const response = await coordinatedRequest(target, signal, requestHeaders, dependencies);
     const status = response.statusCode ?? 0;
     if (![301, 302, 303, 307, 308].includes(status)) return { target, response };
 
@@ -39,6 +154,8 @@ export async function requestFollowingRedirects(
       throw new Error("web_fetch followed too many redirects.");
     response.resume();
     target = await awaitWithAbort(validateUrl(new URL(location, target.url)), signal);
+    // Validators describe the originally cached resource and must not leak to a redirect target.
+    requestHeaders = {};
   }
   throw new Error("web_fetch followed too many redirects.");
 }