crawlforge-mcp-server 6.14.0 → 6.16.0

package/src/tools/extract/processDocument.js

@@ -283,8 +283,9 @@ export class ProcessDocumentTool {
    */
   async processPDFDocument(result, source, sourceType, options, identity = {}) {
     // A remote PDF is a fetch of the target like any other; a local file is not.
+    let gate;
     if (sourceType === 'pdf_url') {
-      const gate = await preflightFetch(source, {
+      gate = await preflightFetch(source, {
         respectRobots: identity.respect_robots,
         userAgent: identity.user_agent,
         tool: 'process_document'
@@ -302,7 +303,7 @@ export class ProcessDocumentTool {
         maxPages: options.maxPages,
         ...(options.pageRange ? { pageRange: options.pageRange } : {})
       }
-    });
+    }, gate);
 
     if (!pdfResult.success) {
       throw new Error(pdfResult.error || 'PDF processing failed');
@@ -369,7 +370,7 @@ export class ProcessDocumentTool {
           enableImages: false,
           captureScreenshot: false
         }
-      });
+      }, gate);
 
       if (!browserResult.success) {
         throw new Error(`Browser processing failed: ${browserResult.error}`);
@@ -381,6 +382,7 @@ export class ProcessDocumentTool {
       // Simple HTTP fetch
       const response = await safeFetch(source, {
         headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
         signal: AbortSignal.timeout(15000)
       });
 

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -380,7 +380,8 @@ export class GenerateLLMsTxtTool {
       });
       const response = await safeFetch(baseUrl, {
         signal: controller.signal,
-        headers: { ...gate.headers }
+        headers: { ...gate.headers },
+        onRedirect: gate.onRedirect
       });
       if (!response.ok) return null;
       // Charset-aware read; .text() decodes UTF-8 only.

package/src/tools/templates/ScrapeTemplateTool.js

@@ -123,10 +123,13 @@ export class ScrapeTemplateTool {
     }
 
     // Robots gate + per-host politeness before any request to the target.
+    // A keyed connector's URL carries the key, and an override writes the URL
+    // to the audit log — so the gate is told what to strip from that row.
     const gate = await preflightFetch(fetchUrl, {
       respectRobots: respect_robots,
       userAgent: user_agent,
-      tool: 'scrape_template'
+      tool: 'scrape_template',
+      redactCredential: apiKey
     });
 
     // Fetch the page
@@ -136,7 +139,8 @@ export class ScrapeTemplateTool {
     try {
       const response = await safeFetch(fetchUrl, {
         signal: controller.signal,
-        headers: { ...gate.headers }
+        headers: { ...gate.headers },
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
 
@@ -200,7 +204,11 @@ export class ScrapeTemplateTool {
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     let response;
     try {
-      response = await safeFetch(url, { signal: controller.signal, headers: { ...pageGate.headers } });
+      response = await safeFetch(url, {
+        signal: controller.signal,
+        headers: { ...pageGate.headers },
+        onRedirect: pageGate.onRedirect
+      });
     } finally {
       clearTimeout(timeoutId);
     }

package/src/tools/tracking/trackChanges/differ.js

@@ -72,6 +72,7 @@ export async function fetchContent(url, options = {}) {
         'Accept-Encoding': 'gzip, deflate',
         'Cache-Control': 'no-cache'
       },
+      onRedirect: gate.onRedirect,
       signal: AbortSignal.timeout(30000)
     });
 

package/src/utils/complianceAudit.js

@@ -13,7 +13,9 @@
  * best-effort: an audit sink that throws must never fail a customer's fetch.
  *
  * The API key is never stored. `apiKeyId` is a truncated SHA-256 of it — stable
- * enough to group a key's overrides, useless as a credential.
+ * enough to group a key's overrides, useless as a credential. A row's `url` is
+ * kept, because the row has to say what was fetched, but not a credential that
+ * rode along in it (see `auditUrl`).
  */
 
 import { createHash } from 'crypto';
@@ -35,6 +37,30 @@ export function apiKeyId(apiKey) {
   return createHash('sha256').update(apiKey).digest('hex').slice(0, 16);
 }
 
+/**
+ * Query parameters whose value is a credential, by name: `token`,
+ * `access_token`, `api_key`, `key`, `sig`, `X-Amz-Signature`, `password`,
+ * `PHPSESSID`, `authorization`. `author` and `keyword` are not.
+ */
+const CREDENTIAL_PARAM_RE = /token|secret|passw|signature|credential|sess(?:ion|id)|api[_-]?key|access[_-]?key|private[_-]?key|jwt|auth(?!or(?!i[sz]))|^(?:key|sig|sid|pwd|pass)$/i;
+
+/**
+ * A URL as a row may keep it: `user:password@` removed, and the value of every
+ * query parameter named like a credential replaced. The path and every other
+ * parameter stay as they were sent. Matching is by name, so a credential under
+ * a name that says nothing (`?k=`, or one carried in the path) is not caught.
+ * @param {string} url
+ * @returns {string}
+ */
+export function auditUrl(url) {
+  if (typeof url !== 'string') return url;
+  return url
+    .replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#@]*@/i, '$1')
+    .replace(/([?&])([^=&#]+)=[^&#]*/g, (pair, separator, name) => (
+      CREDENTIAL_PARAM_RE.test(name) ? `${separator}${name}=[redacted]` : pair
+    ));
+}
+
 async function defaultSink(row) {
   await mkdir(dirname(DEFAULT_LOG_PATH), { recursive: true });
   await appendFile(DEFAULT_LOG_PATH, `${JSON.stringify(row)}\n`, 'utf8');
@@ -46,7 +72,11 @@ async function defaultSink(row) {
  * @returns {{ event: string, timestamp: string }} the row as written
  */
 export function recordComplianceEvent(event = {}) {
-  const row = { timestamp: new Date().toISOString(), ...event };
+  const row = {
+    timestamp: new Date().toISOString(),
+    ...event,
+    ...(event.url === undefined ? {} : { url: auditUrl(event.url) })
+  };
   ring.push(row);
   if (ring.length > RING_SIZE) ring.shift();
 
@@ -60,9 +90,9 @@ export function recordComplianceEvent(event = {}) {
 /**
  * Record that a request is about to go out under a browser identity (a stealth
  * render). Neither the raw key nor the raw owner token is stored: both are
- * reduced to apiKeyId digests. `ownerId` names the website customer behind an
- * internal-proxy request, where `apiKeyId` is always the service's own key.
- * Never throws.
+ * reduced to apiKeyId digests, and the URL is kept as `auditUrl` leaves it.
+ * `ownerId` names the website customer behind an internal-proxy request, where
+ * `apiKeyId` is always the service's own key. Never throws.
  */
 export function recordStealthEscalation({ url, tool, engine, apiKey, ownerToken } = {}) {
   return recordComplianceEvent({

package/src/utils/impitRung.js

@@ -22,7 +22,8 @@
  *
  * impit resolves DNS itself, outside the undici dispatcher ssrfGuard installs,
  * so redirects are followed here, one hop at a time, and every hop is checked
- * with DNS resolution — the same check safeGoto applies to a navigation.
+ * with DNS resolution — the same check safeGoto applies to a navigation — and
+ * put to the caller's `onRedirect` gate before it is requested.
  */
 
 import { load } from 'cheerio';
@@ -84,16 +85,19 @@ export async function loadImpit() {
  * not (a wall, an error page, an empty shell) or the request failed. A null
  * means "run the browser"; this function never throws for a failed fetch.
  *
- * An SSRF refusal on any hop DOES throw: the browser would be refused the same
- * URL, so falling through to it would only fail slower.
+ * An SSRF refusal on any hop DOES throw, and so does a hop the caller's gate
+ * refuses: the browser would be refused the same URL, so falling through to it
+ * would only fail slower.
  *
  * @param {string} url
  * @param {object} [options]
  * @param {Function} [options.Impit] the client class (tests inject a fake)
  * @param {number} [options.timeoutMs]
+ * @param {(to: string) => Promise<void>} [options.onRedirect] awaited with each
+ *   redirect target before it is requested (robotsGate.js `redirectGate`)
  * @returns {Promise<{ html: string, text: string, title: string, url: string, status: number, engine: string } | null>}
  */
-export async function impitFetchPage(url, { Impit, timeoutMs = IMPIT_TIMEOUT_MS } = {}) {
+export async function impitFetchPage(url, { Impit, timeoutMs = IMPIT_TIMEOUT_MS, onRedirect } = {}) {
   const ImpitClass = Impit ?? await loadImpit();
   if (!ImpitClass) return null;
 
@@ -108,7 +112,7 @@ export async function impitFetchPage(url, { Impit, timeoutMs = IMPIT_TIMEOUT_MS
     ...(proxyUrl ? { proxyUrl } : {})
   });
 
-  const deadline = Date.now() + timeoutMs;
+  let deadline = Date.now() + timeoutMs;
   let current = url;
   let response;
   for (let hop = 0; ; hop++) {
@@ -129,6 +133,13 @@ export async function impitFetchPage(url, { Impit, timeoutMs = IMPIT_TIMEOUT_MS
       return null;
     }
     if (!/^https?:\/\//i.test(current)) return null;
+    if (onRedirect) {
+      // The gate may wait out the next host's Crawl-delay. That is our
+      // waiting, not the site being slow, so the deadline moves with it.
+      const gateStartedAt = Date.now();
+      await onRedirect(current);
+      deadline += Date.now() - gateStartedAt;
+    }
   }
 
   let html;

package/src/utils/resignedFetch.js

@@ -0,0 +1,165 @@
+/**
+ * fetch for a crawl request, following redirects one hop at a time so that
+ * every hop is gated and signed for itself.
+ *
+ * fetch follows a redirect on its own, forwarding the request headers
+ * unchanged and asking nobody. That went wrong twice:
+ *
+ *  - The gate ran once, for the URL the caller asked for. A redirect could
+ *    then land on a path robots.txt disallows, or on a host the platform
+ *    blocklist refuses, and be fetched anyway.
+ *  - `preflightFetch` signs once. A same-host redirect arrived with a nonce
+ *    the site had already seen, which a verifier that tracks nonces rejects as
+ *    a replay, and a cross-host redirect arrived with a signature over the
+ *    previous host's `@authority`, which no verifier accepts. Both made a
+ *    genuine CrawlForge request look forged.
+ *
+ * A request takes the manual path when it carries a signature or an
+ * `onRedirect` gate, and follows redirects. Everything else (webhooks, an
+ * explicit `redirect: 'manual'`) is handed to fetch exactly as before.
+ */
+
+import { signRequestHeaders } from './webBotAuth.js';
+
+/** fetch's own limit. */
+const MAX_REDIRECTS = 20;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SIGNATURE_HEADERS = ['signature', 'signature-input', 'signature-agent'];
+/** What fetch drops when a redirect leaves the origin. */
+const CROSS_ORIGIN_HEADERS = ['authorization', 'cookie', 'host', 'proxy-authorization'];
+/** What fetch drops when a redirect turns the request into a GET. */
+const BODY_HEADERS = ['content-encoding', 'content-language', 'content-location', 'content-type', 'content-length'];
+
+/**
+ * Resolve a `Location` header the way fetch does. Some sites send it as raw
+ * UTF-8 rather than percent-encoded; the header arrives as a binary string, so
+ * those bytes are read back as UTF-8 before the URL is parsed.
+ */
+function locationUrl(location, base) {
+  const raw = /[^\x20-\x7e]/.test(location) ? Buffer.from(location, 'latin1').toString('utf8') : location;
+  return new URL(raw, base);
+}
+
+/**
+ * The caller's abort signal with the time a hop's gate takes given back.
+ *
+ * The gate for the URL a caller asked for runs before the caller starts its
+ * timeout. The gate for a redirect runs inside it, and may wait out the next
+ * host's Crawl-delay: 30 s on a host that asks for it, against a fetch timeout
+ * of 15. Charged to the timeout, that wait would fail the fetch as if the site
+ * had not answered. So the caller's abort reaches the fetch late by however
+ * long the gates have held it, and not while one is still holding it.
+ *
+ * @param {AbortSignal} [callerSignal]
+ * @returns {{ signal?: AbortSignal, gated: <T>(gate: () => Promise<T>) => Promise<T> }}
+ *   `signal` replaces the caller's on every later hop; `gated` runs a gate
+ *   with its time counted
+ */
+function gateTimeRefund(callerSignal) {
+  if (!callerSignal) return { gated: (gate) => gate() };
+
+  const controller = new AbortController();
+  let gateMs = 0;
+  let inGate = false;
+  let abortedAt = null;
+  let timer;
+
+  const deliver = () => {
+    if (abortedAt === null || inGate) return;
+    clearTimeout(timer);
+    timer = setTimeout(
+      () => controller.abort(callerSignal.reason),
+      Math.max(0, abortedAt + gateMs - Date.now())
+    );
+    // A fetch that finished long ago must not keep the process up for this.
+    timer.unref?.();
+  };
+  const onAbort = () => {
+    abortedAt = Date.now();
+    deliver();
+  };
+  if (callerSignal.aborted) onAbort();
+  else callerSignal.addEventListener('abort', onAbort, { once: true });
+
+  return {
+    signal: controller.signal,
+    async gated(gate) {
+      const startedAt = Date.now();
+      inGate = true;
+      clearTimeout(timer);
+      try {
+        return await gate();
+      } finally {
+        inGate = false;
+        gateMs += Date.now() - startedAt;
+        deliver();
+      }
+    }
+  };
+}
+
+/**
+ * @param {string} url
+ * @param {RequestInit & { onRedirect?: (to: string) => Promise<void> }} [options]
+ *   `onRedirect` is awaited with each redirect target before it is requested;
+ *   it refuses the hop by throwing. The time it takes is not charged to
+ *   `signal` (see {@link gateTimeRefund}).
+ * @returns {Promise<Response>}
+ */
+export async function fetchResigned(url, options = {}) {
+  const { onRedirect, ...fetchOptions } = options;
+  const signed = new Headers(fetchOptions.headers).has('signature-input');
+  if ((!signed && !onRedirect) || (fetchOptions.redirect ?? 'follow') !== 'follow') {
+    return fetch(url, fetchOptions);
+  }
+
+  let current = String(url);
+  // The first hop goes out with the caller's headers and signal untouched.
+  let init = { ...fetchOptions, redirect: 'manual' };
+  let refund = null;
+
+  for (let hop = 0; ; hop++) {
+    const response = await fetch(current, init);
+    const location = REDIRECT_STATUSES.has(response.status) ? response.headers.get('location') : null;
+    if (!location) {
+      if (hop > 0) Object.defineProperty(response, 'redirected', { value: true });
+      return response;
+    }
+    // Released, not awaited: the next hop must not wait on a body nobody reads.
+    response.body?.cancel().catch(() => {});
+    if (hop >= MAX_REDIRECTS) throw new TypeError('fetch failed: redirect count exceeded');
+
+    const next = locationUrl(location, current);
+    if (next.protocol !== 'http:' && next.protocol !== 'https:') {
+      throw new TypeError('fetch failed: redirect to a URL that is not http(s)');
+    }
+    if (onRedirect) {
+      refund ??= gateTimeRefund(fetchOptions.signal);
+      await refund.gated(() => onRedirect(next.href));
+    }
+
+    const headers = new Headers(init.headers);
+    if (next.origin !== new URL(current).origin) {
+      for (const name of CROSS_ORIGIN_HEADERS) headers.delete(name);
+    }
+
+    const method = (init.method ?? 'GET').toUpperCase();
+    const becomesGet = response.status === 303
+      ? method !== 'GET' && method !== 'HEAD'
+      : method === 'POST' && (response.status === 301 || response.status === 302);
+    if (becomesGet) {
+      for (const name of BODY_HEADERS) headers.delete(name);
+      init = { ...init, method: 'GET', body: undefined };
+    }
+
+    if (signed) {
+      for (const name of SIGNATURE_HEADERS) headers.delete(name);
+      for (const [name, value] of Object.entries(signRequestHeaders(next.href) ?? {})) {
+        headers.set(name, value);
+      }
+    }
+
+    init = { ...init, headers, ...(refund?.signal ? { signal: refund.signal } : {}) };
+    current = next.href;
+  }
+}

package/src/utils/robotsChecker.js

@@ -1,6 +1,8 @@
 import robotsParser from 'robots-parser';
 import { safeFetch } from './ssrfGuard.js';
 import { identityHeaders, CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+import { signRequestHeaders } from './webBotAuth.js';
+import { assertHostAllowed } from './hostBlocklist.js';
 
 /**
  * The token this crawler used to identify as, honoured as a source of disallow
@@ -88,7 +90,10 @@ export class RobotsChecker {
     try {
       const response = await safeFetch(robotsUrl, {
         signal: controller.signal,
-        headers: identityHeaders({ userAgent: this.userAgent })
+        headers: { ...identityHeaders({ userAgent: this.userAgent }), ...signRequestHeaders(robotsUrl) },
+        // robots.txt is not itself subject to robots.txt, but a host on the
+        // platform blocklist is never requested, by redirect or otherwise.
+        onRedirect: async (to) => assertHostAllowed(to)
       });
 
       if (!response.ok) {

package/src/utils/robotsGate.js

@@ -28,11 +28,31 @@ import { markPreflightRefusal, internalOwnerToken } from '../server/requestConte
 import AuthManager from '../core/AuthManager.js';
 import { config } from '../constants/config.js';
 
+/** A URL as it may appear in an error: no query string, which can carry a key. */
+function withoutQuery(url) {
+  const parsed = new URL(url);
+  return `${parsed.origin}${parsed.pathname}`;
+}
+
 export class RobotsDisallowedError extends Error {
-  constructor(url) {
+  /**
+   * @param {string} url the URL robots.txt disallows
+   * @param {{ redirectedFrom?: string, movedFrom?: string }} [options] set when
+   *   `url` is not what the caller asked for, so the message says how it was
+   *   reached: `redirectedFrom` when a redirect led there, `movedFrom` when a
+   *   browser page went there by itself (a click, a form, a script)
+   */
+  constructor(url, { redirectedFrom, movedFrom } = {}) {
+    const host = new URL(url).host;
+    let reached = `robots.txt on ${host} disallows this path for CrawlForge. `;
+    if (redirectedFrom) {
+      reached = `${withoutQuery(redirectedFrom)} redirects to ${withoutQuery(url)}, and robots.txt on ${host} disallows that path for CrawlForge. `;
+    } else if (movedFrom) {
+      reached = `The page moved from ${withoutQuery(movedFrom)} to ${withoutQuery(url)}, and robots.txt on ${host} disallows that path for CrawlForge. `;
+    }
     super(
-      `robots.txt on ${new URL(url).host} disallows this path for CrawlForge. ` +
-      `Pass respect_robots: false to fetch it anyway — that override is recorded ` +
+      reached +
+      `Pass respect_robots: false to ${movedFrom ? 'use' : 'fetch'} it anyway — that override is recorded ` +
       `against your API key and is your decision to make.`
     );
     this.name = 'RobotsDisallowedError';
@@ -41,6 +61,40 @@ export class RobotsDisallowedError extends Error {
   }
 }
 
+/** What a refusal carries as `code`: this gate's three, and the SSRF guard's. */
+const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
+
+/**
+ * The refusal code an error carries, or null when it is a failure rather than
+ * a refusal. A refusal is this gate or the SSRF guard saying no: asking again
+ * gets the same answer, so nothing should retry it.
+ * @param {unknown} error
+ * @returns {string|null}
+ */
+export function gateRefusalCode(error) {
+  for (const code of [error?.code, error?.cause?.code]) {
+    if (REFUSAL_CODES.has(code)) return code;
+  }
+  return null;
+}
+
+/**
+ * Remove a known deployment credential from a URL bound for the audit log: the
+ * raw value and both encodings of it. `URLSearchParams` escapes `~ ! ' ( )` and
+ * writes a space as `+`; `encodeURIComponent` does neither.
+ */
+function redactSecret(value, secret) {
+  if (!secret) return value;
+  const forms = new Set([
+    secret,
+    encodeURIComponent(secret),
+    new URLSearchParams({ s: secret }).toString().slice(2)
+  ]);
+  let redacted = value;
+  for (const form of forms) redacted = redacted.replaceAll(form, '[redacted]');
+  return redacted;
+}
+
 /**
  * One checker per identity, so the robots cache is process-wide rather than
  * per-tool — otherwise every tool would re-fetch the same robots.txt.
@@ -68,6 +122,10 @@ function checkerFor(userAgent) {
  * @param {string}  [options.userAgent] per-request identity override
  * @param {string}  [options.tool] tool name, for the audit row
  * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @param {string}  [options.redactCredential] a deployment credential that
+ *   appears in `url` (a keyed connector puts its key in the query string), to
+ *   remove from the audit row. The row keeps the URL, because it has to say
+ *   what was fetched, and loses the secret.
  * @returns {Promise<{ allowed: boolean, userAgent: string, crawlDelayMs: number,
  *   warnings: string[], overridden: boolean }>}
  * @throws {BlockedHostError} for a permanently blocked host
@@ -119,7 +177,7 @@ export async function robotsPreflight(url, options = {}) {
     const ownerToken = internalOwnerToken();
     recordComplianceEvent({
       event: 'robots_override',
-      url,
+      url: redactSecret(url, options.redactCredential),
       tool: options.tool || null,
       apiKeyId: apiKeyId(options.apiKey ?? AuthManager.getConfig()?.apiKey),
       ...(ownerToken ? { ownerId: apiKeyId(ownerToken) } : {}),
@@ -138,15 +196,57 @@ export async function robotsPreflight(url, options = {}) {
 }
 
 /**
- * The call-site helper: run the gate, honour Crawl-delay and any recorded
- * `Retry-After`, and hand back the identity headers to send.
+ * The gate for a redirect hop, to pass to the fetch as `onRedirect`.
  *
- * @param {string} url
+ * The gate above decides about the URL the caller asked for; a redirect can
+ * lead anywhere. Without this a 301 into a path robots.txt disallows, or onto
+ * a host on the platform blocklist, was fetched because nobody asked again.
+ * Each hop gets the decision a first request would: blocklist, then robots,
+ * with the caller's own `respect_robots` and an audit row when it overrides.
+ *
+ * It then waits its turn as a first request would: the hop is one more request
+ * to its host, so it is spaced by that host's Crawl-delay, by a Retry-After it
+ * has sent and by our own per-host limit. A fetch does not charge that wait to
+ * its timeout (resignedFetch.js). A browser has made the request before anyone
+ * could be asked, so its caller passes `alreadyRequested` and nothing is
+ * slept: there is no request left to space.
+ *
+ * @param {string} from the URL the fetch started at, for the refusal message
  * @param {object} [options] see {@link robotsPreflight}
- * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
- *   warnings: string[], overridden: boolean }>}
- * @throws {BlockedHostError|RobotsDisallowedError}
+ * @returns {(to: string, hop?: { alreadyRequested?: boolean }) => Promise<void>}
+ *   throws BlockedHostError or RobotsDisallowedError to refuse the hop
  */
+export function redirectGate(from, options = {}) {
+  return hopGate(options, { redirectedFrom: from });
+}
+
+/**
+ * The gate for a URL a browser page reached without being sent there: a click
+ * followed a link, a form posted, a script redirected, a client-side router
+ * rewrote the address. The decision is {@link redirectGate}'s; only the
+ * refusal reads differently, because nothing redirected.
+ *
+ * @param {string} [from] the last URL the page was checked at
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {(to: string, hop?: { alreadyRequested?: boolean }) => Promise<void>}
+ *   throws BlockedHostError or RobotsDisallowedError to refuse where the page
+ *   now stands
+ */
+export function pageMoveGate(from, options = {}) {
+  return hopGate(options, { movedFrom: from });
+}
+
+function hopGate(options, reached) {
+  return async (to, { alreadyRequested = false } = {}) => {
+    const decision = await robotsPreflight(to, options);
+    if (!decision.allowed) {
+      markPreflightRefusal('ROBOTS_DISALLOWED');
+      throw new RobotsDisallowedError(to, reached);
+    }
+    if (!alreadyRequested) await throttleHost(to, { crawlDelayMs: decision.crawlDelayMs });
+  };
+}
+
 /**
  * The headers an HTTP page fetch carries: identity, the Web Bot Auth signature
  * when one is configured, and an EMPTY Accept-Language.
@@ -170,6 +270,18 @@ export function outboundHeaders(userAgent, signature = {}) {
   };
 }
 
+/**
+ * The call-site helper: run the gate, honour Crawl-delay and any recorded
+ * `Retry-After`, and hand back the identity headers to send and the gate for
+ * any redirect the fetch follows. Pass both to the fetch.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
+ *   warnings: string[], overridden: boolean,
+ *   onRedirect: (to: string) => Promise<void> }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
 export async function preflightFetch(url, options = {}) {
   const decision = await robotsPreflight(url, options);
   if (!decision.allowed) {
@@ -191,7 +303,8 @@ export async function preflightFetch(url, options = {}) {
     headers: outboundHeaders(decision.userAgent, signature),
     userAgent: decision.userAgent,
     warnings: [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)],
-    overridden: decision.overridden
+    overridden: decision.overridden,
+    onRedirect: redirectGate(url, options)
   };
 }
 

package/src/utils/sitemapParser.js

@@ -679,7 +679,8 @@ export class SitemapParser {
           ...gate.headers,
           'Accept': 'application/xml,text/xml,text/plain,*/*',
           'Accept-Encoding': 'gzip, deflate'
-        }
+        },
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
       if (response.status === 429 || response.status === 503) {