crawlforge-mcp-server 6.14.0 → 6.17.1

package/src/utils/resignedFetch.js

@@ -0,0 +1,165 @@
+/**
+ * fetch for a crawl request, following redirects one hop at a time so that
+ * every hop is gated and signed for itself.
+ *
+ * fetch follows a redirect on its own, forwarding the request headers
+ * unchanged and asking nobody. That went wrong twice:
+ *
+ *  - The gate ran once, for the URL the caller asked for. A redirect could
+ *    then land on a path robots.txt disallows, or on a host the platform
+ *    blocklist refuses, and be fetched anyway.
+ *  - `preflightFetch` signs once. A same-host redirect arrived with a nonce
+ *    the site had already seen, which a verifier that tracks nonces rejects as
+ *    a replay, and a cross-host redirect arrived with a signature over the
+ *    previous host's `@authority`, which no verifier accepts. Both made a
+ *    genuine CrawlForge request look forged.
+ *
+ * A request takes the manual path when it carries a signature or an
+ * `onRedirect` gate, and follows redirects. Everything else (webhooks, an
+ * explicit `redirect: 'manual'`) is handed to fetch exactly as before.
+ */
+
+import { signRequestHeaders } from './webBotAuth.js';
+
+/** fetch's own limit. */
+const MAX_REDIRECTS = 20;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SIGNATURE_HEADERS = ['signature', 'signature-input', 'signature-agent'];
+/** What fetch drops when a redirect leaves the origin. */
+const CROSS_ORIGIN_HEADERS = ['authorization', 'cookie', 'host', 'proxy-authorization'];
+/** What fetch drops when a redirect turns the request into a GET. */
+const BODY_HEADERS = ['content-encoding', 'content-language', 'content-location', 'content-type', 'content-length'];
+
+/**
+ * Resolve a `Location` header the way fetch does. Some sites send it as raw
+ * UTF-8 rather than percent-encoded; the header arrives as a binary string, so
+ * those bytes are read back as UTF-8 before the URL is parsed.
+ */
+function locationUrl(location, base) {
+  const raw = /[^\x20-\x7e]/.test(location) ? Buffer.from(location, 'latin1').toString('utf8') : location;
+  return new URL(raw, base);
+}
+
+/**
+ * The caller's abort signal with the time a hop's gate takes given back.
+ *
+ * The gate for the URL a caller asked for runs before the caller starts its
+ * timeout. The gate for a redirect runs inside it, and may wait out the next
+ * host's Crawl-delay: 30 s on a host that asks for it, against a fetch timeout
+ * of 15. Charged to the timeout, that wait would fail the fetch as if the site
+ * had not answered. So the caller's abort reaches the fetch late by however
+ * long the gates have held it, and not while one is still holding it.
+ *
+ * @param {AbortSignal} [callerSignal]
+ * @returns {{ signal?: AbortSignal, gated: <T>(gate: () => Promise<T>) => Promise<T> }}
+ *   `signal` replaces the caller's on every later hop; `gated` runs a gate
+ *   with its time counted
+ */
+function gateTimeRefund(callerSignal) {
+  if (!callerSignal) return { gated: (gate) => gate() };
+
+  const controller = new AbortController();
+  let gateMs = 0;
+  let inGate = false;
+  let abortedAt = null;
+  let timer;
+
+  const deliver = () => {
+    if (abortedAt === null || inGate) return;
+    clearTimeout(timer);
+    timer = setTimeout(
+      () => controller.abort(callerSignal.reason),
+      Math.max(0, abortedAt + gateMs - Date.now())
+    );
+    // A fetch that finished long ago must not keep the process up for this.
+    timer.unref?.();
+  };
+  const onAbort = () => {
+    abortedAt = Date.now();
+    deliver();
+  };
+  if (callerSignal.aborted) onAbort();
+  else callerSignal.addEventListener('abort', onAbort, { once: true });
+
+  return {
+    signal: controller.signal,
+    async gated(gate) {
+      const startedAt = Date.now();
+      inGate = true;
+      clearTimeout(timer);
+      try {
+        return await gate();
+      } finally {
+        inGate = false;
+        gateMs += Date.now() - startedAt;
+        deliver();
+      }
+    }
+  };
+}
+
+/**
+ * @param {string} url
+ * @param {RequestInit & { onRedirect?: (to: string) => Promise<void> }} [options]
+ *   `onRedirect` is awaited with each redirect target before it is requested;
+ *   it refuses the hop by throwing. The time it takes is not charged to
+ *   `signal` (see {@link gateTimeRefund}).
+ * @returns {Promise<Response>}
+ */
+export async function fetchResigned(url, options = {}) {
+  const { onRedirect, ...fetchOptions } = options;
+  const signed = new Headers(fetchOptions.headers).has('signature-input');
+  if ((!signed && !onRedirect) || (fetchOptions.redirect ?? 'follow') !== 'follow') {
+    return fetch(url, fetchOptions);
+  }
+
+  let current = String(url);
+  // The first hop goes out with the caller's headers and signal untouched.
+  let init = { ...fetchOptions, redirect: 'manual' };
+  let refund = null;
+
+  for (let hop = 0; ; hop++) {
+    const response = await fetch(current, init);
+    const location = REDIRECT_STATUSES.has(response.status) ? response.headers.get('location') : null;
+    if (!location) {
+      if (hop > 0) Object.defineProperty(response, 'redirected', { value: true });
+      return response;
+    }
+    // Released, not awaited: the next hop must not wait on a body nobody reads.
+    response.body?.cancel().catch(() => {});
+    if (hop >= MAX_REDIRECTS) throw new TypeError('fetch failed: redirect count exceeded');
+
+    const next = locationUrl(location, current);
+    if (next.protocol !== 'http:' && next.protocol !== 'https:') {
+      throw new TypeError('fetch failed: redirect to a URL that is not http(s)');
+    }
+    if (onRedirect) {
+      refund ??= gateTimeRefund(fetchOptions.signal);
+      await refund.gated(() => onRedirect(next.href));
+    }
+
+    const headers = new Headers(init.headers);
+    if (next.origin !== new URL(current).origin) {
+      for (const name of CROSS_ORIGIN_HEADERS) headers.delete(name);
+    }
+
+    const method = (init.method ?? 'GET').toUpperCase();
+    const becomesGet = response.status === 303
+      ? method !== 'GET' && method !== 'HEAD'
+      : method === 'POST' && (response.status === 301 || response.status === 302);
+    if (becomesGet) {
+      for (const name of BODY_HEADERS) headers.delete(name);
+      init = { ...init, method: 'GET', body: undefined };
+    }
+
+    if (signed) {
+      for (const name of SIGNATURE_HEADERS) headers.delete(name);
+      for (const [name, value] of Object.entries(signRequestHeaders(next.href) ?? {})) {
+        headers.set(name, value);
+      }
+    }
+
+    init = { ...init, headers, ...(refund?.signal ? { signal: refund.signal } : {}) };
+    current = next.href;
+  }
+}

package/src/utils/robotsChecker.js

@@ -1,6 +1,8 @@
 import robotsParser from 'robots-parser';
 import { safeFetch } from './ssrfGuard.js';
 import { identityHeaders, CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+import { signRequestHeaders } from './webBotAuth.js';
+import { assertHostAllowed } from './hostBlocklist.js';
 
 /**
  * The token this crawler used to identify as, honoured as a source of disallow
@@ -88,7 +90,10 @@ export class RobotsChecker {
     try {
       const response = await safeFetch(robotsUrl, {
         signal: controller.signal,
-        headers: identityHeaders({ userAgent: this.userAgent })
+        headers: { ...identityHeaders({ userAgent: this.userAgent }), ...signRequestHeaders(robotsUrl) },
+        // robots.txt is not itself subject to robots.txt, but a host on the
+        // platform blocklist is never requested, by redirect or otherwise.
+        onRedirect: async (to) => assertHostAllowed(to)
       });
 
       if (!response.ok) {

package/src/utils/robotsGate.js

@@ -28,11 +28,31 @@ import { markPreflightRefusal, internalOwnerToken } from '../server/requestConte
 import AuthManager from '../core/AuthManager.js';
 import { config } from '../constants/config.js';
 
+/** A URL as it may appear in an error: no query string, which can carry a key. */
+function withoutQuery(url) {
+  const parsed = new URL(url);
+  return `${parsed.origin}${parsed.pathname}`;
+}
+
 export class RobotsDisallowedError extends Error {
-  constructor(url) {
+  /**
+   * @param {string} url the URL robots.txt disallows
+   * @param {{ redirectedFrom?: string, movedFrom?: string }} [options] set when
+   *   `url` is not what the caller asked for, so the message says how it was
+   *   reached: `redirectedFrom` when a redirect led there, `movedFrom` when a
+   *   browser page went there by itself (a click, a form, a script)
+   */
+  constructor(url, { redirectedFrom, movedFrom } = {}) {
+    const host = new URL(url).host;
+    let reached = `robots.txt on ${host} disallows this path for CrawlForge. `;
+    if (redirectedFrom) {
+      reached = `${withoutQuery(redirectedFrom)} redirects to ${withoutQuery(url)}, and robots.txt on ${host} disallows that path for CrawlForge. `;
+    } else if (movedFrom) {
+      reached = `The page moved from ${withoutQuery(movedFrom)} to ${withoutQuery(url)}, and robots.txt on ${host} disallows that path for CrawlForge. `;
+    }
     super(
-      `robots.txt on ${new URL(url).host} disallows this path for CrawlForge. ` +
-      `Pass respect_robots: false to fetch it anyway — that override is recorded ` +
+      reached +
+      `Pass respect_robots: false to ${movedFrom ? 'use' : 'fetch'} it anyway — that override is recorded ` +
       `against your API key and is your decision to make.`
     );
     this.name = 'RobotsDisallowedError';
@@ -41,6 +61,40 @@ export class RobotsDisallowedError extends Error {
   }
 }
 
+/** What a refusal carries as `code`: this gate's three, and the SSRF guard's. */
+const REFUSAL_CODES = new Set(['ROBOTS_DISALLOWED', 'HOST_BLOCKED', 'USE_REDDIT_SEARCH', 'SSRF_BLOCKED']);
+
+/**
+ * The refusal code an error carries, or null when it is a failure rather than
+ * a refusal. A refusal is this gate or the SSRF guard saying no: asking again
+ * gets the same answer, so nothing should retry it.
+ * @param {unknown} error
+ * @returns {string|null}
+ */
+export function gateRefusalCode(error) {
+  for (const code of [error?.code, error?.cause?.code]) {
+    if (REFUSAL_CODES.has(code)) return code;
+  }
+  return null;
+}
+
+/**
+ * Remove a known deployment credential from a URL bound for the audit log: the
+ * raw value and both encodings of it. `URLSearchParams` escapes `~ ! ' ( )` and
+ * writes a space as `+`; `encodeURIComponent` does neither.
+ */
+function redactSecret(value, secret) {
+  if (!secret) return value;
+  const forms = new Set([
+    secret,
+    encodeURIComponent(secret),
+    new URLSearchParams({ s: secret }).toString().slice(2)
+  ]);
+  let redacted = value;
+  for (const form of forms) redacted = redacted.replaceAll(form, '[redacted]');
+  return redacted;
+}
+
 /**
  * One checker per identity, so the robots cache is process-wide rather than
  * per-tool — otherwise every tool would re-fetch the same robots.txt.
@@ -68,6 +122,10 @@ function checkerFor(userAgent) {
  * @param {string}  [options.userAgent] per-request identity override
  * @param {string}  [options.tool] tool name, for the audit row
  * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @param {string}  [options.redactCredential] a deployment credential that
+ *   appears in `url` (a keyed connector puts its key in the query string), to
+ *   remove from the audit row. The row keeps the URL, because it has to say
+ *   what was fetched, and loses the secret.
  * @returns {Promise<{ allowed: boolean, userAgent: string, crawlDelayMs: number,
  *   warnings: string[], overridden: boolean }>}
  * @throws {BlockedHostError} for a permanently blocked host
@@ -119,7 +177,7 @@ export async function robotsPreflight(url, options = {}) {
     const ownerToken = internalOwnerToken();
     recordComplianceEvent({
       event: 'robots_override',
-      url,
+      url: redactSecret(url, options.redactCredential),
       tool: options.tool || null,
       apiKeyId: apiKeyId(options.apiKey ?? AuthManager.getConfig()?.apiKey),
       ...(ownerToken ? { ownerId: apiKeyId(ownerToken) } : {}),
@@ -138,15 +196,57 @@ export async function robotsPreflight(url, options = {}) {
 }
 
 /**
- * The call-site helper: run the gate, honour Crawl-delay and any recorded
- * `Retry-After`, and hand back the identity headers to send.
+ * The gate for a redirect hop, to pass to the fetch as `onRedirect`.
  *
- * @param {string} url
+ * The gate above decides about the URL the caller asked for; a redirect can
+ * lead anywhere. Without this a 301 into a path robots.txt disallows, or onto
+ * a host on the platform blocklist, was fetched because nobody asked again.
+ * Each hop gets the decision a first request would: blocklist, then robots,
+ * with the caller's own `respect_robots` and an audit row when it overrides.
+ *
+ * It then waits its turn as a first request would: the hop is one more request
+ * to its host, so it is spaced by that host's Crawl-delay, by a Retry-After it
+ * has sent and by our own per-host limit. A fetch does not charge that wait to
+ * its timeout (resignedFetch.js). A browser has made the request before anyone
+ * could be asked, so its caller passes `alreadyRequested` and nothing is
+ * slept: there is no request left to space.
+ *
+ * @param {string} from the URL the fetch started at, for the refusal message
  * @param {object} [options] see {@link robotsPreflight}
- * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
- *   warnings: string[], overridden: boolean }>}
- * @throws {BlockedHostError|RobotsDisallowedError}
+ * @returns {(to: string, hop?: { alreadyRequested?: boolean }) => Promise<void>}
+ *   throws BlockedHostError or RobotsDisallowedError to refuse the hop
  */
+export function redirectGate(from, options = {}) {
+  return hopGate(options, { redirectedFrom: from });
+}
+
+/**
+ * The gate for a URL a browser page reached without being sent there: a click
+ * followed a link, a form posted, a script redirected, a client-side router
+ * rewrote the address. The decision is {@link redirectGate}'s; only the
+ * refusal reads differently, because nothing redirected.
+ *
+ * @param {string} [from] the last URL the page was checked at
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {(to: string, hop?: { alreadyRequested?: boolean }) => Promise<void>}
+ *   throws BlockedHostError or RobotsDisallowedError to refuse where the page
+ *   now stands
+ */
+export function pageMoveGate(from, options = {}) {
+  return hopGate(options, { movedFrom: from });
+}
+
+function hopGate(options, reached) {
+  return async (to, { alreadyRequested = false } = {}) => {
+    const decision = await robotsPreflight(to, options);
+    if (!decision.allowed) {
+      markPreflightRefusal('ROBOTS_DISALLOWED');
+      throw new RobotsDisallowedError(to, reached);
+    }
+    if (!alreadyRequested) await throttleHost(to, { crawlDelayMs: decision.crawlDelayMs });
+  };
+}
+
 /**
  * The headers an HTTP page fetch carries: identity, the Web Bot Auth signature
  * when one is configured, and an EMPTY Accept-Language.
@@ -170,6 +270,18 @@ export function outboundHeaders(userAgent, signature = {}) {
   };
 }
 
+/**
+ * The call-site helper: run the gate, honour Crawl-delay and any recorded
+ * `Retry-After`, and hand back the identity headers to send and the gate for
+ * any redirect the fetch follows. Pass both to the fetch.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
+ *   warnings: string[], overridden: boolean,
+ *   onRedirect: (to: string) => Promise<void> }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
 export async function preflightFetch(url, options = {}) {
   const decision = await robotsPreflight(url, options);
   if (!decision.allowed) {
@@ -191,7 +303,8 @@ export async function preflightFetch(url, options = {}) {
     headers: outboundHeaders(decision.userAgent, signature),
     userAgent: decision.userAgent,
     warnings: [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)],
-    overridden: decision.overridden
+    overridden: decision.overridden,
+    onRedirect: redirectGate(url, options)
   };
 }
 

package/src/utils/secretMask.js

@@ -11,7 +11,7 @@
 
 import { redactPii } from 'crawlforge-extractors';
 
-const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
+const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|^proxies$|cookie/i;
 
 const MASK = '[REDACTED]';
 const PARTIAL_MASK_LEN = 4; // show last N chars of long secrets

package/src/utils/sitemapParser.js

@@ -679,7 +679,8 @@ export class SitemapParser {
           ...gate.headers,
           'Accept': 'application/xml,text/xml,text/plain,*/*',
           'Accept-Encoding': 'gzip, deflate'
-        }
+        },
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
       if (response.status === 429 || response.status === 503) {

package/src/utils/ssrfGuard.js

@@ -35,6 +35,7 @@ import net from 'node:net';
 import { Agent, buildConnector } from 'undici';
 import { config } from '../constants/config.js';
 import { SSRFProtection } from './ssrfProtection.js';
+import { fetchResigned } from './resignedFetch.js';
 
 // Reused only for its (well-tested) CIDR range math — no network state.
 const _ssrf = new SSRFProtection();
@@ -307,14 +308,18 @@ export function isSsrfError(err) {
  * blocked targets it throws a clear `SSRF Protection: ...` error (pre-flight) or
  * the fetch rejects at connect time with an SSRF_BLOCKED cause.
  *
+ * A crawl request (one carrying a Web Bot Auth signature, or the gate's
+ * `onRedirect`) has its redirects followed one hop at a time, each through the
+ * same dispatcher, each gated and each signed for itself (see resignedFetch.js).
+ *
  * @param {string} url
- * @param {RequestInit} [options]
+ * @param {RequestInit & { onRedirect?: (to: string) => Promise<void> }} [options]
  * @returns {Promise<Response>}
  */
 export async function safeFetch(url, options = {}) {
   const guard = ssrfGuard(url); // throws on protocol / metadata-host / blocklist / IP-literal violations
   try {
-    return await fetch(url, { ...options, ...guard });
+    return await fetchResigned(url, { ...options, ...guard });
   } catch (err) {
     if (isSsrfError(err)) {
       throw new Error(err.cause?.message || err.message);
@@ -323,27 +328,99 @@ export async function safeFetch(url, options = {}) {
   }
 }
 
+/**
+ * Where a navigation went after the URL it was asked for: every HTTP redirect
+ * hop in order, then the URL the page stands on when that is somewhere else
+ * again (a client-side redirect). Fragments are ignored; they never reach a
+ * server.
+ *
+ * @param {string} landed the page's current URL
+ * @param {string} [url] the URL the navigation was asked for
+ * @param {import('playwright').Response|null} [response] what `page.goto` returned
+ * @returns {string[]}
+ */
+export function navigationHops(landed, url, response) {
+  const bare = (value) => {
+    try {
+      const parsed = new URL(value);
+      parsed.hash = '';
+      return parsed.href;
+    } catch {
+      return value;
+    }
+  };
+  const chain = [];
+  for (let request = response?.request?.(); request; request = request.redirectedFrom()) {
+    chain.unshift(request.url());
+  }
+  const hops = chain.slice(1);
+  if (/^https?:\/\//i.test(landed) && bare(landed) !== bare(hops.at(-1) ?? url)) hops.push(landed);
+  return hops;
+}
+
+/**
+ * The checks a navigation gets once it has happened: the SSRF guard on where
+ * the page landed, and the caller's `onRedirect` gate on every hop that led
+ * there.
+ *
+ * A fetch is refused a redirect before the hop is requested (resignedFetch.js).
+ * A browser follows redirects itself and Playwright hands no redirect hop to a
+ * route handler, so here the hop has already been requested by the time anyone
+ * can ask about it. What is held instead is everything after: a refusal empties
+ * the page before it is thrown, so nothing of the refused document can be read,
+ * acted on or captured, including by a caller that keeps the page. For the same
+ * reason the gate is told the hop is `alreadyRequested`: there is nothing left
+ * for a Crawl-delay to space.
+ *
+ * Call it again with the URL a navigation landed on, and no response, before
+ * reading a page that had time to move on by itself.
+ *
+ * @param {import('playwright').Page} page
+ * @param {string} url the URL the navigation was asked for
+ * @param {import('playwright').Response|null} [response] what `page.goto` returned
+ * @param {(to: string, hop?: { alreadyRequested?: boolean }) => Promise<void>} [onRedirect]
+ *   refuses a hop by throwing
+ * @returns {Promise<string>} the URL the page stood on when it was checked
+ */
+export async function assertNavigationAllowed(page, url, response, onRedirect) {
+  try {
+    const landedUrl = page.url();
+    if (/^https?:\/\//i.test(landedUrl)) {
+      await assertUrlAllowed(landedUrl, { resolveDns: true });
+    }
+    if (onRedirect) {
+      for (const hop of navigationHops(landedUrl, url, response)) {
+        await onRedirect(hop, { alreadyRequested: true });
+      }
+    }
+    return landedUrl;
+  } catch (error) {
+    await page.goto('about:blank').catch(() => {});
+    throw error;
+  }
+}
+
 /**
  * SSRF-safe wrapper around Playwright's `page.goto`. The navigate analog of
  * safeFetch: a browser drives its own DNS resolution, so URL/host checks alone
  * miss DNS-rebinding and private-IP targets — resolveDns:true closes that.
- * Checks the URL before navigating, then re-checks the landed URL, because a
- * redirect during navigation could carry us into a blocked range even when the
+ * Checks the URL before navigating, then re-checks where the navigation went
+ * (see assertNavigationAllowed), because a redirect could carry us into a
+ * blocked range, or past the caller's `onRedirect` gate, even when the
  * original URL was safe. Returns the Response so callers can read the status.
- * Options pass through unchanged; behaviour-preserving for allowed URLs.
+ * Every other option passes through unchanged; behaviour-preserving for
+ * allowed URLs.
  *
  * @param {import('playwright').Page} page
  * @param {string} url
- * @param {Parameters<import('playwright').Page['goto']>[1]} [options]
+ * @param {Parameters<import('playwright').Page['goto']>[1] & { onRedirect?: (to: string) => Promise<void> }} [options]
  * @returns {Promise<import('playwright').Response|null>}
  */
 export async function safeGoto(page, url, options = {}) {
+  const { onRedirect, ...gotoOptions } = options;
   await assertUrlAllowed(url, { resolveDns: true });
-  const response = await page.goto(url, options);
-  const landedUrl = page.url();
-  if (/^https?:\/\//i.test(landedUrl)) {
-    await assertUrlAllowed(landedUrl, { resolveDns: true });
-  }
+  const response = await page.goto(url, gotoOptions);
+  await assertNavigationAllowed(page, url, response, onRedirect);
   return response;
 }