crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/utils/ssrfGuard.js

@@ -3,9 +3,10 @@
  *
  * This wires the (previously unused) SSRF protections into the actual scraping
  * fetch helpers. Enforcement happens at TCP connect time via a custom undici
- * dispatcher `lookup`, so it covers the initial request, every redirect hop, and
- * closes the DNS-rebinding window (the validated IP is the one connected to —
- * there is no second, unchecked resolution).
+ * connector, so it covers the initial request, every redirect hop (including
+ * hops that land on a bare IP literal, which Node's `net.connect` never hands
+ * to a `lookup` callback), and closes the DNS-rebinding window (the validated
+ * IP is the one connected to — there is no second, unchecked resolution).
  *
  * Two levels:
  *  - Stage 1 (default): blocks connections to loopback, link-local /
@@ -14,14 +15,24 @@
  *  - Stage 2 (SSRF_STRICT=true): full private-range enforcement (RFC1918, ULA,
  *    multicast, CGNAT, etc.) via the existing SSRFProtection range logic.
  *
+ * IP-literal hostnames (loopback/metadata expressed as dotted-quad, decimal,
+ * hex, or IPv4-mapped/compatible IPv6 such as `::ffff:127.0.0.1`) are checked
+ * directly against the same rules — DNS resolution is not the only path in.
+ *
  * Controls (backwards-compatible defaults):
  *  - SSRF_PROTECTION_ENABLED=false  -> disable the guard entirely (kill switch).
  *  - ALLOWED_DOMAINS=a.com,b.com    -> bypass the guard for trusted hosts
  *    (e.g. a local dev server at localhost). Matches host or any subdomain.
+ *    Checked fresh for every hop (initial request and each redirect), so an
+ *    allowlisted first hop does not unguard a subsequent hop to a different,
+ *    non-allowlisted host.
+ *  - BLOCKED_DOMAINS=a.com,b.com    -> extra hostname denylist (host or
+ *    subdomain match), checked at pre-flight alongside the IP-literal checks.
  *  - SSRF_STRICT=true               -> Stage 2 full enforcement.
  */
 import dns from 'node:dns';
-import { Agent } from 'undici';
+import net from 'node:net';
+import { Agent, buildConnector } from 'undici';
 import { config } from '../constants/config.js';
 import { SSRFProtection } from './ssrfProtection.js';
 
@@ -38,36 +49,144 @@ function strictMode() {
   return process.env.SSRF_STRICT === 'true';
 }
 
+function stripBrackets(host) {
+  return host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host;
+}
+
+/**
+ * Extracts the embedded IPv4 address from an IPv4-mapped/compatible IPv6
+ * literal (e.g. `::ffff:127.0.0.1` or its fully-expanded hex-group form
+ * `0:0:0:0:0:ffff:7f00:1`), returning null if `ip` isn't one of those forms.
+ * @param {string} ip
+ * @returns {string|null}
+ */
+function extractMappedIPv4(ip) {
+  let s = ip.toLowerCase().split('%')[0]; // drop a zone id, if present
+
+  // A dotted-quad tail (e.g. "::ffff:127.0.0.1") -> two hex groups, so the
+  // rest of this function only has to deal with one address shape.
+  const dottedTail = s.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/);
+  if (dottedTail) {
+    const quad = dottedTail[2].split('.').map(Number);
+    if (quad.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return null;
+    const hi = ((quad[0] << 8) | quad[1]).toString(16);
+    const lo = ((quad[2] << 8) | quad[3]).toString(16);
+    s = `${dottedTail[1]}${hi}:${lo}`;
+  }
+
+  let groups;
+  if (s.includes('::')) {
+    const [left, right] = s.split('::');
+    const leftParts = left ? left.split(':').filter(Boolean) : [];
+    const rightParts = right ? right.split(':').filter(Boolean) : [];
+    const missing = 8 - leftParts.length - rightParts.length;
+    if (missing < 0) return null;
+    groups = [...leftParts, ...Array(missing).fill('0'), ...rightParts];
+  } else {
+    groups = s.split(':');
+  }
+  if (groups.length !== 8) return null;
+
+  const first5AreZero = groups.slice(0, 5).every((g) => parseInt(g || '0', 16) === 0);
+  if (!first5AreZero || groups[5] !== 'ffff') return null;
+
+  const hi = parseInt(groups[6], 16);
+  const lo = parseInt(groups[7], 16);
+  if (Number.isNaN(hi) || Number.isNaN(lo)) return null;
+  return `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
+}
+
 /**
- * Whether a resolved IP must be blocked for the current mode.
+ * Whether a resolved (or literal) IP must be blocked for the current mode.
+ * IPv4-mapped/compatible IPv6 literals are normalized to their embedded IPv4
+ * address first, so they can't slip past the same checks a bare IPv4 gets.
  * @param {string} ip
  * @returns {boolean}
  */
 export function ipBlocked(ip) {
+  const mapped = net.isIPv6(ip) ? extractMappedIPv4(ip) : null;
+  const effectiveIp = mapped || ip;
   if (strictMode()) {
     // Full enforcement: anything not explicitly allowed by SSRFProtection.
-    return !_ssrf.isIPAllowed(ip);
+    return !_ssrf.isIPAllowed(effectiveIp);
+  }
+  if (effectiveIp === '127.0.0.1' || effectiveIp === '::1' || effectiveIp === '0.0.0.0') return true;
+  return STAGE1_RANGES.some((range) => _ssrf.isIPInRange(effectiveIp, range));
+}
+
+function ssrfBlockedError(message) {
+  return Object.assign(new Error(`SSRF Protection: ${message}`), { code: 'SSRF_BLOCKED' });
+}
+
+function throwBlocked(message) {
+  throw ssrfBlockedError(message);
+}
+
+function hostMatchesList(host, list) {
+  return (list || []).some((d) => {
+    const dd = String(d).trim().toLowerCase();
+    return dd && (host === dd || host.endsWith('.' + dd));
+  });
+}
+
+function isAllowlisted(host, allowed) {
+  return hostMatchesList(host, allowed);
+}
+
+function isBlockedDomain(host, blocked) {
+  return hostMatchesList(host, blocked);
+}
+
+/**
+ * Shared, synchronous pre-flight: protocol, allowlist, metadata hosts,
+ * BLOCKED_DOMAINS, and (for IP-literal hosts) ipBlocked(). Throws
+ * (code SSRF_BLOCKED) on any violation. Used by both `ssrfGuard()` and
+ * `assertUrlAllowed()` so the two never drift.
+ * @param {URL} u
+ * @param {object} sec config.security.ssrfProtection
+ * @returns {{ host: string, allowlisted: boolean, ipLiteral: number }}
+ */
+function preflightHostCheck(u, sec) {
+  if (!['http:', 'https:'].includes(u.protocol)) {
+    throwBlocked(`protocol '${u.protocol}' is not allowed`);
+  }
+
+  const host = stripBrackets(u.hostname.toLowerCase());
+  if (isAllowlisted(host, sec.allowedDomains)) {
+    return { host, allowlisted: true, ipLiteral: net.isIP(host) };
+  }
+
+  if (METADATA_HOSTS.has(host)) {
+    throwBlocked(`blocked metadata host '${host}'`);
   }
-  if (ip === '127.0.0.1' || ip === '::1' || ip === '0.0.0.0') return true;
-  return STAGE1_RANGES.some((range) => _ssrf.isIPInRange(ip, range));
+  if (isBlockedDomain(host, sec.blockedDomains)) {
+    throwBlocked(`blocked hostname '${host}'`);
+  }
+
+  const ipLiteral = net.isIP(host);
+  if (ipLiteral && ipBlocked(host)) {
+    throwBlocked(`blocked IP literal '${host}'`);
+  }
+
+  return { host, allowlisted: false, ipLiteral };
 }
 
 /**
  * undici connect-time lookup: resolves the host, rejects if ANY resolved address
  * is blocked, otherwise hands undici the validated address(es) — so the socket
- * connects to exactly what we checked (rebinding-safe).
+ * connects to exactly what we checked (rebinding-safe). Allowlisted hostnames
+ * are resolved normally, with no address filtering.
  */
 function ssrfLookup(hostname, opts, callback) {
+  const sec = config.security?.ssrfProtection;
+  if (isAllowlisted(hostname.toLowerCase(), sec?.allowedDomains)) {
+    return dns.lookup(hostname, opts, callback);
+  }
   dns.lookup(hostname, { all: true, verbatim: true }, (err, addresses) => {
     if (err) return callback(err);
     for (const { address } of addresses) {
       if (ipBlocked(address)) {
-        return callback(
-          Object.assign(
-            new Error(`SSRF Protection: ${hostname} resolves to blocked address ${address}`),
-            { code: 'SSRF_BLOCKED' }
-          )
-        );
+        return callback(ssrfBlockedError(`${hostname} resolves to blocked address ${address}`));
       }
     }
     if (opts && opts.all) return callback(null, addresses);
@@ -76,26 +195,43 @@ function ssrfLookup(hostname, opts, callback) {
   });
 }
 
+/**
+ * Wraps undici's base connector to also cover IP-literal hosts, which
+ * `net.connect` never routes through `lookup` — so hostname-based DNS
+ * rebinding checks (ssrfLookup, above) alone can't catch a redirect straight
+ * to e.g. `http://127.0.0.1/`. Checked per-connect, so every hop (initial
+ * request and each redirect) is validated independently.
+ */
+function guardedConnect(baseConnect) {
+  return function connect(opts, callback) {
+    const hostname = stripBrackets(String(opts.hostname || opts.host || '').toLowerCase());
+    if (net.isIP(hostname)) {
+      const sec = config.security?.ssrfProtection;
+      if (!isAllowlisted(hostname, sec?.allowedDomains) && ipBlocked(hostname)) {
+        return callback(ssrfBlockedError(`blocked IP literal '${hostname}'`));
+      }
+    }
+    return baseConnect(opts, callback);
+  };
+}
+
 let _agent = null;
 function guardedDispatcher() {
   if (!_agent) {
-    _agent = new Agent({ connect: { lookup: ssrfLookup } });
+    const baseConnect = buildConnector({ lookup: ssrfLookup });
+    _agent = new Agent({ connect: guardedConnect(baseConnect) });
   }
   return _agent;
 }
 
-function isAllowlisted(host, allowed) {
-  return (allowed || []).some((d) => {
-    const dd = String(d).trim().toLowerCase();
-    return dd && (host === dd || host.endsWith('.' + dd));
-  });
-}
-
 /**
  * Pre-flight check + dispatcher selection for an outbound scrape target.
- * Returns `{ dispatcher }` to spread into fetch options. `dispatcher` is
- * undefined when the guard is disabled or the host is explicitly allowlisted.
- * Throws (code SSRF_BLOCKED) for protocol / metadata-host pre-flight violations.
+ * Returns `{ dispatcher }` to spread into fetch options; the dispatcher is
+ * always the guarded one (it enforces allowlist/blocklist/IP checks per-hop
+ * internally) so redirects can never escape it. `{}` is returned only when
+ * the guard is disabled outright (kill switch). Throws (code SSRF_BLOCKED)
+ * for pre-flight violations (protocol, metadata host, BLOCKED_DOMAINS, or a
+ * blocked IP-literal host) on the initial URL.
  *
  * @param {string} url
  * @returns {{ dispatcher?: import('undici').Agent }}
@@ -111,22 +247,52 @@ export function ssrfGuard(url) {
     return {}; // let fetch surface its own invalid-URL error
   }
 
-  if (!['http:', 'https:'].includes(u.protocol)) {
-    throw Object.assign(new Error(`SSRF Protection: protocol '${u.protocol}' is not allowed`), {
-      code: 'SSRF_BLOCKED',
-    });
-  }
+  preflightHostCheck(u, sec); // throws on violation
 
-  const host = u.hostname.toLowerCase();
-  if (isAllowlisted(host, sec.allowedDomains)) return {}; // explicit escape hatch
+  return { dispatcher: guardedDispatcher() };
+}
 
-  if (METADATA_HOSTS.has(host)) {
-    throw Object.assign(new Error(`SSRF Protection: blocked metadata host '${host}'`), {
-      code: 'SSRF_BLOCKED',
-    });
+/**
+ * Shared pre-flight helper for subsystems that don't go through `safeFetch`
+ * (e.g. Playwright navigation). Resolves (returns undefined) when the URL is
+ * allowed; throws an Error with `code: 'SSRF_BLOCKED'` (message starting
+ * `SSRF Protection:`) when blocked.
+ *
+ * Kill switch and allowlist behave exactly as in `ssrfGuard()`. When
+ * `resolveDns` is true and the host is a (non-allowlisted, non-IP-literal)
+ * hostname, it is resolved via `dns.promises.lookup` and every returned
+ * address is checked with `ipBlocked()`; a DNS failure here is not itself
+ * treated as a block (the caller's own fetch/connect will surface it).
+ *
+ * @param {string} urlString
+ * @param {{ resolveDns?: boolean }} [opts]
+ * @returns {Promise<void>}
+ */
+export async function assertUrlAllowed(urlString, { resolveDns = false } = {}) {
+  const sec = config.security?.ssrfProtection;
+  if (!sec || sec.enabled === false) return; // kill switch
+
+  let u;
+  try {
+    u = new URL(urlString);
+  } catch {
+    return; // let the caller's own URL parsing surface the error
   }
 
-  return { dispatcher: guardedDispatcher() };
+  const { host, allowlisted, ipLiteral } = preflightHostCheck(u, sec); // throws on violation
+  if (allowlisted || ipLiteral || !resolveDns) return;
+
+  let addresses;
+  try {
+    addresses = await dns.promises.lookup(host, { all: true, verbatim: true });
+  } catch {
+    return; // DNS failures are not ours to enforce; let the caller's own fetch surface them
+  }
+  for (const { address } of addresses) {
+    if (ipBlocked(address)) {
+      throwBlocked(`${host} resolves to blocked address ${address}`);
+    }
+  }
 }
 
 /** True if an error (or its fetch `cause`) came from the SSRF guard. */
@@ -146,7 +312,7 @@ export function isSsrfError(err) {
  * @returns {Promise<Response>}
  */
 export async function safeFetch(url, options = {}) {
-  const guard = ssrfGuard(url); // throws on protocol / metadata-host violations
+  const guard = ssrfGuard(url); // throws on protocol / metadata-host / blocklist / IP-literal violations
   try {
     return await fetch(url, { ...options, ...guard });
   } catch (err) {
@@ -158,4 +324,10 @@ export async function safeFetch(url, options = {}) {
 }
 
 // Exposed for unit tests.
-export const __ssrfInternals = { ssrfLookup, isAllowlisted, STAGE1_RANGES };
+export const __ssrfInternals = {
+  ssrfLookup,
+  isAllowlisted,
+  isBlockedDomain,
+  extractMappedIPv4,
+  STAGE1_RANGES,
+};

package/src/utils/urlNormalizer.js

@@ -20,8 +20,8 @@ export function normalizeUrl(url) {
     if (urlObj.search) {
       const params = new URLSearchParams(urlObj.search);
       const sortedParams = new URLSearchParams();
-      [...params.keys()].sort().forEach(key => {
-        sortedParams.append(key, params.get(key));
+      [...params.entries()].sort(([a], [b]) => a.localeCompare(b)).forEach(([key, value]) => {
+        sortedParams.append(key, value);
       });
       urlObj.search = sortedParams.toString();
     }