crawlforge-mcp-server 5.2.8 → 5.3.0

package/src/utils/robotsGate.js

@@ -0,0 +1,206 @@
+/**
+ * robotsGate — the one pre-fetch gate every fetching tool goes through.
+ *
+ * Before this module, `RobotsChecker` was instantiated in exactly one place
+ * (BFSCrawler), so only `crawl_deep` honoured robots.txt; `scrape`,
+ * `batch_scrape`, `scrape_template`, `track_changes`, `map_site` and every
+ * `extract_*` tool did no robots check at all. Ground rule G5 says every
+ * fetching tool respects robots.txt by default, so the check has to live at the
+ * fetch boundary rather than in one crawler.
+ *
+ * Order matters. The platform blocklist (G7) is consulted first and is not
+ * overridable by anything a caller can send; robots (G5) is next and *is*
+ * overridable, but only explicitly, with a warning and an audit row; the
+ * host's Crawl-delay (G6) then feeds the per-host throttle.
+ *
+ * Callers replace `await throttleHost(url)` with `await preflightFetch(url, …)`
+ * and spread the returned `headers` into the request.
+ */
+
+import { RobotsChecker } from './robotsChecker.js';
+import { assertHostAllowed } from './hostBlocklist.js';
+import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
+import { throttleHost } from './hostRateLimiter.js';
+import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
+import { signRequestHeaders } from './webBotAuth.js';
+import { markPreflightRefusal } from '../server/requestContext.js';
+import { config } from '../constants/config.js';
+
+export class RobotsDisallowedError extends Error {
+  constructor(url) {
+    super(
+      `robots.txt on ${new URL(url).host} disallows this path for CrawlForge. ` +
+      `Pass respect_robots: false to fetch it anyway — that override is recorded ` +
+      `against your API key and is your decision to make.`
+    );
+    this.name = 'RobotsDisallowedError';
+    this.code = 'ROBOTS_DISALLOWED';
+    this.url = url;
+  }
+}
+
+/**
+ * One checker per identity, so the robots cache is process-wide rather than
+ * per-tool — otherwise every tool would re-fetch the same robots.txt.
+ * @type {Map<string, RobotsChecker>}
+ */
+const checkers = new Map();
+
+function checkerFor(userAgent) {
+  let checker = checkers.get(userAgent);
+  if (!checker) {
+    checker = new RobotsChecker(userAgent);
+    checkers.set(userAgent, checker);
+  }
+  return checker;
+}
+
+/**
+ * Decide whether a URL may be fetched. Pure decision — does no throttling and
+ * sends no request other than the (cached) robots.txt lookup.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ * @param {boolean} [options.respectRobots] per-request override; defaults to
+ *   `config.crawling.respectRobots`. `false` is honoured, warned about, audited.
+ * @param {string}  [options.userAgent] per-request identity override
+ * @param {string}  [options.tool] tool name, for the audit row
+ * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @returns {Promise<{ allowed: boolean, userAgent: string, crawlDelayMs: number,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError} for a permanently blocked host
+ */
+export async function robotsPreflight(url, options = {}) {
+  // G7 — first, and not overridable. Stamp before rethrowing so a blocked host
+  // costs the caller nothing: we refused, we fetched nothing.
+  try {
+    assertHostAllowed(url);
+  } catch (error) {
+    if (error?.code === 'HOST_BLOCKED') markPreflightRefusal('HOST_BLOCKED');
+    throw error;
+  }
+
+  const userAgent = resolveUserAgent(options.userAgent);
+  const warnings = [];
+
+  const explicitOverride = options.respectRobots === false;
+  const respect = options.respectRobots === undefined
+    ? config.crawling.respectRobots
+    : options.respectRobots !== false;
+
+  const checker = checkerFor(userAgent);
+  let allowed = true;
+  let crawlDelayMs = 0;
+
+  try {
+    allowed = await checker.canFetch(url);
+    crawlDelayMs = (await checker.fetchCrawlDelay(url)) * 1000;
+  } catch {
+    // Unreadable robots.txt is not a disallow (see RobotsChecker.canFetch).
+    allowed = true;
+  }
+
+  if (explicitOverride) {
+    warnings.push(
+      allowed
+        ? 'respect_robots was disabled for this request. robots.txt did not disallow this URL, so the override changed nothing. The request is recorded against your API key.'
+        : `respect_robots was disabled for this request and robots.txt on ${new URL(url).host} disallows this path. Fetching anyway is your decision and is recorded against your API key.`
+    );
+    recordComplianceEvent({
+      event: 'robots_override',
+      url,
+      tool: options.tool || null,
+      apiKeyId: apiKeyId(options.apiKey),
+      userAgent,
+      robotsAllowed: allowed
+    });
+  }
+
+  return {
+    allowed: allowed || !respect,
+    userAgent,
+    crawlDelayMs,
+    warnings,
+    overridden: explicitOverride && !allowed
+  };
+}
+
+/**
+ * The call-site helper: run the gate, honour Crawl-delay and any recorded
+ * `Retry-After`, and hand back the identity headers to send.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function preflightFetch(url, options = {}) {
+  const decision = await robotsPreflight(url, options);
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url);
+  }
+
+  await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
+
+  // Web Bot Auth: when a signing key is configured, every request also carries
+  // a signature a site owner can verify against our published key. No key
+  // configured means no headers and no behaviour change. Requests with a
+  // caller-supplied userAgent override are still signed — the signature covers
+  // @authority, not the UA, and it identifies the operator (us), not the
+  // identity the caller asked us to present.
+  const signature = signRequestHeaders(url) || {};
+
+  return {
+    headers: { ...identityHeaders({ userAgent: decision.userAgent }), ...signature },
+    userAgent: decision.userAgent,
+    warnings: decision.warnings,
+    overridden: decision.overridden
+  };
+}
+
+/**
+ * The gate for browser paths. Same decision as {@link preflightFetch}, minus
+ * the identity and signature headers — those belong on an HTTP fetch, not on a
+ * browser context that presents its own identity.
+ *
+ * Deliberately takes no `userAgent`: robots.txt is matched against our
+ * canonical product token even when the browser presents another UA. Matching
+ * on the presented UA would let browser traffic walk past the rules our own
+ * token is bound by, which is the G5 hole this gate exists to close.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ * @param {boolean} [options.respectRobots] per-request override
+ * @param {string}  [options.tool] tool name, for the audit row
+ * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @returns {Promise<string[]>} warnings to surface on the response
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function browserPreflight(url, options = {}) {
+  const decision = await robotsPreflight(url, {
+    respectRobots: options.respectRobots,
+    tool: options.tool,
+    apiKey: options.apiKey
+  });
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url);
+  }
+
+  await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
+  return decision.warnings;
+}
+
+/** Test/diagnostic hook: drop every cached robots.txt. */
+export function _resetRobotsGate() {
+  checkers.clear();
+}
+
+/** Test/diagnostic hook: total robots.txt requests made across all identities. */
+export function _robotsFetchCount() {
+  let total = 0;
+  for (const checker of checkers.values()) total += checker.fetchCount;
+  return total;
+}

package/src/utils/sitemapParser.js

@@ -4,13 +4,16 @@ import { promisify } from 'util';
 import { CacheManager } from '../core/cache/CacheManager.js';
 import { normalizeUrl } from './urlNormalizer.js';
 import { safeFetch } from './ssrfGuard.js';
+import { CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+import { preflightFetch } from './robotsGate.js';
+import { noteRetryAfter } from './hostRateLimiter.js';
 
 const gunzip = promisify(zlib.gunzip);
 
 export class SitemapParser {
   constructor(options = {}) {
     const {
-      userAgent = 'CrawlForge/1.0',
+      userAgent = CRAWLFORGE_USER_AGENT,
       timeout = 10000,
       maxRecursionDepth = 3,
       maxUrlsPerSitemap = 50000,
@@ -50,13 +53,17 @@ export class SitemapParser {
    * Parse a sitemap from a URL with full feature support
    * @param {string} url - Sitemap URL
    * @param {Object} options - Parsing options
+   * @param {boolean} [options.respectRobots] - Per-request robots override.
+   *   Per call, not per instance: MapSiteTool builds one SitemapParser and
+   *   reuses it, so a flag stored on the instance would leak between requests.
    * @returns {Promise<Object>} Parsed sitemap data
    */
   async parseSitemap(url, options = {}) {
     const {
       includeMetadata = true,
       followIndexes = true,
-      maxDepth = this.maxRecursionDepth
+      maxDepth = this.maxRecursionDepth,
+      respectRobots
     } = options;
 
     // Reset stats for new parsing session
@@ -72,7 +79,8 @@ export class SitemapParser {
     try {
       const result = await this._parseSitemapRecursive(url, 0, maxDepth, {
         includeMetadata,
-        followIndexes
+        followIndexes,
+        respectRobots
       });
 
       return {
@@ -99,11 +107,12 @@ export class SitemapParser {
   /**
    * Parse sitemap index files and return all contained sitemaps
    * @param {string} indexUrl - Sitemap index URL
+   * @param {boolean} [respectRobots] - Per-request robots override
    * @returns {Promise<Array>} Array of sitemap URLs with metadata
    */
-  async parseSitemapIndex(indexUrl) {
+  async parseSitemapIndex(indexUrl, respectRobots) {
     try {
-      const content = await this._fetchSitemapContent(indexUrl);
+      const content = await this._fetchSitemapContent(indexUrl, respectRobots);
       if (!content) return [];
 
       const $ = load(content, { xmlMode: true });
@@ -260,9 +269,10 @@ export class SitemapParser {
    * Discover sitemap URLs from various sources
    * @param {string} baseUrl - Base URL of the website
    * @param {Object} sources - Sources to check
+   * @param {boolean} [respectRobots] - Per-request robots override
    * @returns {Promise<Array>} Array of discovered sitemap URLs
    */
-  async discoverSitemaps(baseUrl, sources = {}) {
+  async discoverSitemaps(baseUrl, sources = {}, respectRobots) {
     const {
       checkRobotsTxt = true,
       checkCommonPaths = true,
@@ -277,7 +287,7 @@ export class SitemapParser {
     if (checkRobotsTxt) {
       try {
         const robotsUrl = `${baseOrigin}/robots.txt`;
-        const robotsContent = await this._fetchWithTimeout(robotsUrl);
+        const robotsContent = await this._fetchWithTimeout(robotsUrl, respectRobots);
         if (robotsContent) {
           const sitemapMatches = robotsContent.match(/^Sitemap:\s*(.+)$/gmi);
           if (sitemapMatches) {
@@ -308,7 +318,7 @@ export class SitemapParser {
       for (const path of commonPaths) {
         const sitemapUrl = `${baseOrigin}${path}`;
         try {
-          const response = await this._fetchWithTimeoutResponse(sitemapUrl);
+          const response = await this._fetchWithTimeoutResponse(sitemapUrl, respectRobots);
           if (response && response.ok) {
             discovered.add(sitemapUrl);
           }
@@ -344,7 +354,7 @@ export class SitemapParser {
     }
 
     try {
-      const content = await this._fetchSitemapContent(url);
+      const content = await this._fetchSitemapContent(url, options.respectRobots);
       if (!content) {
         throw new Error(`Failed to fetch sitemap content from ${url}`);
       }
@@ -385,9 +395,9 @@ export class SitemapParser {
    * Fetch and decompress sitemap content
    * @private
    */
-  async _fetchSitemapContent(url) {
+  async _fetchSitemapContent(url, respectRobots) {
     try {
-      const response = await this._fetchWithTimeoutResponse(url);
+      const response = await this._fetchWithTimeoutResponse(url, respectRobots);
       if (!response || !response.ok) {
         return null;
       }
@@ -621,8 +631,8 @@ export class SitemapParser {
    * Fetch with timeout
    * @private
    */
-  async _fetchWithTimeout(url) {
-    const response = await this._fetchWithTimeoutResponse(url);
+  async _fetchWithTimeout(url, respectRobots) {
+    const response = await this._fetchWithTimeoutResponse(url, respectRobots);
     return response ? await response.text() : null;
   }
 
@@ -630,7 +640,12 @@ export class SitemapParser {
    * Fetch with timeout returning response object
    * @private
    */
-  async _fetchWithTimeoutResponse(url) {
+  async _fetchWithTimeoutResponse(url, respectRobots) {
+    const gate = await preflightFetch(url, {
+      respectRobots,
+      userAgent: this.userAgent,
+      tool: 'sitemap'
+    });
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
@@ -638,12 +653,15 @@ export class SitemapParser {
       const response = await safeFetch(url, {
         signal: controller.signal,
         headers: {
-          'User-Agent': this.userAgent,
+          ...gate.headers,
           'Accept': 'application/xml,text/xml,text/plain,*/*',
           'Accept-Encoding': 'gzip, deflate'
         }
       });
       clearTimeout(timeoutId);
+      if (response.status === 429 || response.status === 503) {
+        noteRetryAfter(url, response.headers.get('retry-after'));
+      }
       return response;
     } catch (error) {
       clearTimeout(timeoutId);

package/src/utils/ssrfProtection.js

@@ -6,6 +6,7 @@
 import { promisify } from 'util';
 import dns from 'dns';
 import net from 'net';
+import { identityHeaders } from './fetchIdentity.js';
 
 const dnsLookup = promisify(dns.lookup);
 
@@ -560,7 +561,7 @@ export class SSRFProtection {
         timeout: Math.min(fetchOptions.timeout || 30000, this.config.maxTimeout),
         redirect: 'manual', // Handle redirects manually
         headers: {
-          'User-Agent': 'CrawlForge/3.0 (Security Enhanced)',
+          ...identityHeaders({ role: 'health-check' }),
           ...fetchOptions.headers
         }
       };

package/src/utils/webBotAuth.js

@@ -0,0 +1,193 @@
+/**
+ * webBotAuth — signs outbound requests so a site owner can verify who we are.
+ *
+ * A User-Agent is a claim anyone can make. Web Bot Auth turns our identity into
+ * something a site can check: an Ed25519 signature over the request, per
+ * RFC 9421 (HTTP Message Signatures) with the `web-bot-auth` profile from
+ * draft-meunier-web-bot-auth-architecture. The public key is published at
+ * `/.well-known/http-message-signatures-directory` on crawlforge.dev.
+ *
+ * This is the mechanism behind ground rule G4. Honest identification only helps
+ * a site owner if it cannot be spoofed by someone else claiming to be us.
+ *
+ * Signing is OPT-IN and absent by default: with no key configured every export
+ * here is a no-op and requests go out exactly as before. Key material lives
+ * only in the environment, never in the repo.
+ *
+ * Verified against the official test vectors (architecture draft Appendix A.2.1
+ * with the RFC 9421 Appendix B.1.4 key) — see tests/unit/webBotAuth.test.js.
+ */
+
+import { createHash, createPrivateKey, createPublicKey, sign, randomBytes } from 'crypto';
+
+/** The profile tag every web-bot-auth signature carries. */
+const WEB_BOT_AUTH_TAG = 'web-bot-auth';
+
+/** The draft RECOMMENDS an expiry no more than 24 hours; ours is far shorter. */
+const DEFAULT_LIFETIME_SECONDS = 300;
+
+let cachedKey; // undefined = not yet resolved, null = none configured
+
+/**
+ * The raw 32-byte Ed25519 public key as base64url, which is the JWK `x`.
+ * @param {import('crypto').KeyObject} publicKey
+ * @returns {string}
+ */
+function publicKeyX(publicKey) {
+  // An Ed25519 SPKI DER is a 12-byte header followed by the 32-byte key.
+  const der = publicKey.export({ type: 'spki', format: 'der' });
+  return Buffer.from(der.subarray(der.length - 32)).toString('base64url');
+}
+
+/**
+ * RFC 8037 Appendix A.3 thumbprint: SHA-256 over the canonical JWK with its
+ * member names in lexicographic order and no whitespace, base64url unpadded.
+ * The member order is load-bearing — reordering it changes the key id.
+ * @param {string} x base64url raw public key
+ * @returns {string}
+ */
+export function jwkThumbprint(x) {
+  const canonical = JSON.stringify({ crv: 'Ed25519', kty: 'OKP', x });
+  return createHash('sha256').update(canonical).digest('base64url');
+}
+
+/**
+ * The public JWK for a key pair, in the shape the directory publishes.
+ * @param {import('crypto').KeyObject} publicKey
+ */
+export function publicJwk(publicKey) {
+  const x = publicKeyX(publicKey);
+  return { kty: 'OKP', crv: 'Ed25519', kid: jwkThumbprint(x), x, use: 'sig', alg: 'ed25519' };
+}
+
+/**
+ * Resolve the signing key from the environment, or null when none is set.
+ *
+ * `CRAWLFORGE_SIGNING_KEY` holds an Ed25519 private key as a PKCS#8 PEM —
+ * either literally (with real newlines) or base64-encoded, since most secret
+ * stores mangle multi-line values. A malformed key is a configuration error we
+ * surface once and then ignore: it must not take every fetch down with it.
+ *
+ * @returns {{ privateKey: import('crypto').KeyObject, jwk: object } | null}
+ */
+export function getSigningKey() {
+  if (cachedKey !== undefined) return cachedKey;
+
+  const raw = process.env.CRAWLFORGE_SIGNING_KEY;
+  if (!raw || !raw.trim()) {
+    cachedKey = null;
+    return cachedKey;
+  }
+
+  try {
+    const pem = raw.includes('-----BEGIN')
+      ? raw.replace(/\\n/g, '\n')
+      : Buffer.from(raw.trim(), 'base64').toString('utf8');
+
+    const privateKey = createPrivateKey(pem);
+    if (privateKey.asymmetricKeyType !== 'ed25519') {
+      throw new Error(`expected an ed25519 key, got ${privateKey.asymmetricKeyType}`);
+    }
+    cachedKey = { privateKey, jwk: publicJwk(createPublicKey(privateKey)) };
+  } catch (error) {
+    console.error(
+      `[web-bot-auth] CRAWLFORGE_SIGNING_KEY could not be loaded, so requests will go out unsigned: ${error.message}`
+    );
+    cachedKey = null;
+  }
+
+  return cachedKey;
+}
+
+/**
+ * Serialise the @signature-params of a signature base.
+ * Parameter order is part of the signed bytes, so it must match what the
+ * verifier reconstructs — it is the draft's order, not an arbitrary one.
+ */
+function signatureParams(components, { created, expires, keyid, nonce }) {
+  const covered = components.map((c) => `"${c}"`).join(' ');
+  return (
+    `(${covered})` +
+    `;created=${created}` +
+    `;keyid="${keyid}"` +
+    `;alg="ed25519"` +
+    `;expires=${expires}` +
+    `;nonce="${nonce}"` +
+    `;tag="${WEB_BOT_AUTH_TAG}"`
+  );
+}
+
+/**
+ * Build the RFC 9421 signature base for a request.
+ *
+ * Exported for the test vectors: the base is the exact byte string that gets
+ * signed, so reproducing the published one is what proves the implementation
+ * interoperates rather than merely agreeing with itself.
+ *
+ * @param {{ authority: string, signatureAgent?: string|null }} request
+ * @param {{ created: number, expires: number, keyid: string, nonce: string }} params
+ * @returns {{ base: string, components: string[], params: string }}
+ */
+export function buildSignatureBase(request, params) {
+  const components = ['@authority'];
+  const lines = [`"@authority": ${request.authority}`];
+
+  // The draft requires Signature-Agent to be covered whenever it is sent.
+  if (request.signatureAgent) {
+    components.push('signature-agent');
+    lines.push(`"signature-agent": "${request.signatureAgent}"`);
+  }
+
+  const serialised = signatureParams(components, params);
+  lines.push(`"@signature-params": ${serialised}`);
+
+  return { base: lines.join('\n'), components, params: serialised };
+}
+
+/**
+ * Signature headers for an outbound request, or null when signing is off.
+ *
+ * @param {string} url the request target
+ * @param {object} [options]
+ * @param {string|null} [options.signatureAgent] directory URL to advertise
+ * @param {number} [options.now] epoch seconds, for deterministic tests
+ * @param {number} [options.expires] epoch seconds, for deterministic tests
+ * @param {string} [options.nonce] base64 nonce, for deterministic tests
+ * @returns {Record<string,string>|null}
+ */
+export function signRequestHeaders(url, options = {}) {
+  const key = getSigningKey();
+  if (!key) return null;
+
+  let authority;
+  try {
+    authority = new URL(url).host;
+  } catch {
+    return null; // not our job to validate URLs; the fetch path already does
+  }
+
+  const created = options.now ?? Math.floor(Date.now() / 1000);
+  const expires = options.expires ?? created + DEFAULT_LIFETIME_SECONDS;
+  // The draft RECOMMENDS 64 random bytes, unique within the validity window.
+  const nonce = options.nonce ?? randomBytes(64).toString('base64');
+  const signatureAgent = options.signatureAgent ?? process.env.WEB_BOT_AUTH_DIRECTORY ?? null;
+
+  const { base, params } = buildSignatureBase(
+    { authority, signatureAgent },
+    { created, expires, keyid: key.jwk.kid, nonce }
+  );
+
+  const signature = sign(null, Buffer.from(base, 'utf8'), key.privateKey).toString('base64');
+
+  const headers = {
+    'Signature-Input': `sig1=${params}`,
+    'Signature': `sig1=:${signature}:`
+  };
+  if (signatureAgent) headers['Signature-Agent'] = `"${signatureAgent}"`;
+  return headers;
+}
+
+/** Test hook: forget the resolved key so a changed env var is picked up. */
+export function _resetSigningKey() {
+  cachedKey = undefined;
+}