crawlforge-mcp-server 5.2.9 → 5.3.1

package/src/utils/robotsChecker.js

@@ -1,68 +1,123 @@
 import robotsParser from 'robots-parser';
 import { safeFetch } from './ssrfGuard.js';
+import { identityHeaders, CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+
+/**
+ * The token this crawler used to identify as, honoured as a source of disallow
+ * only so that robots.txt rules written against the old name keep working.
+ */
+const LEGACY_PRODUCT_TOKEN = 'CrawlForge-Bot';
+
+/** How long a parsed robots.txt stays good for. */
+const DEFAULT_TTL_MS = parseInt(process.env.ROBOTS_CACHE_TTL_MS || '3600000', 10); // 1h
 
 export class RobotsChecker {
-  constructor(userAgent = 'CrawlForge/1.0') {
+  /**
+   * @param {string} [userAgent] identity the robots rules are evaluated against
+   * @param {{ ttlMs?: number }} [options]
+   */
+  constructor(userAgent = CRAWLFORGE_USER_AGENT, options = {}) {
     this.userAgent = userAgent;
+    this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
+    /** @type {Map<string, { robots: unknown, fetchedAt: number }>} */
     this.robotsCache = new Map();
+    /** In-flight fetches, so N concurrent requests to one host fetch robots once. */
+    this.inflight = new Map();
+    /** Diagnostic: how many robots.txt requests this checker has actually made. */
+    this.fetchCount = 0;
+  }
+
+  static robotsUrlFor(url) {
+    const urlObj = new URL(url);
+    return `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+  }
+
+  /**
+   * Parsed robots.txt for a URL's host, served from cache while it is fresh.
+   * Concurrent callers share one in-flight fetch rather than each starting one.
+   */
+  async getRobots(url) {
+    const robotsUrl = RobotsChecker.robotsUrlFor(url);
+
+    const cached = this.robotsCache.get(robotsUrl);
+    if (cached && Date.now() - cached.fetchedAt < this.ttlMs) return cached.robots;
+
+    const pending = this.inflight.get(robotsUrl);
+    if (pending) return pending;
+
+    const promise = (async () => {
+      const robotsTxt = await this.fetchRobotsTxt(robotsUrl);
+      const robots = robotsParser(robotsUrl, robotsTxt);
+      this.robotsCache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+      return robots;
+    })().finally(() => this.inflight.delete(robotsUrl));
+
+    this.inflight.set(robotsUrl, promise);
+    return promise;
   }
 
   async canFetch(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      
-      let robots = this.robotsCache.get(robotsUrl);
-      
-      if (!robots) {
-        const robotsTxt = await this.fetchRobotsTxt(robotsUrl);
-        robots = robotsParser(robotsUrl, robotsTxt);
-        this.robotsCache.set(robotsUrl, robots);
-      }
-      
-      return robots.isAllowed(url, this.userAgent);
+      const robots = await this.getRobots(url);
+      // robots-parser returns undefined when it has no opinion — that is "allowed".
+      // The legacy token is consulted as a source of disallow only: unifying on
+      // CrawlForge would otherwise silently un-block every site owner who had
+      // already written `User-agent: CrawlForge-Bot`, discarding a decision they
+      // made about us (G7). Where a file names neither token both fall through to
+      // the same `*` group, so this is a no-op.
+      const allowedFor = (ua) => robots.isAllowed(url, ua) !== false;
+      return allowedFor(this.userAgent) && allowedFor(LEGACY_PRODUCT_TOKEN);
     } catch (error) {
-      // If we can't fetch robots.txt, assume we can crawl
+      // A robots.txt we cannot read is not a disallow. Standard practice, and
+      // the alternative (fail closed on a network blip) blocks legitimate work.
       console.warn(`Failed to check robots.txt for ${url}:`, error.message);
       return true;
     }
   }
 
   async fetchRobotsTxt(robotsUrl) {
+    this.fetchCount++;
+    const controller = new AbortController();
+    // The timeout must stay armed for the body read, not just until headers
+    // arrive: a host that sends headers then trickles robots.txt forever would
+    // otherwise pin every tool behind the gate. Now that the gate runs before
+    // every fetching tool rather than only crawl_deep, one slow host would
+    // hang all of them. clearTimeout moves to the finally accordingly.
+    const timeoutId = setTimeout(() => controller.abort(), 5000);
+
     try {
-      const controller = new AbortController();
-      const timeoutId = setTimeout(() => controller.abort(), 5000);
-      
       const response = await safeFetch(robotsUrl, {
         signal: controller.signal,
-        headers: {
-          'User-Agent': this.userAgent
-        }
+        headers: identityHeaders({ userAgent: this.userAgent })
       });
-      
-      clearTimeout(timeoutId);
-      
+
       if (!response.ok) {
         return ''; // Empty robots.txt means everything is allowed
       }
-      
+
       return await response.text();
     } catch (error) {
       return ''; // If we can't fetch, assume no restrictions
+    } finally {
+      clearTimeout(timeoutId);
     }
   }
 
+  /** Crawl-delay in seconds from an already-cached robots.txt (0 if unknown). */
   getCrawlDelay(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      const robots = this.robotsCache.get(robotsUrl);
-      
-      if (robots) {
-        return robots.getCrawlDelay(this.userAgent) || 0;
-      }
-      
+      const cached = this.robotsCache.get(RobotsChecker.robotsUrlFor(url));
+      return cached ? cached.robots.getCrawlDelay(this.userAgent) || 0 : 0;
+    } catch {
       return 0;
+    }
+  }
+
+  /** Crawl-delay in seconds, fetching robots.txt if it is not cached yet. */
+  async fetchCrawlDelay(url) {
+    try {
+      const robots = await this.getRobots(url);
+      return robots.getCrawlDelay(this.userAgent) || 0;
     } catch {
       return 0;
     }
@@ -70,15 +125,8 @@ export class RobotsChecker {
 
   getSitemaps(url) {
     try {
-      const urlObj = new URL(url);
-      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
-      const robots = this.robotsCache.get(robotsUrl);
-      
-      if (robots) {
-        return robots.getSitemaps() || [];
-      }
-      
-      return [];
+      const cached = this.robotsCache.get(RobotsChecker.robotsUrlFor(url));
+      return cached ? cached.robots.getSitemaps() || [] : [];
     } catch {
       return [];
     }
@@ -86,7 +134,6 @@ export class RobotsChecker {
 
   clearCache() {
     this.robotsCache.clear();
+    this.inflight.clear();
   }
 }
-
-export default RobotsChecker;

package/src/utils/robotsGate.js

@@ -0,0 +1,206 @@
+/**
+ * robotsGate — the one pre-fetch gate every fetching tool goes through.
+ *
+ * Before this module, `RobotsChecker` was instantiated in exactly one place
+ * (BFSCrawler), so only `crawl_deep` honoured robots.txt; `scrape`,
+ * `batch_scrape`, `scrape_template`, `track_changes`, `map_site` and every
+ * `extract_*` tool did no robots check at all. Ground rule G5 says every
+ * fetching tool respects robots.txt by default, so the check has to live at the
+ * fetch boundary rather than in one crawler.
+ *
+ * Order matters. The platform blocklist (G7) is consulted first and is not
+ * overridable by anything a caller can send; robots (G5) is next and *is*
+ * overridable, but only explicitly, with a warning and an audit row; the
+ * host's Crawl-delay (G6) then feeds the per-host throttle.
+ *
+ * Callers replace `await throttleHost(url)` with `await preflightFetch(url, …)`
+ * and spread the returned `headers` into the request.
+ */
+
+import { RobotsChecker } from './robotsChecker.js';
+import { assertHostAllowed } from './hostBlocklist.js';
+import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
+import { throttleHost } from './hostRateLimiter.js';
+import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
+import { signRequestHeaders } from './webBotAuth.js';
+import { markPreflightRefusal } from '../server/requestContext.js';
+import { config } from '../constants/config.js';
+
+export class RobotsDisallowedError extends Error {
+  constructor(url) {
+    super(
+      `robots.txt on ${new URL(url).host} disallows this path for CrawlForge. ` +
+      `Pass respect_robots: false to fetch it anyway — that override is recorded ` +
+      `against your API key and is your decision to make.`
+    );
+    this.name = 'RobotsDisallowedError';
+    this.code = 'ROBOTS_DISALLOWED';
+    this.url = url;
+  }
+}
+
+/**
+ * One checker per identity, so the robots cache is process-wide rather than
+ * per-tool — otherwise every tool would re-fetch the same robots.txt.
+ * @type {Map<string, RobotsChecker>}
+ */
+const checkers = new Map();
+
+function checkerFor(userAgent) {
+  let checker = checkers.get(userAgent);
+  if (!checker) {
+    checker = new RobotsChecker(userAgent);
+    checkers.set(userAgent, checker);
+  }
+  return checker;
+}
+
+/**
+ * Decide whether a URL may be fetched. Pure decision — does no throttling and
+ * sends no request other than the (cached) robots.txt lookup.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ * @param {boolean} [options.respectRobots] per-request override; defaults to
+ *   `config.crawling.respectRobots`. `false` is honoured, warned about, audited.
+ * @param {string}  [options.userAgent] per-request identity override
+ * @param {string}  [options.tool] tool name, for the audit row
+ * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @returns {Promise<{ allowed: boolean, userAgent: string, crawlDelayMs: number,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError} for a permanently blocked host
+ */
+export async function robotsPreflight(url, options = {}) {
+  // G7 — first, and not overridable. Stamp before rethrowing so a blocked host
+  // costs the caller nothing: we refused, we fetched nothing.
+  try {
+    assertHostAllowed(url);
+  } catch (error) {
+    if (error?.code === 'HOST_BLOCKED') markPreflightRefusal('HOST_BLOCKED');
+    throw error;
+  }
+
+  const userAgent = resolveUserAgent(options.userAgent);
+  const warnings = [];
+
+  const explicitOverride = options.respectRobots === false;
+  const respect = options.respectRobots === undefined
+    ? config.crawling.respectRobots
+    : options.respectRobots !== false;
+
+  const checker = checkerFor(userAgent);
+  let allowed = true;
+  let crawlDelayMs = 0;
+
+  try {
+    allowed = await checker.canFetch(url);
+    crawlDelayMs = (await checker.fetchCrawlDelay(url)) * 1000;
+  } catch {
+    // Unreadable robots.txt is not a disallow (see RobotsChecker.canFetch).
+    allowed = true;
+  }
+
+  if (explicitOverride) {
+    warnings.push(
+      allowed
+        ? 'respect_robots was disabled for this request. robots.txt did not disallow this URL, so the override changed nothing. The request is recorded against your API key.'
+        : `respect_robots was disabled for this request and robots.txt on ${new URL(url).host} disallows this path. Fetching anyway is your decision and is recorded against your API key.`
+    );
+    recordComplianceEvent({
+      event: 'robots_override',
+      url,
+      tool: options.tool || null,
+      apiKeyId: apiKeyId(options.apiKey),
+      userAgent,
+      robotsAllowed: allowed
+    });
+  }
+
+  return {
+    allowed: allowed || !respect,
+    userAgent,
+    crawlDelayMs,
+    warnings,
+    overridden: explicitOverride && !allowed
+  };
+}
+
+/**
+ * The call-site helper: run the gate, honour Crawl-delay and any recorded
+ * `Retry-After`, and hand back the identity headers to send.
+ *
+ * @param {string} url
+ * @param {object} [options] see {@link robotsPreflight}
+ * @returns {Promise<{ headers: Record<string,string>, userAgent: string,
+ *   warnings: string[], overridden: boolean }>}
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function preflightFetch(url, options = {}) {
+  const decision = await robotsPreflight(url, options);
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url);
+  }
+
+  await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
+
+  // Web Bot Auth: when a signing key is configured, every request also carries
+  // a signature a site owner can verify against our published key. No key
+  // configured means no headers and no behaviour change. Requests with a
+  // caller-supplied userAgent override are still signed — the signature covers
+  // @authority, not the UA, and it identifies the operator (us), not the
+  // identity the caller asked us to present.
+  const signature = signRequestHeaders(url) || {};
+
+  return {
+    headers: { ...identityHeaders({ userAgent: decision.userAgent }), ...signature },
+    userAgent: decision.userAgent,
+    warnings: decision.warnings,
+    overridden: decision.overridden
+  };
+}
+
+/**
+ * The gate for browser paths. Same decision as {@link preflightFetch}, minus
+ * the identity and signature headers — those belong on an HTTP fetch, not on a
+ * browser context that presents its own identity.
+ *
+ * Deliberately takes no `userAgent`: robots.txt is matched against our
+ * canonical product token even when the browser presents another UA. Matching
+ * on the presented UA would let browser traffic walk past the rules our own
+ * token is bound by, which is the G5 hole this gate exists to close.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ * @param {boolean} [options.respectRobots] per-request override
+ * @param {string}  [options.tool] tool name, for the audit row
+ * @param {string}  [options.apiKey] hashed into the audit row, never stored raw
+ * @returns {Promise<string[]>} warnings to surface on the response
+ * @throws {BlockedHostError|RobotsDisallowedError}
+ */
+export async function browserPreflight(url, options = {}) {
+  const decision = await robotsPreflight(url, {
+    respectRobots: options.respectRobots,
+    tool: options.tool,
+    apiKey: options.apiKey
+  });
+  if (!decision.allowed) {
+    markPreflightRefusal('ROBOTS_DISALLOWED');
+    throw new RobotsDisallowedError(url);
+  }
+
+  await throttleHost(url, { crawlDelayMs: decision.crawlDelayMs });
+  return decision.warnings;
+}
+
+/** Test/diagnostic hook: drop every cached robots.txt. */
+export function _resetRobotsGate() {
+  checkers.clear();
+}
+
+/** Test/diagnostic hook: total robots.txt requests made across all identities. */
+export function _robotsFetchCount() {
+  let total = 0;
+  for (const checker of checkers.values()) total += checker.fetchCount;
+  return total;
+}

package/src/utils/sitemapParser.js

@@ -4,13 +4,16 @@ import { promisify } from 'util';
 import { CacheManager } from '../core/cache/CacheManager.js';
 import { normalizeUrl } from './urlNormalizer.js';
 import { safeFetch } from './ssrfGuard.js';
+import { CRAWLFORGE_USER_AGENT } from './fetchIdentity.js';
+import { preflightFetch } from './robotsGate.js';
+import { noteRetryAfter } from './hostRateLimiter.js';
 
 const gunzip = promisify(zlib.gunzip);
 
 export class SitemapParser {
   constructor(options = {}) {
     const {
-      userAgent = 'CrawlForge/1.0',
+      userAgent = CRAWLFORGE_USER_AGENT,
       timeout = 10000,
       maxRecursionDepth = 3,
       maxUrlsPerSitemap = 50000,
@@ -50,13 +53,17 @@ export class SitemapParser {
    * Parse a sitemap from a URL with full feature support
    * @param {string} url - Sitemap URL
    * @param {Object} options - Parsing options
+   * @param {boolean} [options.respectRobots] - Per-request robots override.
+   *   Per call, not per instance: MapSiteTool builds one SitemapParser and
+   *   reuses it, so a flag stored on the instance would leak between requests.
    * @returns {Promise<Object>} Parsed sitemap data
    */
   async parseSitemap(url, options = {}) {
     const {
       includeMetadata = true,
       followIndexes = true,
-      maxDepth = this.maxRecursionDepth
+      maxDepth = this.maxRecursionDepth,
+      respectRobots
     } = options;
 
     // Reset stats for new parsing session
@@ -72,7 +79,8 @@ export class SitemapParser {
     try {
       const result = await this._parseSitemapRecursive(url, 0, maxDepth, {
         includeMetadata,
-        followIndexes
+        followIndexes,
+        respectRobots
       });
 
       return {
@@ -99,11 +107,12 @@ export class SitemapParser {
   /**
    * Parse sitemap index files and return all contained sitemaps
    * @param {string} indexUrl - Sitemap index URL
+   * @param {boolean} [respectRobots] - Per-request robots override
    * @returns {Promise<Array>} Array of sitemap URLs with metadata
    */
-  async parseSitemapIndex(indexUrl) {
+  async parseSitemapIndex(indexUrl, respectRobots) {
     try {
-      const content = await this._fetchSitemapContent(indexUrl);
+      const content = await this._fetchSitemapContent(indexUrl, respectRobots);
       if (!content) return [];
 
       const $ = load(content, { xmlMode: true });
@@ -260,9 +269,10 @@ export class SitemapParser {
    * Discover sitemap URLs from various sources
    * @param {string} baseUrl - Base URL of the website
    * @param {Object} sources - Sources to check
+   * @param {boolean} [respectRobots] - Per-request robots override
    * @returns {Promise<Array>} Array of discovered sitemap URLs
    */
-  async discoverSitemaps(baseUrl, sources = {}) {
+  async discoverSitemaps(baseUrl, sources = {}, respectRobots) {
     const {
       checkRobotsTxt = true,
       checkCommonPaths = true,
@@ -277,7 +287,7 @@ export class SitemapParser {
     if (checkRobotsTxt) {
       try {
         const robotsUrl = `${baseOrigin}/robots.txt`;
-        const robotsContent = await this._fetchWithTimeout(robotsUrl);
+        const robotsContent = await this._fetchWithTimeout(robotsUrl, respectRobots);
         if (robotsContent) {
           const sitemapMatches = robotsContent.match(/^Sitemap:\s*(.+)$/gmi);
           if (sitemapMatches) {
@@ -308,7 +318,7 @@ export class SitemapParser {
       for (const path of commonPaths) {
         const sitemapUrl = `${baseOrigin}${path}`;
         try {
-          const response = await this._fetchWithTimeoutResponse(sitemapUrl);
+          const response = await this._fetchWithTimeoutResponse(sitemapUrl, respectRobots);
           if (response && response.ok) {
             discovered.add(sitemapUrl);
           }
@@ -344,7 +354,7 @@ export class SitemapParser {
     }
 
     try {
-      const content = await this._fetchSitemapContent(url);
+      const content = await this._fetchSitemapContent(url, options.respectRobots);
       if (!content) {
         throw new Error(`Failed to fetch sitemap content from ${url}`);
       }
@@ -385,9 +395,9 @@ export class SitemapParser {
    * Fetch and decompress sitemap content
    * @private
    */
-  async _fetchSitemapContent(url) {
+  async _fetchSitemapContent(url, respectRobots) {
     try {
-      const response = await this._fetchWithTimeoutResponse(url);
+      const response = await this._fetchWithTimeoutResponse(url, respectRobots);
       if (!response || !response.ok) {
         return null;
       }
@@ -621,8 +631,8 @@ export class SitemapParser {
    * Fetch with timeout
    * @private
    */
-  async _fetchWithTimeout(url) {
-    const response = await this._fetchWithTimeoutResponse(url);
+  async _fetchWithTimeout(url, respectRobots) {
+    const response = await this._fetchWithTimeoutResponse(url, respectRobots);
     return response ? await response.text() : null;
   }
 
@@ -630,7 +640,12 @@ export class SitemapParser {
    * Fetch with timeout returning response object
    * @private
    */
-  async _fetchWithTimeoutResponse(url) {
+  async _fetchWithTimeoutResponse(url, respectRobots) {
+    const gate = await preflightFetch(url, {
+      respectRobots,
+      userAgent: this.userAgent,
+      tool: 'sitemap'
+    });
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
@@ -638,12 +653,15 @@ export class SitemapParser {
       const response = await safeFetch(url, {
         signal: controller.signal,
         headers: {
-          'User-Agent': this.userAgent,
+          ...gate.headers,
           'Accept': 'application/xml,text/xml,text/plain,*/*',
           'Accept-Encoding': 'gzip, deflate'
         }
       });
       clearTimeout(timeoutId);
+      if (response.status === 429 || response.status === 503) {
+        noteRetryAfter(url, response.headers.get('retry-after'));
+      }
       return response;
     } catch (error) {
       clearTimeout(timeoutId);

package/src/utils/ssrfProtection.js

@@ -6,6 +6,7 @@
 import { promisify } from 'util';
 import dns from 'dns';
 import net from 'net';
+import { identityHeaders } from './fetchIdentity.js';
 
 const dnsLookup = promisify(dns.lookup);
 
@@ -560,7 +561,7 @@ export class SSRFProtection {
         timeout: Math.min(fetchOptions.timeout || 30000, this.config.maxTimeout),
         redirect: 'manual', // Handle redirects manually
         headers: {
-          'User-Agent': 'CrawlForge/3.0 (Security Enhanced)',
+          ...identityHeaders({ role: 'health-check' }),
           ...fetchOptions.headers
         }
       };