crawlforge-mcp-server 5.2.8 → 5.3.0

package/src/core/crawlers/BFSCrawler.js

@@ -2,12 +2,14 @@ import { load } from 'cheerio';
 import { QueueManager } from '../queue/QueueManager.js';
 import { CacheManager } from '../cache/CacheManager.js';
 import { RateLimiter } from '../../utils/rateLimiter.js';
-import { RobotsChecker } from '../../utils/robotsChecker.js';
 import { DomainFilter } from '../../utils/domainFilter.js';
 import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { robotsPreflight } from '../../utils/robotsGate.js';
+import { throttleHost } from '../../utils/hostRateLimiter.js';
+import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -18,7 +20,7 @@ export class BFSCrawler {
       maxPages = 100,
       followExternal = false,
       respectRobots = true,
-      userAgent = 'MCP-WebScraper/1.0',
+      userAgent = CRAWLFORGE_USER_AGENT,
       timeout = 30000,
       concurrency = 10,
       domainFilter = null,
@@ -59,7 +61,6 @@ export class BFSCrawler {
     // effectiveRateLimit hasn't changed, rather than recreating it on every URL.
     this.rateLimiter = new RateLimiter({ requestsPerSecond: 10 });
     this._domainRateLimiters = new Map();
-    this.robotsChecker = respectRobots ? new RobotsChecker(userAgent) : null;
     
     // Initialize domain filter (create new if not provided)
     this.domainFilter = domainFilter || new DomainFilter({
@@ -109,14 +110,18 @@ export class BFSCrawler {
     const normalizedStart = normalizeUrl(startUrl);
     this.baseUrl = new URL(normalizedStart);
     
-    // Check if start URL is allowed
-    const startUrlDecision = this.domainFilter.isAllowed(normalizedStart);
+    // Check if start URL is allowed. isSeed exempts it from the include-pattern gate:
+    // include patterns scope where the crawl may go next, not whether the URL the caller
+    // explicitly asked for may be fetched. Blacklist and exclude patterns still apply.
+    // The un-normalized startUrl is passed so a pattern written with a trailing slash
+    // ('/docs/') is tested against the form the caller wrote.
+    const startUrlDecision = this.domainFilter.isAllowed(startUrl, { isSeed: true });
     if (!startUrlDecision.allowed) {
       throw new Error(`Start URL blocked by domain filter: ${startUrlDecision.reason}`);
     }
     
     // Initialize queue with starting URL
-    await this.queue.add(() => this.processUrl(normalizedStart, 0));
+    await this.queue.add(() => this.processUrl(startUrl, 0));
 
     // Wait for crawling to complete
     await this.queue.onIdle();
@@ -148,8 +153,11 @@ export class BFSCrawler {
       return;
     }
 
+    // Only the seed is queued at depth 0; children are always depth >= 1.
+    const isSeed = depth === 0;
+
     // Check domain filter (replaces old pattern checking)
-    const filterDecision = this.domainFilter.isAllowed(normalizedUrl);
+    const filterDecision = this.domainFilter.isAllowed(url, { isSeed });
     this.filterDecisions.push({
       url: normalizedUrl,
       decision: filterDecision,
@@ -162,18 +170,22 @@ export class BFSCrawler {
     }
 
     // Backward compatibility: also check legacy patterns
-    if (!this.shouldCrawlUrl(normalizedUrl)) {
+    if (!this.shouldCrawlUrl(normalizedUrl, url, isSeed)) {
       logger.debug(`Legacy pattern blocks: ${normalizedUrl}`);
       return;
     }
 
-    // Check robots.txt
-    if (this.respectRobots && this.robotsChecker) {
-      const canFetch = await this.robotsChecker.canFetch(normalizedUrl);
-      if (!canFetch) {
-        logger.debug(`Robots.txt blocks: ${normalizedUrl}`);
-        return;
-      }
+    // Check robots.txt through the shared gate, so this crawl reuses the same
+    // cached robots.txt every other tool fetched (and honours the platform
+    // blocklist, which no per-request flag can switch off).
+    const gate = await robotsPreflight(normalizedUrl, {
+      respectRobots: this.respectRobots,
+      userAgent: this.userAgent,
+      tool: 'crawl_deep'
+    });
+    if (!gate.allowed) {
+      logger.debug(`Robots.txt blocks: ${normalizedUrl}`);
+      return;
     }
 
     // Mark as visited. Re-check the cap and dedupe first: the checks at the
@@ -209,6 +221,12 @@ export class BFSCrawler {
 
         await this._domainRateLimiters.get(domain).checkLimit(normalizedUrl);
 
+        // The host's own Crawl-delay, on top of our per-domain limit. Only
+        // when it asked for one — otherwise the domain limiter above stands.
+        if (gate.crawlDelayMs > 0) {
+          await throttleHost(normalizedUrl, { crawlDelayMs: gate.crawlDelayMs });
+        }
+
         // Fetch the page
         pageData = await this.fetchPage(normalizedUrl);
         
@@ -292,7 +310,7 @@ export class BFSCrawler {
       const domainRules = this.domainFilter.getDomainRules(urlObj.hostname);
       
       const defaultHeaders = {
-        'User-Agent': this.userAgent,
+        ...identityHeaders({ userAgent: this.userAgent }),
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'en-US,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
@@ -409,16 +427,21 @@ export class BFSCrawler {
     }
   }
 
-  shouldCrawlUrl(url) {
-    // Check include patterns
-    if (this.includePatterns.length > 0) {
-      const matches = this.includePatterns.some(pattern => pattern.test(url));
+  shouldCrawlUrl(url, rawUrl = url, isSeed = false) {
+    // Check include patterns. The seed is exempt for the same reason as in crawl():
+    // it is the URL the caller named, not a link the crawl chose to follow.
+    if (!isSeed && this.includePatterns.length > 0) {
+      const matches = this.includePatterns.some(
+        pattern => pattern.test(url) || pattern.test(rawUrl)
+      );
       if (!matches) return false;
     }
 
     // Check exclude patterns
     if (this.excludePatterns.length > 0) {
-      const excluded = this.excludePatterns.some(pattern => pattern.test(url));
+      const excluded = this.excludePatterns.some(
+        pattern => pattern.test(url) || pattern.test(rawUrl)
+      );
       if (excluded) return false;
     }