crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/tools/basic/_fetch.js

@@ -13,6 +13,35 @@ const _require = createRequire(import.meta.url);
 const _pkg = _require('../../../package.json');
 const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
 
+/**
+ * Determine the charset to decode a response body with: Content-Type header
+ * first, then a <meta charset> sniff of the first bytes, defaulting to utf-8.
+ * @param {Response} response
+ * @param {Uint8Array} bytes
+ * @returns {string}
+ */
+function detectCharset(response, bytes) {
+  const contentType = response.headers?.get?.('content-type') || '';
+  const headerMatch = /charset=["']?([\w-]+)/i.exec(contentType);
+  if (headerMatch) {
+    return headerMatch[1].trim().toLowerCase();
+  }
+
+  // <meta charset> tags must appear within the first 1024 bytes per the
+  // HTML5 spec's prescan algorithm; ASCII-range bytes decode identically
+  // under latin1 regardless of the document's real encoding.
+  const sniffLength = Math.min(bytes.byteLength, 1024);
+  const sniffText = new TextDecoder('latin1').decode(bytes.subarray(0, sniffLength));
+  const metaMatch =
+    /<meta[^>]+charset=["']?([\w-]+)/i.exec(sniffText) ||
+    /<meta[^>]+http-equiv=["']?content-type["']?[^>]*content=["'][^"']*charset=([\w-]+)/i.exec(sniffText);
+  if (metaMatch) {
+    return metaMatch[1].trim().toLowerCase();
+  }
+
+  return 'utf-8';
+}
+
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
  *
@@ -39,83 +68,109 @@ export async function fetchWithTimeout(url, options = {}) {
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
 
-  let response;
+  // The timeout must stay armed for the entire body read, not just until
+  // headers arrive — a stalled/trickling body (slowloris, hung proxy) would
+  // otherwise hang the awaiting reader.read() forever. clearTimeout runs in
+  // this finally, after the body has been fully consumed (or an error has
+  // already ended the request), and any abort() during that window rejects
+  // the in-flight reader.read() with an AbortError, which we map below.
   try {
-    response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        'User-Agent': CRAWLFORGE_UA,
-        ...headers
-      },
-      ...guard
-    });
-    clearTimeout(timeoutId);
-  } catch (error) {
-    clearTimeout(timeoutId);
-    if (isSsrfError(error)) {
-      throw new Error(error.cause?.message || error.message);
+    let response;
+    try {
+      response = await fetch(url, {
+        signal: controller.signal,
+        headers: {
+          'User-Agent': CRAWLFORGE_UA,
+          ...headers
+        },
+        ...guard
+      });
+    } catch (error) {
+      if (isSsrfError(error)) {
+        throw new Error(error.cause?.message || error.message);
+      }
+      if (error.name === 'AbortError') {
+        throw new Error(`Request timeout after ${timeout}ms`);
+      }
+      throw error;
+    }
+
+    // --- Body-size cap ---
+
+    // Early rejection via Content-Length (servers may omit or lie — guard below
+    // handles that case). Optional-chained so non-standard responses (e.g. test
+    // mocks) without a Headers object don't throw.
+    const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
+    if (contentLengthHeader !== null) {
+      const declared = parseInt(contentLengthHeader, 10);
+      if (!isNaN(declared) && declared > maxBodySize) {
+        throw new Error(
+          `Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`
+        );
+      }
     }
-    if (error.name === 'AbortError') {
-      throw new Error(`Request timeout after ${timeout}ms`);
+
+    // Only the streaming byte-count guard requires a readable body. Responses
+    // without a ReadableStream body (already-buffered responses, test mocks)
+    // are returned unchanged so callers' native .text()/.json() still work.
+    if (!response.body || typeof response.body.getReader !== 'function') {
+      return response;
     }
-    throw error;
-  }
 
-  // --- Body-size cap ---
-
-  // Early rejection via Content-Length (servers may omit or lie — guard below
-  // handles that case). Optional-chained so non-standard responses (e.g. test
-  // mocks) without a Headers object don't throw.
-  const contentLengthHeader = response.headers?.get?.('content-length') ?? null;
-  if (contentLengthHeader !== null) {
-    const declared = parseInt(contentLengthHeader, 10);
-    if (!isNaN(declared) && declared > maxBodySize) {
-      throw new Error(
-        `Response body too large: Content-Length ${declared} exceeds limit of ${maxBodySize} bytes`
-      );
+    // Stream the body and abort if accumulated bytes exceed the cap.
+    const reader = response.body.getReader();
+    const chunks = [];
+    let totalBytes = 0;
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        totalBytes += value.byteLength;
+        if (totalBytes > maxBodySize) {
+          reader.cancel();
+          throw new Error(
+            `Response body too large: exceeded limit of ${maxBodySize} bytes`
+          );
+        }
+        chunks.push(value);
+      }
+    } catch (error) {
+      if (error.name === 'AbortError') {
+        throw new Error(`Request timeout after ${timeout}ms`);
+      }
+      throw error;
     }
-  }
 
-  // Only the streaming byte-count guard requires a readable body. Responses
-  // without a ReadableStream body (already-buffered responses, test mocks)
-  // are returned unchanged so callers' native .text()/.json() still work.
-  if (!response.body || typeof response.body.getReader !== 'function') {
-    return response;
-  }
+    // Reassemble the raw bytes in a single pass (totalBytes is already known,
+    // so this is one allocation + one copy per chunk, not the O(n^2) cost of
+    // reallocating/copying the whole buffer on every chunk), then decode using
+    // the response's actual charset (Content-Type header, falling back to a
+    // <meta charset> sniff) instead of always assuming UTF-8.
+    const mergedBytes = new Uint8Array(totalBytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+      mergedBytes.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
 
-  // Stream the body and abort if accumulated bytes exceed the cap.
-  const reader = response.body.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    totalBytes += value.byteLength;
-    if (totalBytes > maxBodySize) {
-      reader.cancel();
-      throw new Error(
-        `Response body too large: exceeded limit of ${maxBodySize} bytes`
-      );
+    const charset = detectCharset(response, mergedBytes);
+    let bodyText;
+    try {
+      bodyText = new TextDecoder(charset).decode(mergedBytes);
+    } catch {
+      // Unrecognized charset label — fall back to UTF-8 rather than throwing.
+      bodyText = new TextDecoder().decode(mergedBytes);
     }
-    chunks.push(value);
-  }
 
-  // Reassemble and expose as a response-like object that callers can use.
-  const bodyText = new TextDecoder().decode(
-    chunks.reduce((acc, chunk) => {
-      const merged = new Uint8Array(acc.byteLength + chunk.byteLength);
-      merged.set(acc, 0);
-      merged.set(chunk, acc.byteLength);
-      return merged;
-    }, new Uint8Array(0))
-  );
-
-  // Attach the pre-read text so callers can call .text() on the result.
-  // We wrap it in a minimal compatible object.
-  return Object.assign(response, {
-    text: () => Promise.resolve(bodyText),
-    json: () => Promise.resolve(JSON.parse(bodyText)),
-    _body: bodyText
-  });
+    // Attach the pre-read text so callers can call .text() on the result.
+    // We wrap it in a minimal compatible object.
+    return Object.assign(response, {
+      text: () => Promise.resolve(bodyText),
+      json: () => Promise.resolve(JSON.parse(bodyText)),
+      _body: bodyText
+    });
+  } finally {
+    clearTimeout(timeoutId);
+  }
 }

package/src/tools/basic/extractLinks.js

@@ -19,8 +19,18 @@ export async function extractLinksHandler({ url, filter_external, base_url }) {
     const html = await response.text();
     const $ = load(html);
 
-    const baseUrl = base_url || new URL(url).origin;
-    const pageUrl = new URL(url);
+    const finalUrl = response.url || url;
+    const pageUrl = new URL(finalUrl);
+
+    // <base href>, if present, overrides the page URL as the resolution base
+    // for relative links (but an explicit base_url override wins over both).
+    let docBase = finalUrl;
+    const baseHref = $('base[href]').first().attr('href');
+    if (baseHref) {
+      try { docBase = new URL(baseHref, finalUrl).toString(); } catch { /* ignore invalid <base href> */ }
+    }
+
+    const baseUrl = base_url || docBase;
     const links = [];
 
     $('a[href]').each((_, element) => {
@@ -29,17 +39,9 @@ export async function extractLinksHandler({ url, filter_external, base_url }) {
 
       if (!href) return;
 
-      let absoluteUrl;
-      let isExternal = false;
-
       try {
-        if (href.startsWith('http://') || href.startsWith('https://')) {
-          absoluteUrl = href;
-          isExternal = new URL(href).origin !== pageUrl.origin;
-        } else {
-          absoluteUrl = new URL(href, baseUrl).toString();
-          isExternal = false;
-        }
+        const absoluteUrl = new URL(href, baseUrl).toString();
+        const isExternal = new URL(absoluteUrl).origin !== pageUrl.origin;
 
         if (filter_external && !isExternal) return;
 

package/src/tools/basic/scrapeStructured.js

@@ -8,18 +8,32 @@
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
 
+// Matches a trailing "@attr" suffix (e.g. "@href", "@data-id") — the attribute
+// name must look like a real attribute, not the "@" of a CSS attribute-value
+// selector such as a[href*="@"].
+const ATTR_SUFFIX_RE = /@[A-Za-z_:][\w:.-]*$/;
+
 /**
  * Parse a selector string that may include an attribute suffix: "css@attr"
  * e.g. "a.link@href" -> { selector: "a.link", attribute: "href" }
  *      "img@src"      -> { selector: "img",    attribute: "src" }
  *      "h1"           -> { selector: "h1",      attribute: null }
+ *      'a[href*="@"]'  -> { selector: 'a[href*="@"]', attribute: null }
  * @param {string} raw
  * @returns {{ selector: string, attribute: string|null }}
  */
 function parseSelectorSpec(raw) {
-  const atIdx = raw.lastIndexOf('@');
-  if (atIdx > 0) {
-    return { selector: raw.slice(0, atIdx), attribute: raw.slice(atIdx + 1) };
+  const match = ATTR_SUFFIX_RE.exec(raw);
+  // Only treat it as an attribute suffix if it isn't inside brackets/quotes,
+  // i.e. the selector portion before it has balanced [ ] and quotes.
+  if (match && match.index > 0) {
+    const selectorPart = raw.slice(0, match.index);
+    const openBrackets = (selectorPart.match(/\[/g) || []).length;
+    const closeBrackets = (selectorPart.match(/\]/g) || []).length;
+    const quoteCount = (selectorPart.match(/["']/g) || []).length;
+    if (openBrackets === closeBrackets && quoteCount % 2 === 0) {
+      return { selector: selectorPart, attribute: raw.slice(match.index + 1) };
+    }
   }
   return { selector: raw, attribute: null };
 }
@@ -64,7 +78,10 @@ export async function scrapeStructuredHandler({ url, selectors, max_results }) {
           if (elements.length === 1) {
             results[fieldName] = extract(elements.get(0));
           } else {
-            results[fieldName] = elements.map((_, el) => extract(el)).get();
+            // cheerio's .map().get() drops null/undefined results, which would
+            // desynchronize this array from elements_found and from parallel
+            // fields. Build it from toArray() instead so length always matches.
+            results[fieldName] = elements.toArray().map(extract);
           }
         }
       } catch (selectorError) {

package/src/tools/crawl/crawlDeep.js

@@ -82,7 +82,12 @@ export class CrawlDeepTool {
       userAgent = 'MCP-WebScraper/1.0',
       timeout = 30000,
       cacheEnabled = true,
-      cacheTTL = 3600000
+      cacheTTL = 3600000,
+      maxDepth = 5,
+      maxPages = 100,
+      respectRobots = true,
+      followExternal = false,
+      concurrency = 10
     } = options;
 
     this.userAgent = userAgent;
@@ -91,6 +96,13 @@ export class CrawlDeepTool {
     this.cache = cacheEnabled ? new CacheManager({ ttl: cacheTTL }) : null;
     // D1.4: Elicitation helper
     this._elicitation = new ElicitationHelper({});
+    // Server-configured ceilings/defaults (MAX_CRAWL_DEPTH, MAX_PAGES_PER_CRAWL,
+    // RESPECT_ROBOTS_TXT, FOLLOW_EXTERNAL_LINKS, QUEUE_CONCURRENCY)
+    this.configMaxDepth = maxDepth;
+    this.configMaxPages = maxPages;
+    this.configRespectRobots = respectRobots;
+    this.configFollowExternal = followExternal;
+    this.configConcurrency = concurrency;
   }
 
   /** D1.4: Wire MCP server for elicitation. Call from server.js after instantiation. */
@@ -102,21 +114,38 @@ export class CrawlDeepTool {
     try {
       const validated = CrawlDeepSchema.parse(params);
 
+      // Apply server-configured ceilings/defaults: max_depth/max_pages are
+      // clamped to the operator's configured maxima; respect_robots/
+      // follow_external/concurrency fall back to the configured value only
+      // when the caller left them unspecified (raw params, since zod has
+      // already filled in its own schema default by this point).
+      const effectiveMaxDepth = Math.min(validated.max_depth, this.configMaxDepth);
+      const effectiveMaxPages = Math.min(validated.max_pages, this.configMaxPages);
+      const effectiveRespectRobots = params.respect_robots === undefined ? this.configRespectRobots : validated.respect_robots;
+      const effectiveFollowExternal = params.follow_external === undefined ? this.configFollowExternal : validated.follow_external;
+      const effectiveConcurrency = params.concurrency === undefined ? this.configConcurrency : validated.concurrency;
+
       // Cache dedup: skip re-crawling the same root URL within the TTL window
       if (this.cache) {
-        const cacheKey = this.cache.generateKey('crawl_deep', { url: validated.url, depth: validated.max_depth, pages: validated.max_pages });
+        const cacheKey = this._buildCacheKey(validated, {
+          maxDepth: effectiveMaxDepth,
+          maxPages: effectiveMaxPages,
+          respectRobots: effectiveRespectRobots,
+          followExternal: effectiveFollowExternal,
+          concurrency: effectiveConcurrency
+        });
         const cached = await this.cache.get(cacheKey);
         if (cached) return cached;
       }
 
       // D1.4: Elicitation — warn when max_pages is very high
-      if (validated.max_pages > 500) {
+      if (effectiveMaxPages > 500) {
         const proceed = await this._elicitation.confirm(
-          `crawl_deep will crawl up to ${validated.max_pages} pages from ${validated.url}. Large crawls consume many credits.`,
+          `crawl_deep will crawl up to ${effectiveMaxPages} pages from ${validated.url}. Large crawls consume many credits.`,
           {
             url: validated.url,
-            max_pages: validated.max_pages,
-            max_depth: validated.max_depth,
+            max_pages: effectiveMaxPages,
+            max_depth: effectiveMaxDepth,
           }
         );
         if (!proceed) {
@@ -142,8 +171,8 @@ export class CrawlDeepTool {
       } else if (validated.domain_filter) {
         // Create from inline configuration
         domainFilter = new DomainFilter({
-          allowSubdomains: !validated.follow_external,
-          defaultMaxDepth: validated.max_depth,
+          allowSubdomains: !effectiveFollowExternal,
+          defaultMaxDepth: effectiveMaxDepth,
           defaultRateLimit: 10
         });
         
@@ -187,60 +216,93 @@ export class CrawlDeepTool {
 
       // Create crawler instance
       const crawler = new BFSCrawler({
-        maxDepth: validated.max_depth,
-        maxPages: validated.max_pages,
-        followExternal: validated.follow_external,
-        respectRobots: validated.respect_robots,
+        maxDepth: effectiveMaxDepth,
+        maxPages: effectiveMaxPages,
+        followExternal: effectiveFollowExternal,
+        respectRobots: effectiveRespectRobots,
         userAgent: this.userAgent,
         timeout: this.timeout,
-        concurrency: validated.concurrency,
+        concurrency: effectiveConcurrency,
         domainFilter: domainFilter,
         enableLinkAnalysis: validated.enable_link_analysis,
         linkAnalyzerOptions: validated.link_analysis_options,
         sessionContext
       });
-      
+
       // Start crawling
-      const startTime = Date.now();
-      const results = await crawler.crawl(validated.url, {
-        includePatterns: validated.include_patterns,
-        excludePatterns: validated.exclude_patterns,
-        extractContent: validated.extract_content
-      });
-      const duration = Date.now() - startTime;
-      
-      // Process and format results
-      const response = {
-        url: validated.url,
-        crawl_depth: validated.max_depth,
-        pages_crawled: results.urls.length,
-        pages_found: results.results.length,
-        errors: results.errors.length,
-        duration_ms: duration,
-        pages_per_second: results.urls.length / (duration / 1000),
-        results: this.formatResults(results.results, validated.extract_content, validated.content_max_length),
-        errors: results.errors,
-        stats: results.stats,
-        site_structure: this.analyzeSiteStructure(results.urls),
-        domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
-        link_analysis: results.linkAnalysis,
-        session: sessionContext
-          ? { enabled: true, cookies_captured: sessionContext.cookieCount }
-          : { enabled: false }
-      };
-      
-      // Store in cache before returning
-      if (this.cache) {
-        const cacheKey = this.cache.generateKey('crawl_deep', { url: validated.url, depth: validated.max_depth, pages: validated.max_pages });
-        await this.cache.set(cacheKey, response);
-      }
+      try {
+        const startTime = Date.now();
+        const results = await crawler.crawl(validated.url, {
+          includePatterns: validated.include_patterns,
+          excludePatterns: validated.exclude_patterns,
+          extractContent: validated.extract_content
+        });
+        const duration = Date.now() - startTime;
+
+        // Process and format results
+        const response = {
+          url: validated.url,
+          crawl_depth: effectiveMaxDepth,
+          pages_crawled: results.urls.length,
+          pages_found: results.results.length,
+          error_count: results.errors.length,
+          duration_ms: duration,
+          pages_per_second: results.urls.length / (duration / 1000),
+          results: this.formatResults(results.results, validated.extract_content, validated.content_max_length),
+          errors: results.errors,
+          stats: results.stats,
+          site_structure: this.analyzeSiteStructure(results.urls),
+          domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
+          link_analysis: results.linkAnalysis,
+          session: sessionContext
+            ? { enabled: true, cookies_captured: sessionContext.cookieCount }
+            : { enabled: false }
+        };
+
+        // Store in cache before returning
+        if (this.cache) {
+          const cacheKey = this._buildCacheKey(validated, {
+            maxDepth: effectiveMaxDepth,
+            maxPages: effectiveMaxPages,
+            respectRobots: effectiveRespectRobots,
+            followExternal: effectiveFollowExternal,
+            concurrency: effectiveConcurrency
+          });
+          await this.cache.set(cacheKey, response);
+        }
 
-      return response;
+        return response;
+      } finally {
+        // Release the per-crawl CacheManager's timers so the crawler
+        // instance (and its cached page bodies) can be garbage collected.
+        crawler.destroy();
+      }
     } catch (error) {
       throw new Error(`Crawl failed: ${error.message}`);
     }
   }
 
+  // Cache key must reflect every validated/effective field that changes the
+  // response, otherwise a hit under one set of options silently returns a
+  // result produced under different extraction/filtering/session settings.
+  _buildCacheKey(validated, effective) {
+    return this.cache.generateKey('crawl_deep', {
+      url: validated.url,
+      depth: effective.maxDepth,
+      pages: effective.maxPages,
+      extractContent: validated.extract_content,
+      contentMaxLength: validated.content_max_length,
+      includePatterns: validated.include_patterns,
+      excludePatterns: validated.exclude_patterns,
+      followExternal: effective.followExternal,
+      respectRobots: effective.respectRobots,
+      concurrency: effective.concurrency,
+      domainFilter: validated.domain_filter ?? null,
+      importFilterConfig: validated.import_filter_config ?? null,
+      sessionEnabled: validated.session?.enabled ?? false
+    });
+  }
+
   formatResults(results, includeContent, contentMaxLength = 500) {
     return results.map(result => {
       const formatted = {

package/src/tools/crawl/mapSite.js

@@ -5,6 +5,7 @@ import { normalizeUrl, getBaseUrl } from '../../utils/urlNormalizer.js';
 import { CacheManager } from '../../core/cache/CacheManager.js';
 import { SitemapParser } from '../../utils/sitemapParser.js';
 import { ResultRanker } from '../search/ranking/ResultRanker.js';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
@@ -52,7 +53,7 @@ export class MapSiteTool {
 
       // Cache dedup: skip re-mapping the same site within the TTL window
       if (this.cache) {
-        const cacheKey = this.cache.generateKey('map_site', { url: validated.url, maxUrls: validated.max_urls });
+        const cacheKey = this._buildCacheKey(validated);
         const cached = await this.cache.get(cacheKey);
         if (cached) return cached;
       }
@@ -93,7 +94,7 @@ export class MapSiteTool {
 
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
-        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter);
+        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, validated.max_urls);
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
@@ -150,7 +151,7 @@ export class MapSiteTool {
 
       // Store in cache before returning
       if (this.cache) {
-        const cacheKey = this.cache.generateKey('map_site', { url: validated.url, maxUrls: validated.max_urls });
+        const cacheKey = this._buildCacheKey(validated);
         await this.cache.set(cacheKey, result);
       }
 
@@ -160,7 +161,22 @@ export class MapSiteTool {
     }
   }
 
-  async fetchSitemapUrls(baseUrl, domainFilter = null) {
+  // Cache key must reflect every validated field that changes the response,
+  // otherwise a hit under one set of options silently returns another's
+  // result (e.g. search ranking or domain filtering getting dropped/leaked).
+  _buildCacheKey(validated) {
+    return this.cache.generateKey('map_site', {
+      url: validated.url,
+      maxUrls: validated.max_urls,
+      search: validated.search ?? null,
+      domainFilter: validated.domain_filter ?? null,
+      importFilterConfig: validated.import_filter_config ?? null,
+      includeMetadata: validated.include_metadata,
+      groupByPath: validated.group_by_path
+    });
+  }
+
+  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity) {
     // Discover sitemaps via robots.txt and common paths, then parse with full
     // SitemapParser support (sitemap-index recursion, gzip, CDATA/entities).
     const discovered = await this.sitemapParser.discoverSitemaps(baseUrl, {
@@ -182,9 +198,12 @@ export class MapSiteTool {
             if (!domainFilter || domainFilter.isAllowed(url).allowed) {
               urls.add(url);
             }
+            if (urls.size >= maxUrls) break;
           }
         }
-        if (urls.size > 0) break;
+        // Keep accumulating across every discovered sitemap (a site can
+        // declare several) instead of stopping at the first productive one.
+        if (urls.size >= maxUrls) break;
       } catch {
         // Continue to next discovered sitemap
       }
@@ -261,7 +280,7 @@ export class MapSiteTool {
     const timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
     try {
-      const response = await fetch(url, {
+      const response = await safeFetch(url, {
         signal: controller.signal,
         headers: {
           'User-Agent': this.userAgent