crawlforge-mcp-server 6.3.1 → 6.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.3.1",
+  "version": "6.4.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.3.1",
+  version: "6.4.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -876,7 +876,8 @@ registerToolIfEnabled("get_batch_results", {
   inputSchema: {
     batchId: z.string().describe("The batch ID returned by batch_scrape"),
     page: z.number().min(1).default(1).describe("Page number (1-based)"),
-    pageSize: z.number().min(1).max(100).default(25).describe("Number of results per page")
+    pageSize: z.number().min(1).max(100).default(25).describe("Number of results per page"),
+    ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("get_batch_results", async ({ batchId, page = 1, pageSize = 25 }) => {
   try {

package/src/core/AgentOrchestrator.js

@@ -333,6 +333,23 @@ export class AgentOrchestrator {
         quotedTerms.every(t => q.toLowerCase().includes(t.toLowerCase())) ? q : `${q} ${quotedTerms.join(' ')}`.trim()
       );
     }
+    // A current-state plan is told to make its first query the bare entity
+    // name, and a small model often stops there. The bare query surfaces the
+    // live front page, which rarely states the fact asked for: "what does
+    // Southwest charge for a first checked bag" fetched southwest.com's home,
+    // booking and careers pages and answered that the fee is not stated
+    // (R20, 2026-09-07). Keep the entity query first — the domain vote and
+    // live-root promotion depend on it — and add the task's own words as a
+    // second query so the page that states the fact is in the queue too.
+    if (currentState && searchQueries.length === 1) {
+      try {
+        const { compactSearchTopic, clampSearchQuery } = await import('./ResearchOrchestrator.js');
+        const factQuery = clampSearchQuery(compactSearchTopic(prompt));
+        if (factQuery && factQuery.toLowerCase() !== searchQueries[0].toLowerCase()) {
+          searchQueries.push(factQuery);
+        }
+      } catch { /* the entity query alone is the pre-R20 behaviour */ }
+    }
 
     // ── GATHER (search) ───────────────────────────────────────────────────────
     const urlQueue = [...seedUrls]; // start with any user-provided seeds
@@ -389,7 +406,11 @@ export class AgentOrchestrator {
     // and never enters evidence.
     if (currentState && searchResults.length > 0) {
       const originCounts = new Map();
+      // Only the bare entity query votes: the fact query added above returns
+      // guides and news sites, which must not outvote the official domain.
+      const entityQuery = searchQueries[0];
       for (const s of searchResults) {
+        if (s.query !== entityQuery) continue;
         try {
           const origin = new URL(s.url).origin;
           originCounts.set(origin, (originCounts.get(origin) || 0) + 1);
@@ -536,7 +557,7 @@ export class AgentOrchestrator {
         // Short and imperative on purpose: the executing model is a small
         // local one (gemma3:4b-class) and ignores hedged phrasing.
         (currentState
-          ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page). NEVER present older or dated content as the current answer.\n`
+          ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page); if the FIRST source does not state the answer, take it from the next sources in order. NEVER present older or dated content as the current answer.\n`
           : '') +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +

package/src/core/crawlers/BFSCrawler.js

@@ -12,6 +12,7 @@ import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.j
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { extractMainContent, isThinMainContent } from '../../tools/scrape/_mainContent.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -391,9 +392,27 @@ export class BFSCrawler {
     // Extract title
     const title = pageTitle($) || $('h1').first().text().trim() || '';
     
-    // Extract main content
+    // Extract main content. A page's chrome (header, mega-menu, footer)
+    // repeats on every page of a crawl and comes first in body order, so the
+    // 500-char preview showed the same "Explore Products" menu for all six
+    // Cessna pages, whose menu is plain <div>s outside any <nav> (R20,
+    // 2026-09-07). Use the same main-content pass scrape uses, fall back to
+    // the body minus its landmark chrome when Readability finds nothing or
+    // only a thin fragment, and to the whole body as a last resort.
     $('script, style, noscript').remove();
-    const content = $('body').text().replace(/\s+/g, ' ').trim();
+    const squash = (text) => text.replace(/\s+/g, ' ').trim();
+    let content = '';
+    try {
+      const main = extractMainContent(html, url);
+      if (main.html && !isThinMainContent(main.html, html)) {
+        content = squash(load(main.html)('body').text());
+      }
+    } catch { /* fall through to the body */ }
+    if (!content) {
+      const $body = $('body').clone();
+      $body.find('header, nav, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"]').remove();
+      content = squash($body.text()) || squash($('body').text());
+    }
     
     // Extract metadata
     const metadata = {

package/src/server/inlineThreshold.js

@@ -30,6 +30,9 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   extract_content: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.cleanedHTML'], truncate: true },
   crawl_deep: { textPaths: [], truncate: true },
   batch_scrape: { textPaths: [], truncate: true },
+  // A page of 25 markdown results is the same payload batch_scrape shapes;
+  // an async job's page came back as 111 KB whole (R20, 2026-09-07).
+  get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
   scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
   process_document: { textPaths: ['content.text'], truncate: true },

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -73,6 +73,8 @@ completed or in-progress job. Cheap (1 credit) because the batch was already
 paid for. Completed jobs are also exposed as `crawlforge://job/{jobId}`
 resources. Stored batch results share the local 1-hour result store that
 `read_result` reads, with the same eviction, so page through a batch within the hour.
+Like `batch_scrape`, it takes `max_inline_chars` (default 40,000): a page over the
+limit comes back as a `preview` plus a `result_handle` for `read_result`.
 
 ## scrape_with_actions — interact, then scrape (cost: 5)
 

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -13,6 +13,7 @@ import ExtractContentTool from '../extract/extractContent.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
+import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 
 // Recording / replay helpers
 import {
@@ -726,6 +727,26 @@ export class ScrapeWithActionsTool extends EventEmitter {
         }
       }
 
+      // extractContent hands back text alone when Readability finds no
+      // article. support.southwest.com's help centre is a JS shell: after an
+      // 8 s wait the body text was there, `html` came back empty and
+      // `markdown` was the "Content not available" placeholder, reported as
+      // success (R20, 2026-09-07). The post-action DOM is in hand, so serve
+      // the requested formats from it instead of a placeholder.
+      if (chainResult?.finalHtml) {
+        extractResult.content = extractResult.content || {};
+        if (params.formats?.includes('markdown') && !extractResult.content.markdown) {
+          const bodyMarkdown = htmlToMarkdown(chainResult.finalHtml);
+          if (bodyMarkdown) {
+            extractResult.content.markdown = bodyMarkdown;
+            extractResult.content.markdownSource = 'body';
+          }
+        }
+        if (params.formats?.includes('html') && !extractResult.content.html) {
+          extractResult.content.html = chainResult.finalHtml;
+        }
+      }
+
       return extractResult;
     } catch (error) {
       this.log('warn', `Final content extraction failed: ${error.message}`);

package/src/tools/basic/extractLinks.js

@@ -42,7 +42,9 @@ export async function extractLinksHandler({ url, filter_external, base_url, user
       const href = $(element).attr('href');
       const text = $(element).text().trim();
 
-      if (!href) return;
+      // A javascript: pseudo-link ("Cookie Settings") is a button, not a
+      // link; it was counted as an external link on boeing.com (R20).
+      if (!href || /^\s*javascript:/i.test(href)) return;
 
       try {
         const absoluteUrl = new URL(href, baseUrl).toString();

package/src/tools/scrape/_mainContent.js

@@ -172,6 +172,22 @@ function isDataTable(table) {
   return rows >= 10 || columns > 4;
 }
 
+/**
+ * A table whose author marked header cells carries data whatever its size —
+ * layout tables do not use <th>. The size test above misses every small fee
+ * or spec table: WestJet's checked-bag fees are 6 rows × 3 columns inside a
+ * `com-tabs` component (Readability's `negative` regex matches `com-`), and
+ * the page came back reading "fees are as follows:" with nothing following
+ * (R20, 2026-09-07).
+ * @param {HTMLTableElement} table
+ * @returns {boolean}
+ */
+function isHeadedTable(table) {
+  if (table.rows.length < 2) return false;
+  if (!table.querySelector('th')) return false;
+  return Array.from(table.rows).some((row) => row.cells.length >= 2);
+}
+
 /**
  * A data table's text with its structure kept: one line per row, cells joined
  * by " | ". A bare `textContent` runs every cell together
@@ -216,7 +232,7 @@ export function recoverDroppedTables(html, url, keptText = '') {
       // A nested table travels with its parent; re-attaching it separately
       // would duplicate it.
       .filter((table) => !table.parentElement?.closest('table'))
-      .filter(isDataTable)
+      .filter((table) => isDataTable(table) || isHeadedTable(table))
       .filter((table) => {
         const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
         return signature.length > 0 && !kept.includes(signature);

package/src/tools/search/redditSearch.js

@@ -373,7 +373,18 @@ export class RedditSearchTool {
       // caller the 422. Thread mode returned above, so both remaining modes
       // (posts, comments) narrow.
       const narrowable = Boolean(v.query) && !v.after;
-      if (!error.retryable || !narrowable) throw error;
+      if (!error.retryable || !narrowable) {
+        // The caller's window is respected, so the 422 is theirs to act on —
+        // but the generic hint ("add subreddit or author") is useless on a
+        // search that is already scoped (R20: r/aviation "737 MAX" after=30d).
+        if (error.retryable && v.query && v.after) {
+          throw new Error(
+            `${error.message} — Arctic Shift timed out searching your after=${v.after} window. ` +
+            'Pass a narrower after (7d, 3d, 1d), drop the query to list the newest posts in the scope, or read one post with mode:"thread" and link_id.'
+          );
+        }
+        throw error;
+      }
       let lastError = error;
       for (const window of ['7d', '3d', '1d']) {
         await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));

package/src/utils/htmlToMarkdown.js

@@ -16,10 +16,42 @@
 
 import TurndownService from 'turndown';
 import { gfm } from 'turndown-plugin-gfm';
+import { load } from 'cheerio';
 import { stripHiddenHtml } from './hiddenContent.js';
 
 let _td = null;
 
+/**
+ * turndown-plugin-gfm renders a table as a pipe table only when its first row
+ * is entirely <th>. A corner cell written as an empty <td> — WestJet's fee
+ * table opens `<td> </td><th>1st Bag</th><th>2nd Bag</th>` — fails that test,
+ * so the whole table fell to the layout-table rule and flattened to text
+ * lines with the columns lost (R20, 2026-09-07). Promote empty corner cells
+ * in an otherwise all-<th> first row so the table renders as a table.
+ * @param {string} html
+ * @returns {string}
+ */
+export function promoteCornerHeaderCells(html) {
+  if (!/<th[\s>]/i.test(html)) return html;
+  try {
+    const $ = load(html);
+    let changed = false;
+    $('table').each((_, table) => {
+      const firstRow = $(table).find('tr').first();
+      if (!firstRow.length || firstRow.closest('table')[0] !== table) return;
+      const cells = firstRow.children('th, td');
+      const tds = cells.filter('td');
+      if (cells.filter('th').length === 0 || tds.length === 0) return;
+      if (tds.toArray().some((td) => $(td).text().trim() !== '')) return;
+      tds.each((__, td) => { td.name = 'th'; });
+      changed = true;
+    });
+    return changed ? $.html() : html;
+  } catch {
+    return html;
+  }
+}
+
 // Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
 // whether it converts a table or keeps it as raw HTML.
 function isHeadingRow(tr) {
@@ -112,7 +144,7 @@ export function htmlToMarkdown(html, options = {}) {
     const visible = options.keepHiddenContent
       ? html
       : stripHiddenHtml(html, { css: options.css });
-    return getTurndown().turndown(visible).trim();
+    return getTurndown().turndown(promoteCornerHeaderCells(visible)).trim();
   } catch {
     // Fallback: strip tags, return plain text
     return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

package/src/utils/sitemapParser.js

@@ -10,6 +10,23 @@ import { noteRetryAfter } from './hostRateLimiter.js';
 
 const gunzip = promisify(zlib.gunzip);
 
+/**
+ * A sitemap <loc> resolved against the sitemap's own URL and normalized;
+ * null when it is empty or not a URL. The protocol wants absolute locs, but
+ * boeing.com's 1,878-entry sitemap is written with relative paths ("/",
+ * "/commercial"): `normalizeUrl("/")` threw out of the entry loop, the whole
+ * sitemap read as empty, and map_site fell back to crawling links — 75 URLs
+ * (R20, 2026-09-07). One bad entry must not discard the rest either.
+ */
+function resolveLoc(loc, base) {
+  if (!loc) return null;
+  try {
+    return normalizeUrl(new URL(loc, base).href);
+  } catch {
+    return null;
+  }
+}
+
 export class SitemapParser {
   constructor(options = {}) {
     const {
@@ -121,11 +138,11 @@ export class SitemapParser {
       // Parse sitemap index entries
       $('sitemap').each((_, element) => {
         const $sitemap = $(element);
-        const loc = $sitemap.find('loc').text().trim();
-        
+        const loc = resolveLoc($sitemap.find('loc').text().trim(), indexUrl);
+
         if (loc) {
           const sitemap = {
-            url: normalizeUrl(loc),
+            url: loc,
             lastmod: $sitemap.find('lastmod').text().trim() || null
           };
           sitemaps.push(sitemap);
@@ -347,7 +364,9 @@ export class SitemapParser {
     const cacheKey = this.cache?.generateKey(url, { depth: currentDepth });
     if (this.cache && cacheKey) {
       const cached = await this.cache.get(cacheKey);
-      if (cached) {
+      // An empty cached parse is a failure that got remembered (see the
+      // write side below); re-parse rather than serve it for an hour.
+      if (cached && (cached.urls?.length > 0 || cached.sitemaps?.length > 0)) {
         this.stats.cacheHits++;
         return cached;
       }
@@ -378,8 +397,12 @@ export class SitemapParser {
         }
       }
 
-      // Cache the result
-      if (this.cache && cacheKey) {
+      // Cache the result — but never an empty one. A parse that yielded no
+      // URL and no child sitemap is far more likely a failure than a fact
+      // (the relative-<loc> throw above sat in the disk cache for an hour
+      // and map_site kept answering 75 for boeing.com after the parser was
+      // fixed, R20 2026-09-07).
+      if (this.cache && cacheKey && (result.urls.length > 0 || result.sitemaps.length > 0)) {
         await this.cache.set(cacheKey, result);
       }
 
@@ -469,11 +492,11 @@ export class SitemapParser {
     // Parse standard URLs
     $('url').each((_, element) => {
       const $url = $(element);
-      const loc = $url.find('loc').text().trim();
-      
+      const loc = resolveLoc($url.find('loc').text().trim(), url);
+
       if (loc && result.urls.length < this.maxUrlsPerSitemap) {
         const urlData = {
-          loc: normalizeUrl(loc),
+          loc,
           lastmod: $url.find('lastmod').text().trim() || null,
           changefreq: $url.find('changefreq').text().trim() || null,
           priority: $url.find('priority').text().trim() || null
@@ -565,10 +588,10 @@ export class SitemapParser {
 
     $('sitemap').each((_, element) => {
       const $sitemap = $(element);
-      const loc = $sitemap.find('loc').text().trim();
-      
+      const loc = resolveLoc($sitemap.find('loc').text().trim(), url);
+
       if (loc) {
-        result.sitemaps.push(normalizeUrl(loc));
+        result.sitemaps.push(loc);
       }
     });