crawlforge-mcp-server 5.6.7 → 5.6.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.7",
+  "version": "5.6.9",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -116,7 +116,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.6.3",
+    "crawlforge-extractors": "^1.6.4",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -34,6 +34,7 @@ import { LocalizationManager } from "./src/core/LocalizationManager.js";
 import * as cheerio from "cheerio";
 import { htmlToMarkdown } from "./src/utils/htmlToMarkdown.js";
 import { detectChallengePage } from "./src/utils/challengeDetection.js";
+import { stealthDocumentVerdict } from "./src/utils/stealthVerdict.js";
 import { browserPreflight } from "./src/utils/robotsGate.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
@@ -106,7 +107,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.7",
+  version: "5.6.9",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1404,28 +1405,20 @@ registerToolIfEnabled("stealth_mode", {
 
         // A bot-wall interstitial arrives as HTTP 200 with a title and prose
         // of its own; reported as a successful scrape it hid the block for
-        // three rounds (R10 Q1 → R15). Name it — the content is still returned
-        // so the caller can see what the site served.
-        const challenge = detectChallengePage(scraped);
+        // three rounds (R10 Q1 → R15). An HTTP error page and a short
+        // error-titled placeholder did the same in R18. Name each — the
+        // content is still returned so the caller can see what the site served.
+        const verdict = stealthDocumentVerdict(scraped, { waitedMs: (wait_for || 0) + (scraped.gracedMs || 0) });
         result = {
-          success: !challenge,
+          success: verdict.success,
           url: scraped.url,
           title: scraped.title,
+          status: verdict.status,
           content: stealthScrapeFormats(formats, scraped)
         };
-        if (challenge) {
-          result.blocked = challenge;
-          result.error = `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); the stealth browser did not pass it.`;
-        } else if (!scraped.title && !String(scraped.text || '').trim()) {
-          // A document with no title and no text is not a scraped page; it
-          // is a render that has not happened yet (or an empty response).
-          // Reported as success it reads like a page with nothing on it.
-          result.success = false;
-          result.error =
-            `The stealth browser reached ${scraped.url} but the document rendered no title and no text` +
-            ` after ${wait_for || 0}ms of extra wait (${(scraped.html || '').length} bytes of HTML).` +
-            ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.';
-        }
+        if (verdict.blocked) result.blocked = verdict.blocked;
+        if (verdict.error) result.error = verdict.error;
+        if (scraped.gracedMs > 0) result.waited_for_render_ms = scraped.gracedMs;
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as
         // scrape and scrape_with_actions, so the base64 never bloats the result.

package/src/core/ActionExecutor.js

@@ -381,6 +381,7 @@ export class ActionExecutor extends EventEmitter {
         url,
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
+        navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
         screenshots: executionContext.screenshots,
@@ -1171,7 +1172,7 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
-    await page.goto(url, {
+    const response = await page.goto(url, {
       waitUntil: options.waitUntil || 'domcontentloaded',
       timeout: options.timeout || 30000
     });
@@ -1182,6 +1183,14 @@ export class ActionExecutor extends EventEmitter {
     if (/^https?:\/\//i.test(landedUrl)) {
       await assertUrlAllowed(landedUrl, { resolveDns: true });
     }
+
+    // Remembered on the page so the chain result can report the status of
+    // the last navigation: tesla.com's Akamai denial ran a full action chain
+    // and came back success:true (R18, 2026-09-04).
+    let status = null;
+    try { status = response ? response.status() : null; } catch { status = null; }
+    page.__crawlforgeNavigation = { url: landedUrl, status };
+    return { url: landedUrl, status };
   }
 
   /**

package/src/core/AgentOrchestrator.js

@@ -34,7 +34,11 @@ const DEFAULT_MAX_URLS = 10;
 function isRelevant(text, query) {
   if (!text || !query) return true; // fail-open
   const lc = text.toLowerCase();
-  return query.toLowerCase().split(/\s+/).some(term => term.length > 3 && lc.includes(term));
+  // Terms are compared bare: the quoted entity in `npm package "commander"?`
+  // arrived as `"commander"?` and matched nothing (R17, 2026-09-04).
+  return query.toLowerCase().split(/\s+/)
+    .map(term => term.replace(/[^\p{L}\p{N}]/gu, ''))
+    .some(term => term.length > 3 && lc.includes(term));
 }
 
 /**
@@ -45,6 +49,46 @@ function truncate(text, maxChars = 8000) {
   return text.slice(0, maxChars) + '\n[...truncated]';
 }
 
+/**
+ * Literal provenance check: every version, date and large number in the
+ * answer must appear somewhere in the fetched source text. Returns the raw
+ * strings that do not (deduplicated, in answer order). Comparison is
+ * case-insensitive with whitespace and thousands separators removed, so
+ * "6.3.3" matches "6.3.3" and "1,234" matches "1 234". URLs in the answer are
+ * ignored: they are citations, not claims.
+ */
+const MONTHS = '(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*';
+const VALUE_PATTERNS = [
+  /\b\d+\.\d+(?:\.\d+){0,2}\b/g,                                   // versions 1.2 / 1.2.3 / 1.2.3.4
+  /\b\d{4}-\d{2}-\d{2}\b/g,                                          // ISO dates
+  new RegExp(`\\b${MONTHS}\\.?\\s+\\d{1,2}(?:st|nd|rd|th)?,?\\s+\\d{4}\\b`, 'gi'), // January 18, 2024
+  new RegExp(`\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTHS}\\.?\\s+\\d{4}\\b`, 'gi'),   // 18 January 2024
+  /\b\d{1,3}(?:,\d{3})+\b|\b\d{4,}\b/g,                              // 1,234 / 12345
+];
+
+function normalizeValue(value) {
+  return value.toLowerCase().replace(/[\s,]/g, '');
+}
+
+export function unverifiedValues(answer, sourceText) {
+  if (!answer) return [];
+  const stripped = answer.replace(/https?:\/\/\S+/g, ' ');
+  const haystack = normalizeValue(sourceText || '');
+  const missing = [];
+  const seen = new Set();
+  for (const pattern of VALUE_PATTERNS) {
+    for (const match of stripped.matchAll(pattern)) {
+      const raw = match[0];
+      const key = normalizeValue(raw);
+      if (!key || seen.has(key)) continue;
+      seen.add(key);
+      if (!haystack.includes(key)) missing.push(raw);
+    }
+  }
+  // "2024" inside an already-flagged "January 18, 2024" is the same claim.
+  return missing.filter((v, i) => !missing.some((o, j) => j !== i && o.length > v.length && o.includes(v)));
+}
+
 /**
  * Current-state task gate: prompts about the live "now" ("right now",
  * "currently", "today", "latest", "#1 … now") go stale through search alone —
@@ -222,6 +266,17 @@ export class AgentOrchestrator {
     } catch {
       // Sampling unavailable — use raw prompt
     }
+    // A quoted term in the prompt ("commander") is the thing being asked
+    // about. A planner told to search the bare entity reduced
+    // 'npm package "commander"' to "npm", fetched only npm's own docs and
+    // answered from memory (R17, 2026-09-04). Every query carries the quoted
+    // terms.
+    const quotedTerms = [...prompt.matchAll(/"([^"\n]{2,60})"/g)].map(m => m[1].trim()).filter(Boolean);
+    if (quotedTerms.length > 0) {
+      searchQueries = searchQueries.map(q =>
+        quotedTerms.every(t => q.toLowerCase().includes(t.toLowerCase())) ? q : `${q} ${quotedTerms.join(' ')}`.trim()
+      );
+    }
 
     // ── GATHER (search) ───────────────────────────────────────────────────────
     const urlQueue = [...seedUrls]; // start with any user-provided seeds
@@ -238,6 +293,8 @@ export class AgentOrchestrator {
       if (!priorityUrls.includes(url)) priorityUrls.push(url);
     }
     const searchResults = [];
+    /** url -> { title, text }: the engine's excerpt, evidence of last resort for a page that cannot be fetched. */
+    const excerpts = new Map();
 
     if (urlQueue.length < capUrls) {
       try {
@@ -253,6 +310,11 @@ export class AgentOrchestrator {
               for (const r of parsed.results) {
                 if (r.link && !urlQueue.includes(r.link)) urlQueue.push(r.link);
                 searchResults.push({ query: q, title: r.title || '', url: r.link || '', snippet: r.snippet || '' });
+                // What the engine saw of the page: the snippet plus the
+                // page's own meta description, which for npmjs.com carries
+                // "Latest version: 15.0.0" while the snippet does not.
+                const excerpt = [r.snippet, r.pagemap?.metatags?.description].filter(Boolean).join(' ');
+                if (r.link && excerpt && !excerpts.has(r.link)) excerpts.set(r.link, { title: r.title || '', text: excerpt });
               }
             }
           } catch { /* skip failed search */ }
@@ -318,7 +380,17 @@ export class AgentOrchestrator {
           text: truncate(textContent),
           step
         });
-      } catch { /* skip unreachable URL */ }
+      } catch {
+        // The page is out of reach (challenge page, 403, timeout) but the
+        // search engine's excerpt of it is not: npmjs.com's "Latest version:
+        // 15.0.0" sat in the snippet while every fetch of the package page was
+        // challenged, and the run ended with no evidence at all (R17,
+        // 2026-09-04). Keep a relevant snippet as evidence, labelled as one.
+        const excerpt = excerpts.get(url);
+        if (excerpt && isRelevant(excerpt.text, prompt)) {
+          evidence.push({ url, title: excerpt.title, text: excerpt.text, snippet: true, step });
+        }
+      }
     }
 
     // ── SHAPE ─────────────────────────────────────────────────────────────────
@@ -340,7 +412,7 @@ export class AgentOrchestrator {
       .sort((a, b) => b._score - a._score);
     const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
     const combinedText = orderedEvidence
-      .map(e => `--- Source: ${e.url} ---\n${truncate(e.text, perSourceCap)}`)
+      .map(e => `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${truncate(e.text, perSourceCap)}`)
       .join('\n\n');
 
     if (!combinedText.trim()) {
@@ -386,6 +458,7 @@ export class AgentOrchestrator {
     let answer = null;
     let degraded = false;
     let degradedReason;
+    let unverified = [];
 
     try {
       // Wording matters for small local models (llama3.2-class): without the
@@ -414,6 +487,34 @@ export class AgentOrchestrator {
 
       const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024 });
       answer = text;
+
+      // A small model fills gaps from memory: "Swift 5.9.3", "commander 8.6.2
+      // published January 18, 2024" — none of it on any fetched page (R17,
+      // 2026-09-04). Versions, dates and counts in the answer must appear in
+      // the source text; what does not gets one corrective rewrite, and
+      // whatever still remains is named in the answer and in `provenance`.
+      unverified = unverifiedValues(answer, combinedText);
+      if (unverified.length > 0) {
+        const retryPrompt =
+          `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
+          `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` +
+          `Rewrite the answer. Keep only values that the sources state, spelled as the sources spell them. ` +
+          `Where the sources do not state a value, say that it is not stated instead of guessing.`;
+        try {
+          const { text: rewritten } = await this._getSamplingClient().complete(retryPrompt, { maxTokens: 1024 });
+          if (rewritten && rewritten.trim()) {
+            answer = rewritten;
+            unverified = unverifiedValues(answer, combinedText);
+          }
+        } catch {
+          // Keep the first answer; it is flagged below.
+        }
+      }
+      if (unverified.length > 0) {
+        answer +=
+          `\n\nProvenance warning: ${unverified.map(v => `"${v}"`).join(', ')} ` +
+          `${unverified.length === 1 ? 'does' : 'do'} not appear in the fetched sources and may be invented.`;
+      }
     } catch (err) {
       degraded = true;
       degradedReason = `LLM synthesis unavailable: ${err.message}`;
@@ -425,11 +526,12 @@ export class AgentOrchestrator {
       success: true,
       answer,
       search_results: searchResults,
-      evidence: degraded ? evidence : evidence.map(e => ({ url: e.url })),
+      evidence: degraded ? evidence : evidence.map(e => (e.snippet ? { url: e.url, snippet: true } : { url: e.url })),
       degraded,
       reason: degradedReason,
       steps: step,
-      urls_fetched: urlsFetched
+      urls_fetched: urlsFetched,
+      provenance: { checked: !degraded, unverified }
     };
   }
 

package/src/core/ChangeTracker.js

@@ -13,6 +13,7 @@ import { load } from 'cheerio';
 import { diffWords, diffLines, diffChars } from 'diff';
 import { structureSignature, structuralSimilarity } from 'crawlforge-extractors';
 import { calculateSimilarity as calculateContentSimilarity } from '../tools/tracking/trackChanges/differ.js';
+import { pageTitle } from '../utils/pageTitle.js';
 
 const ChangeTrackingSchema = z.object({
   url: z.string().url(),
@@ -892,7 +893,7 @@ export class ChangeTracker extends EventEmitter {
   
   extractMetadata($, options) {
     const metadata = {
-      title: $('title').text() || '',
+      title: pageTitle($) || '',
       headings: [],
       links: [],
       images: [],

package/src/core/LLMsTxtAnalyzer.js

@@ -9,6 +9,7 @@ import { safeFetch } from '../utils/ssrfGuard.js';
 import { resolveUserAgent } from '../utils/fetchIdentity.js';
 import { preflightFetch } from '../utils/robotsGate.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
+import { pageTitle } from '../utils/pageTitle.js';
 
 const logger = new Logger('LLMsTxtAnalyzer');
 
@@ -722,7 +723,7 @@ export class LLMsTxtAnalyzer {
         type: 'static', 
         confidence: 0.7,
         metadata: { 
-          title: $('title').text().trim(),
+          title: pageTitle($),
           contentLength: html.length 
         }
       };

package/src/core/LocalizationManager.js

@@ -262,7 +262,7 @@ export class LocalizationManager extends EventEmitter {
       timezone: validatedInput.timezone || countryData.timezone,
       currency: validatedInput.currency || countryData.currency,
       searchDomain: countryData.searchDomain,
-      acceptLanguage: this.buildAcceptLanguageHeader(validatedInput.language || countryData.language),
+      acceptLanguage: this.buildAcceptLanguageHeader(validatedInput.language || countryData.language, validatedInput.countryCode),
       customHeaders: validatedInput.customHeaders || {},
       geoLocation: validatedInput.geoLocation,
       proxySettings: validatedInput.proxySettings,
@@ -601,8 +601,18 @@ export class LocalizationManager extends EventEmitter {
     return await this.configureCountry(countryCode);
   }
   
-  buildAcceptLanguageHeader(language) {
-    const langCode = language.split('-')[0];
+  /**
+   * The header a browser in `countryCode` sends for `language`. A bare
+   * language takes the country as its region — configure_country GB used to
+   * answer en-US (R18, 2026-09-04) because the country never reached here.
+   */
+  buildAcceptLanguageHeader(language, countryCode) {
+    const [langCode, region] = String(language).split('-');
+    const cc = region || (countryCode ? String(countryCode).toUpperCase() : null);
+    if (cc) {
+      const tag = `${langCode}-${cc}`;
+      return langCode === 'en' ? `${tag},en;q=0.9` : `${tag},${langCode};q=0.9,en;q=0.8`;
+    }
     return LANGUAGE_MAPPINGS[langCode] || `${language},${langCode};q=0.9,en;q=0.8`;
   }
   
@@ -679,11 +689,15 @@ export class LocalizationManager extends EventEmitter {
       return this.timezoneCache.get(timezone);
     }
     
-    // Calculate dynamically if not cached
+    // Calculate dynamically if not cached. Compare the target zone against
+    // UTC, never against the host zone: the old form added the host offset
+    // once more, so America/Sao_Paulo came out as -420 from a UTC-4 host
+    // (R17, 2026-09-04). Both wall-clock strings are parsed the same way, so
+    // the difference is the zone's current offset in minutes.
     const now = new Date();
-    const utc = new Date(now.getTime() + (now.getTimezoneOffset() * 60000));
-    const targetTime = new Date(utc.toLocaleString('en-US', { timeZone: timezone }));
-    const offset = (targetTime.getTime() - utc.getTime()) / (1000 * 60);
+    const asTarget = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
+    const asUtc = new Date(now.toLocaleString('en-US', { timeZone: 'UTC' }));
+    const offset = Math.round((asTarget.getTime() - asUtc.getTime()) / 60000);
     
     this.timezoneCache.set(timezone, offset);
     return offset;
@@ -1344,10 +1358,13 @@ export class LocalizationManager extends EventEmitter {
     }
     
     // CJK script detection
+    // Kana and Hangul are tested before the unified ideographs: Japanese
+    // text is full of kanji, so a chinese-first order labelled every
+    // Japanese page "chinese" (R17, 2026-09-04).
     const cjkPatterns = {
-      chinese: /[\u4E00-\u9FFF]/,
       japanese: /[\u3040-\u309F\u30A0-\u30FF]/,
-      korean: /[\uAC00-\uD7AF]/
+      korean: /[\uAC00-\uD7AF]/,
+      chinese: /[\u4E00-\u9FFF]/
     };
     
     for (const [script, pattern] of Object.entries(cjkPatterns)) {

package/src/core/ResearchOrchestrator.js

@@ -79,6 +79,41 @@ const MAX_CONFLICT_PAIRS = 40;
  * - Advanced provenance tracking and activity logging
  * - Smart URL prioritization based on content quality
  */
+// A research topic is often a paragraph ("Why does X get stuck on Y and what
+// makes Z pass it in 2026: A, B, C, D and known working approaches"). Google
+// answers a 40-word query with nothing: the same run found 1 source from the
+// paragraph and 8 from a six-word restatement (R18, 2026-09-04). The full
+// topic still steers the LLM expansion and the relevance ranking; the search
+// backend gets a keyword-dense form of it.
+export const SEARCH_TOPIC_MAX_WORDS = 12;
+export const SEARCH_QUERY_MAX_WORDS = 16;
+const TOPIC_STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'for', 'with', 'from', 'by', 'as', 'at', 'into',
+  'what', 'why', 'how', 'when', 'where', 'which', 'who', 'whom', 'whose', 'that', 'this', 'these', 'those',
+  'does', 'do', 'did', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'it', 'its', 'their', 'there', 'they',
+  'can', 'could', 'should', 'would', 'will', 'may', 'might', 'get', 'gets', 'make', 'makes', 'made', 'about',
+  'known', 'required', 'versus', 'vs', 'i', 'we', 'you', 'my', 'our', 'your', 'me', 'us', 'them', 'not', 'no', 'so'
+]);
+
+export function compactSearchTopic(topic) {
+  const clean = String(topic || '').replace(/\s+/g, ' ').trim();
+  const words = clean.split(' ').filter(Boolean);
+  if (words.length <= SEARCH_TOPIC_MAX_WORDS) return clean;
+  const bare = clean.replace(/[“”"'`()[\]{}:;,?!.]/g, ' ').split(/\s+/).filter(Boolean);
+  const kept = [];
+  for (const word of bare) {
+    if (TOPIC_STOPWORDS.has(word.toLowerCase())) continue;
+    kept.push(word);
+    if (kept.length >= SEARCH_TOPIC_MAX_WORDS) break;
+  }
+  return (kept.length > 0 ? kept : bare.slice(0, SEARCH_TOPIC_MAX_WORDS)).join(' ');
+}
+
+export function clampSearchQuery(query, maxWords = SEARCH_QUERY_MAX_WORDS) {
+  const words = String(query || '').replace(/\s+/g, ' ').trim().split(' ').filter(Boolean);
+  return (words.length > maxWords ? words.slice(0, maxWords) : words).join(' ');
+}
+
 export class ResearchOrchestrator extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -374,9 +409,17 @@ export class ResearchOrchestrator extends EventEmitter {
         });
       }
 
-      // Add research-specific query variations
-      const researchVariations = this.generateResearchVariations(topic);
-      const allQueries = [...new Set([topic, ...expandedQueries, ...researchVariations])];
+      // Add research-specific query variations. The search backend sees the
+      // keyword-dense form of a long topic; every query is clamped to what a
+      // search engine answers.
+      const searchTopic = compactSearchTopic(topic);
+      if (searchTopic !== topic) {
+        this.logger.info('Long research topic compacted for search', { topic, searchTopic });
+      }
+      const researchVariations = this.generateResearchVariations(searchTopic);
+      const allQueries = [...new Set(
+        [searchTopic, ...expandedQueries, ...researchVariations].map((q) => clampSearchQuery(q)).filter(Boolean)
+      )];
 
       // Rank queries by research relevance with semantic understanding
       const rankedQueries = await this.rankResearchQueriesWithSemantics(allQueries, topic);
@@ -390,7 +433,7 @@ export class ResearchOrchestrator extends EventEmitter {
     } catch (error) {
       this.logger.warn('Topic expansion failed, using original topic', { error: error.message });
       this.metrics.queryExpansionTime += Date.now() - startTime;
-      return [topic];
+      return [compactSearchTopic(topic)];
     }
   }