crawlforge-mcp-server 5.0.3 → 5.0.4

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.3
+**Current Version:** 5.0.4
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.3",
+  "version": "5.0.4",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/src/constants/config.js

@@ -303,7 +303,11 @@ export function validateConfig() {
 export function getToolConfig(toolName) {
   const toolConfigs = {
     search_web: {
-      apiKey: config.crawlforge.apiKey,
+      // Read env at call time, not import time: the CLI resolves the key
+      // (--api-key flag / env / ~/.crawlforge/config.json) in a preAction
+      // hook that runs AFTER this module was imported, so the frozen
+      // config.crawlforge.apiKey snapshot misses stored keys.
+      apiKey: process.env.CRAWLFORGE_API_KEY || config.crawlforge.apiKey,
       apiBaseUrl: config.crawlforge.apiBaseUrl,
       
       // Common configuration

package/src/core/AgentOrchestrator.js

@@ -200,9 +200,13 @@ export class AgentOrchestrator {
     // "news.ycombinator.com") are the most authoritative sources for the task —
     // queue them ahead of search results.
     const namedSites = prompt.match(/https?:\/\/[^\s"'<>]+|(?<![\w.@/-])[a-z0-9][\w-]*(?:\.[a-z0-9][\w-]*)*\.[a-z]{2,}(?![\w-])/gi) || [];
+    // Seeds and prompt-named sites also outrank search results at SHAPE time
+    // (see priorityUrls below) — generic term overlap must not bury them.
+    const priorityUrls = [...seedUrls];
     for (const site of namedSites) {
       const url = (/^https?:\/\//i.test(site) ? site : `https://${site}`).replace(/[.,;:!?)]+$/, '');
       if (!urlQueue.includes(url)) urlQueue.push(url);
+      if (!priorityUrls.includes(url)) priorityUrls.push(url);
     }
     const searchResults = [];
 
@@ -261,10 +265,16 @@ export class AgentOrchestrator {
     // url+title+text) instead of raw queue order, then give every source a
     // per-source slice of the synthesis budget so no source is silently cut off.
     const promptTerms = prompt.toLowerCase().split(/\s+/).filter(t => t.length > 3);
+    // Seeds/prompt-named sites get a fixed boost above any term-overlap score:
+    // a generic article mentioning the task's words ("title", "story") must
+    // never outrank the source the user explicitly pointed at (live retest
+    // 2026-08-20: an NFL article buried the named news.ycombinator.com page).
+    const isPriority = url => priorityUrls.some(p => url === p || url.startsWith(`${p}/`) || `${url}/` === p);
     const orderedEvidence = evidence
       .map(e => ({
         ...e,
-        _score: promptTerms.filter(t => `${e.url} ${e.title || ''} ${e.text}`.toLowerCase().includes(t)).length
+        _score: (isPriority(e.url) ? 1000 : 0) +
+          promptTerms.filter(t => `${e.url} ${e.title || ''} ${e.text}`.toLowerCase().includes(t)).length
       }))
       .sort((a, b) => b._score - a._score);
     const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
@@ -317,14 +327,19 @@ export class AgentOrchestrator {
     let degradedReason;
 
     try {
+      // Wording matters for small local models (llama3.2-class): without the
+      // "already fetched / do not refuse" framing they answer "I cannot access
+      // the internet" or "the sources do not contain it" even when the answer
+      // sits in the first source (live retest 2026-08-20).
       const synthesisPrompt =
-        `You are a research assistant. Answer this task using ONLY the sources below:\n\n` +
+        `You are a data-extraction assistant. The web sources below have ALREADY been fetched for you and their text is included — no internet access is needed; do not refuse for lack of browsing ability. They are ordered most-relevant first. Answer the task using ONLY this source text:\n\n` +
         `Task: ${prompt}\n\n` +
         `${combinedText}\n\n` +
         `Rules:\n` +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
+        `- Read the sources carefully before concluding anything is missing from them.\n` +
         `- Cite the exact source URL(s) you used.\n` +
-        `- If the sources do not contain the answer, say so explicitly.\n` +
+        `- If, after careful reading, the sources genuinely do not contain the answer, say so explicitly.\n` +
         `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
 

package/src/tools/extract/_fetchAndParse.js

@@ -96,6 +96,37 @@ function classifyContentType(contentType) {
   return 'binary';
 }
 
+/**
+ * Flatten a parsed document's <body> to text while preserving line structure.
+ *
+ * Cheerio's .text() joins adjacent elements with no separator, which welds
+ * table rows / list items together ("1.Story title329 points") and starves
+ * downstream LLM extraction (AgentOrchestrator, extractStructured,
+ * extractWithLlm) of any structure to parse. Works on a detached clone so the
+ * caller's $ tree is untouched — unifiedScrape reuses $ for other formats.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string}
+ */
+export function flattenBodyText($) {
+  // Block boundaries are marked with a U+E000 private-use sentinel so the
+  // HTML source's own insignificant newlines can be collapsed to spaces
+  // first, and only the sentinels become line breaks. (NUL won't survive:
+  // .after()/.replaceWith() parse their argument as HTML and the parser
+  // strips NUL; U+E000 passes through and never occurs in real page text.)
+  const $body = $('body').clone();
+  $body.find('br').replaceWith('\uE000');
+  $body.find('td, th').after(' ');
+  $body
+    .find('p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, section, article, header, footer')
+    .after('\uE000');
+  return $body
+    .text()
+    .replace(/\s+/g, ' ')
+    .replace(/ ?(?:\uE000 ?)+/g, '\n')
+    .trim();
+}
+
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
  *
@@ -149,7 +180,7 @@ export async function fetchAndParse(url, options = {}) {
     $(stripTags.join(', ')).remove();
   }
 
-  const textContent = $('body').text().replace(/\s+/g, ' ').trim();
+  const textContent = flattenBodyText($);
 
   return { html, $, textContent, finalUrl: response.url };
 }