crawlforge-mcp-server 6.22.0 → 6.22.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.22.0",
+  "version": "6.22.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.22.0",
+  version: "6.22.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/AgentOrchestrator.js

@@ -20,6 +20,7 @@
  * pro model: delegates to ResearchOrchestrator.conductResearch() for richer synthesis.
  */
 
+import { load } from 'cheerio';
 import { fetchAndParse } from '../tools/extract/_fetchAndParse.js';
 import { SamplingClient } from './SamplingClient.js';
 import { fenceUntrusted } from '../utils/untrustedContent.js';
@@ -75,7 +76,7 @@ function truncate(text, maxChars = 8000) {
  * strings that do not (deduplicated, in answer order). Comparison is
  * case-insensitive with whitespace and thousands separators removed, so
  * "6.3.3" matches "6.3.3" and "1,234" matches "1 234". URLs in the answer are
- * ignored: they are citations, not claims.
+ * skipped here: they are citations, checked by unverifiedUrls.
  */
 const MONTHS = '(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*';
 const VALUE_PATTERNS = [
@@ -109,6 +110,71 @@ export function unverifiedValues(answer, sourceText) {
   return missing.filter((v, i) => !missing.some((o, j) => j !== i && o.length > v.length && o.includes(v)));
 }
 
+/**
+ * The http(s) URLs in a text, trailing punctuation removed. A closing
+ * parenthesis stays only when it closes one opened inside the URL, so
+ * "[HN](https://x/item?id=1)" yields the link and "/wiki/Foo_(bar)" survives.
+ */
+export function urlsIn(text) {
+  return (String(text || '').match(/https?:\/\/[^\s<>"'`\]]+/g) || []).map(url => {
+    for (;;) {
+      if (/[.,;:!?*'"]$/.test(url)) url = url.slice(0, -1);
+      else if (url.endsWith(')') && url.split('(').length < url.split(')').length) url = url.slice(0, -1);
+      else return url;
+    }
+  });
+}
+
+/**
+ * A URL as a citation is compared: scheme, "www.", fragment and trailing
+ * slash dropped, host lowercased, "&amp;" decoded. Null when it does not parse.
+ */
+function citationKey(url) {
+  try {
+    const u = new URL(String(url).replace(/&amp;/g, '&'));
+    return `${u.host.toLowerCase().replace(/^www\./, '')}${u.pathname.replace(/\/+$/, '')}${u.search}`;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * URLs the answer cites that no fetched page supports: not a fetched URL
+ * (before or after redirects), not a link on a fetched page, and not written
+ * in the source text. The R25 re-check (2026-10-19) answered the current HN
+ * top story with the right title and a link to a 2021 story that none of the
+ * sources contained, and the value-only check above skipped it as a citation.
+ *
+ * @param {string} answer
+ * @param {Iterable<string>} citable fetched URLs, their final URLs and the links on them
+ * @param {string} [sourceText]
+ * @returns {string[]} the cited URLs as written, deduplicated
+ */
+export function unverifiedUrls(answer, citable, sourceText = '') {
+  const known = new Set([...citable, ...urlsIn(sourceText)].map(citationKey).filter(Boolean));
+  const seen = new Set();
+  return urlsIn(answer).filter(url => {
+    const key = citationKey(url);
+    if (!key || seen.has(key)) return false;
+    seen.add(key);
+    return !known.has(key);
+  });
+}
+
+/** Absolute http(s) hrefs of the links on a parsed page. */
+function pageLinks($, baseUrl) {
+  const links = [];
+  $('a[href]').each((_, el) => {
+    try {
+      const url = new URL($(el).attr('href'), baseUrl);
+      if (url.protocol === 'http:' || url.protocol === 'https:') links.push(url.href);
+    } catch {
+      // An unparsable href cites nothing.
+    }
+  });
+  return links;
+}
+
 /**
  * Current-state task gate: prompts about the live "now" ("right now",
  * "current", "currently", "today", "latest", "#1 … now") go stale through search alone —
@@ -596,6 +662,9 @@ export class AgentOrchestrator {
     // cap whenever it was smaller than capUrls, leaving maxUrls unreachable
     // at its default. Both caps remain fully enforced (neither is weakened).
     const evidence = [];
+    // URLs an answer may cite: every evidence page's URL (requested and
+    // final) and the links on it.
+    const citable = [];
     let urlsFetched = 0;
     let step = 0;
     const warnings = [];
@@ -631,6 +700,7 @@ export class AgentOrchestrator {
             text: truncate(plain.textContent),
             step
           });
+          citable.push(url, plain.finalUrl, ...pageLinks(plain.$, plain.finalUrl));
           continue;
         }
         // A real page that is simply off-topic: nothing to retry.
@@ -675,6 +745,7 @@ export class AgentOrchestrator {
               via: 'stealth',
               step
             });
+            citable.push(url, stealth.url || url, ...pageLinks(load(stealth.html || ''), stealth.url || url));
             continue;
           }
           warnings.push(`stealth retry of ${url} ${rendered.success ? 'returned a page unrelated to the task' : `did not get the page either${rendered.blocked?.vendor ? ` (${rendered.blocked.vendor})` : ''}`}`);
@@ -685,6 +756,7 @@ export class AgentOrchestrator {
 
       if (relevantExcerpt) {
         evidence.push({ url, title: relevantExcerpt.title, text: relevantExcerpt.text, snippet: true, step });
+        citable.push(url);
       }
     }
 
@@ -777,6 +849,7 @@ export class AgentOrchestrator {
     let degraded = false;
     let degradedReason;
     let unverified = [];
+    let unverifiedLinks = [];
     let forumOnlyVersions = [];
     // Distinct from `degraded`: a run can be degraded because its version
     // claims are unsupported and still have had its provenance checked.
@@ -820,11 +893,16 @@ export class AgentOrchestrator {
       // 2026-09-04). Versions, dates and counts in the answer must appear in
       // the source text; what does not gets one corrective rewrite, and
       // whatever still remains is named in the answer and in `provenance`.
+      // A cited URL is checked the same way (R25 re-check, 2026-10-19: the
+      // right HN title with an invented item link).
+      const sourceText = evidence.map(e => e.text || '').join('\n');
       unverified = unverifiedValues(answer, combinedText);
-      if (unverified.length > 0 && remaining() >= REWRITE_MIN_REMAINING_MS) {
+      unverifiedLinks = unverifiedUrls(answer, citable, sourceText);
+      if ((unverified.length > 0 || unverifiedLinks.length > 0) && remaining() >= REWRITE_MIN_REMAINING_MS) {
         const retryPrompt =
           `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
-          `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` +
+          (unverified.length > 0 ? `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` : '') +
+          (unverifiedLinks.length > 0 ? `It cites URLs that are not among the sources or the links on them: ${unverifiedLinks.join(', ')}. Cite only the source URLs given above. ` : '') +
           `Rewrite the answer. Keep only values that the sources state, spelled as the sources spell them. ` +
           `Where the sources do not state a value, say that it is not stated instead of guessing.`;
         try {
@@ -832,6 +910,7 @@ export class AgentOrchestrator {
           if (rewritten && rewritten.trim()) {
             answer = rewritten;
             unverified = unverifiedValues(answer, combinedText);
+            unverifiedLinks = unverifiedUrls(answer, citable, sourceText);
           }
         } catch {
           // Keep the first answer; it is flagged below.
@@ -842,6 +921,11 @@ export class AgentOrchestrator {
           `\n\nProvenance warning: ${unverified.map(v => `"${v}"`).join(', ')} ` +
           `${unverified.length === 1 ? 'does' : 'do'} not appear in the fetched sources and may be invented.`;
       }
+      if (unverifiedLinks.length > 0) {
+        answer +=
+          `\n\nProvenance warning: ${unverifiedLinks.join(', ')} ` +
+          `${unverifiedLinks.length === 1 ? 'is' : 'are'} not among the fetched pages or the links on them and may be invented.`;
+      }
 
       // Asked for a current version, the agent answered "2.4.3" from a forum
       // thread about a different project while the project's own site and its
@@ -931,6 +1015,7 @@ export class AgentOrchestrator {
       provenance: {
         checked: !synthesisFailed,
         unverified,
+        ...(unverifiedLinks.length > 0 ? { unverified_urls: unverifiedLinks } : {}),
         ...(forumOnlyVersions.length > 0 ? { unsupported_versions: forumOnlyVersions } : {})
       }
     };

package/src/core/ResearchOrchestrator.js

@@ -67,6 +67,27 @@ export function reportedFindings(keyFindings) {
 // At most three digits, so a year such as "(2024)" stays.
 const CITATION_MARKER = /\s*(?:\(\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\)|\[\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\])/g;
 const LIST_MARKER = /^(?:[-*\u2022\u25AA\u2013]|\d{1,2}[.)])\s+/;
+// A lowercase word run straight into a capitalized one: "glanceWhat". A
+// camelCase name starts with a capital ("JavaScript", "DataDome") and never matches.
+const WELDED_WORDS = /\b[a-z]{2,}[A-Z][a-z]/g;
+
+/**
+ * A line that is a table row or headings welded together, not a statement.
+ * extract_content reads a table one row per line, cells joined by " | " (a
+ * two-cell row has just one), and crawlforge-extractors' flattenText joins
+ * inline elements with no separator, so a table of contents of `<a class="block">` links came back as "The six
+ * vendors at a glanceWhat the independent benchmarks showWhy your approach…"
+ * (R25 re-check, 2026-10-19). A sentence ends in punctuation; welded headings
+ * have none.
+ *
+ * @param {string} line
+ * @returns {boolean}
+ */
+function isPageFragment(line) {
+  if (/\s\|\s/.test(line) || (line.match(/\|/g) || []).length >= 2) return true;
+  return line.length <= 300 && !/[.!?](?:\s|$)/.test(line) &&
+    (line.match(WELDED_WORDS) || []).length >= 2;
+}
 
 /**
  * The statements in one extracted key point. A key point is a sentence as the
@@ -74,18 +95,25 @@ const LIST_MARKER = /^(?:[-*\u2022\u25AA\u2013]|\d{1,2}[.)])\s+/;
  * string ("Built-in encryption\nQUIC was built with security in mind.").
  * Each line becomes its own statement, except a line starting in lowercase,
  * which continues the one before it (wrapped text). List and citation markers
- * are removed. Heading-only lines are left for isAdmissibleClaim to reject.
+ * are removed. Table rows and welded headings are dropped; heading-only lines
+ * are left for isAdmissibleClaim to reject.
  *
  * @param {string} point
  * @returns {string[]}
  */
 export function claimStatements(point) {
   const blocks = [];
+  let afterFragment = false;
   for (const raw of String(point).split('\n')) {
     const line = raw.trim();
     if (!line) continue;
-    if (blocks.length > 0 && /^[a-z]/.test(line)) blocks[blocks.length - 1] += ` ${line}`;
+    if (isPageFragment(line)) {
+      afterFragment = true;
+      continue;
+    }
+    if (blocks.length > 0 && !afterFragment && /^[a-z]/.test(line)) blocks[blocks.length - 1] += ` ${line}`;
     else blocks.push(line);
+    afterFragment = false;
   }
   return blocks
     .map(block => block.replace(LIST_MARKER, '').replace(CITATION_MARKER, '').replace(/\s+/g, ' ').trim())

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.22.0
+  version: 6.22.1
   source: crawlforge-mcp-server
 ---