crawlforge-mcp-server 6.22.2 → 6.22.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.22.2",
+  "version": "6.22.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.22.2",
+  version: "6.22.3",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/AgentOrchestrator.js

@@ -161,13 +161,15 @@ export function unverifiedUrls(answer, citable, sourceText = '') {
   });
 }
 
-/** Absolute http(s) hrefs of the links on a parsed page. */
+/** The http(s) links on a parsed page: absolute href and anchor text. */
 function pageLinks($, baseUrl) {
   const links = [];
   $('a[href]').each((_, el) => {
     try {
       const url = new URL($(el).attr('href'), baseUrl);
-      if (url.protocol === 'http:' || url.protocol === 'https:') links.push(url.href);
+      if (url.protocol === 'http:' || url.protocol === 'https:') {
+        links.push({ url: url.href, text: $(el).text().replace(/\s+/g, ' ').trim() });
+      }
     } catch {
       // An unparsable href cites nothing.
     }
@@ -175,6 +177,38 @@ function pageLinks($, baseUrl) {
   return links;
 }
 
+/** Most links one source lists under its text in the synthesis prompt. */
+const SOURCE_LINKS_MAX = 8;
+
+/**
+ * The links a source lists under its text in the synthesis prompt. Page text
+ * carries no link targets: the model could name the current HN top story but
+ * not link it, and cited an item URL from memory (R25 re-check, 2026-10-19).
+ * Listed: links whose anchor text is printed in the excerpt the model reads
+ * and is long enough to name something (three words: a title, not "97
+ * comments" or "next"), links to the page itself left out. Those whose anchor
+ * text carries one of the task's terms come first, then page order.
+ *
+ * @param {{url: string, text: string}[]} links
+ * @param {string} excerpt the source text the prompt carries
+ * @param {string} pageUrl
+ * @param {string[]} terms the task's terms, lowercase
+ * @returns {{url: string, text: string}[]}
+ */
+export function linksForSynthesis(links, excerpt, pageUrl, terms) {
+  const self = citationKey(pageUrl);
+  const seen = new Set();
+  const listed = (links || []).filter(({ url, text }) => {
+    const key = citationKey(url);
+    if (!key || key === self || seen.has(key)) return false;
+    if (text.split(' ').length < 3 || !excerpt.includes(text)) return false;
+    seen.add(key);
+    return true;
+  });
+  const onTask = link => terms.some(t => link.text.toLowerCase().includes(t));
+  return [...listed.filter(onTask), ...listed.filter(l => !onTask(l))].slice(0, SOURCE_LINKS_MAX);
+}
+
 /**
  * Current-state task gate: prompts about the live "now" ("right now",
  * "current", "currently", "today", "latest", "#1 … now") go stale through search alone —
@@ -665,6 +699,8 @@ export class AgentOrchestrator {
     // URLs an answer may cite: every evidence page's URL (requested and
     // final) and the links on it.
     const citable = [];
+    /** evidence url -> the links on that page, for the synthesis prompt. */
+    const linksOf = new Map();
     let urlsFetched = 0;
     let step = 0;
     const warnings = [];
@@ -700,7 +736,9 @@ export class AgentOrchestrator {
             text: truncate(plain.textContent),
             step
           });
-          citable.push(url, plain.finalUrl, ...pageLinks(plain.$, plain.finalUrl));
+          const links = pageLinks(plain.$, plain.finalUrl);
+          linksOf.set(plain.finalUrl, links);
+          citable.push(url, plain.finalUrl, ...links.map(l => l.url));
           continue;
         }
         // A real page that is simply off-topic: nothing to retry.
@@ -745,7 +783,9 @@ export class AgentOrchestrator {
               via: 'stealth',
               step
             });
-            citable.push(url, stealth.url || url, ...pageLinks(load(stealth.html || ''), stealth.url || url));
+            const links = pageLinks(load(stealth.html || ''), stealth.url || url);
+            linksOf.set(stealth.url || url, links);
+            citable.push(url, stealth.url || url, ...links.map(l => l.url));
             continue;
           }
           warnings.push(`stealth retry of ${url} ${rendered.success ? 'returned a page unrelated to the task' : `did not get the page either${rendered.blocked?.vendor ? ` (${rendered.blocked.vendor})` : ''}`}`);
@@ -782,7 +822,12 @@ export class AgentOrchestrator {
     const sliceTerms = promptTerms.map(t => t.replace(/[^\p{L}\p{N}]/gu, ''))
       .filter(t => t.length > 3 && !/^(what|which|when|where|does|that|this|with|from|have|about)$/.test(t));
     const combinedText = orderedEvidence
-      .map(e => `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${sliceForSynthesis(e.text, perSourceCap, sliceTerms)}`)
+      .map(e => {
+        const excerpt = sliceForSynthesis(e.text, perSourceCap, sliceTerms);
+        const links = linksForSynthesis(linksOf.get(e.url), excerpt, e.url, sliceTerms);
+        return `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${excerpt}` +
+          (links.length > 0 ? `\nLinks on this page:\n${links.map(l => `- ${l.text}: ${l.url}`).join('\n')}` : '');
+      })
       .join('\n\n');
 
     if (!combinedText.trim()) {
@@ -880,7 +925,7 @@ export class AgentOrchestrator {
             : '') +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +
-        `- Cite the exact source URL(s) you used.\n` +
+        `- Cite the exact URL(s) you used: a source URL, or the URL an item has under "Links on this page".\n` +
         `- If, after careful reading, the sources genuinely do not contain the answer, say so explicitly.\n` +
         `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
@@ -971,14 +1016,22 @@ export class AgentOrchestrator {
       // it. R25 (2026-10-19) answered "not stated" over a promoted Wikipedia
       // Main Page, and the HN question from thehackernews.com, both with
       // degraded:false. The answer counts as the live page's when it cites
-      // that host or states a value (version, date, number) the page shows.
+      // that host, states a value (version, date, number) the page shows, or
+      // names an item the page links (anchor text of three words or more:
+      // the HN title "DeepSeek just open-sourced its inference engine",
+      // 2026-10-20). An invented URL on that host is no citation of it: it is
+      // taken out before the host is looked for.
       if (currentState && promotedOrigin && !degraded) {
         const host = u => { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return null; } };
         const liveHost = host(promotedOrigin);
-        const liveText = evidence.filter(e => host(e.url) === liveHost).map(e => e.text || '').join('\n');
+        const livePages = evidence.filter(e => host(e.url) === liveHost);
+        const liveText = livePages.map(e => e.text || '').join('\n');
         // unverifiedValues against an empty text lists every value the answer states.
         const stated = unverifiedValues(answer, '');
-        const fromLivePage = answer.toLowerCase().includes(liveHost) ||
+        const cited = unverifiedLinks.reduce((text, url) => text.split(url).join(' '), answer).toLowerCase();
+        const namesLiveItem = livePages.some(e => (linksOf.get(e.url) || [])
+          .some(l => l.text.split(' ').length >= 3 && cited.includes(l.text.toLowerCase())));
+        const fromLivePage = cited.includes(liveHost) || namesLiveItem ||
           (liveText !== '' && unverifiedValues(answer, liveText).length < stated.length);
         if (!fromLivePage) {
           degraded = true;

package/src/core/ResearchOrchestrator.js

@@ -120,6 +120,25 @@ export function claimStatements(point) {
     .filter(Boolean);
 }
 
+/**
+ * The first `max` characters of a page's text, ended at the last sentence or
+ * line end inside them. A plain cut leaves the last sentence half written, and
+ * the summarizer took it as a key point: "…the same hostname can range from no
+ * bot pro" was a finding in the R25 re-check (2026-10-19). Cut as before when
+ * no end falls in the second half.
+ *
+ * @param {string} text
+ * @param {number} max
+ * @returns {string}
+ */
+export function cutAtSentence(text, max) {
+  if (text.length <= max) return text;
+  const cut = text.slice(0, max);
+  const ends = [...cut.matchAll(/[.!?]["'\u201D\u2019)]*(?=\s)|\n/g)];
+  const end = ends.length > 0 ? ends[ends.length - 1].index + ends[ends.length - 1][0].length : 0;
+  return end > max / 2 ? cut.slice(0, end).trimEnd() : cut;
+}
+
 /**
  * Page text for the plain-fetch fallback, one line per block element. The
  * regex tag-strip it replaces ran every block together on one line.
@@ -896,7 +915,7 @@ export class ResearchOrchestrator extends EventEmitter {
                   const textContent = fallbackPageText(html);
                   if (textContent.length > 50) {
                     contentData = {
-                      content: textContent.slice(0, 5000),
+                      content: cutAtSentence(textContent, 5000),
                       metadata: { title: source.title || '' },
                       structuredData: {},
                       fallback: true
@@ -1425,7 +1444,7 @@ export class ResearchOrchestrator extends EventEmitter {
       try {
         if (!source.extractedContent) continue;
         
-        const content = source.extractedContent.substring(0, 5000); // Limit content length
+        const content = cutAtSentence(source.extractedContent, 5000); // Limit content length
         
         // Use summarization to extract key points
         const summary = await this.summarizeTool.execute({

package/src/core/research/claimFilters.js

@@ -40,6 +40,21 @@ const FRONT_MATTER_PATTERNS = [
   /\bperformance (?:&|and) security by\b/i
 ];
 
+// Lines that are not statements about the topic, all three key findings in the
+// R25 re-check (2026-10-19, "What anti-bot systems do major websites use in
+// 2026"): a question, which is an FAQ heading ("What makes a good anti-bot
+// solution?"); an opening hook addressed to the reader ("If you've ever
+// rage-clicked on a … puzzle, you already understand the problem with
+// CAPTCHAs."); and a scoring rubric's weight ("Detection technology
+// (signatures, ML, rules, session/event models): 25%"). Its live re-run added
+// "What you are experiencing is the cat-and-mouse game…" and the signpost
+// "Here's how Cloudflare, Akamai, Kasada, and PerimeterX each weigh…".
+const NON_STATEMENT_PATTERNS = [
+  /\?["'”’)]*\s*$/,
+  /^(?:if you(?:['’]ve| have) ever|have you ever|ever (?:wondered|tried|noticed)|imagine|picture this|let['’]s|what you(?:['’]re| are)|here['’]s (?:how|what|why|where))\b/i,
+  /^[^.!?]{2,150}:\s*\d{1,3}(?:\.\d+)?\s*%\.?$/
+];
+
 // Tokens that mark an affiliation line rather than a sentence about the topic.
 const AFFILIATION_TOKENS = new Set([
   'university', 'universities', 'institute', 'institut', 'department', 'dept',
@@ -159,10 +174,14 @@ export function properNounDensity(text) {
 // author or affiliation block rather than a sentence about the topic.
 const PROPER_NOUN_DENSITY_MAX = 0.5;
 
-/** A candidate claim is admissible when it is a verb-bearing, non-front-matter sentence. */
+/**
+ * A candidate claim is admissible when it is a verb-bearing, non-front-matter
+ * statement: not a question, a hook addressed to the reader, or a rubric weight.
+ */
 export function isAdmissibleClaim(text) {
   if (typeof text !== 'string' || text.trim().length === 0) return false;
   if (isFrontMatter(text)) return false;
+  if (NON_STATEMENT_PATTERNS.some(pattern => pattern.test(text.trim()))) return false;
   if (!hasFiniteVerb(text)) return false;
   return properNounDensity(text) <= PROPER_NOUN_DENSITY_MAX;
 }

package/src/server/inlineThreshold.js

@@ -84,8 +84,10 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   scrape_template: { textPaths: [], truncate: true },
   map_site: { textPaths: [], truncate: true, keep: ['site_map', 'statistics'] },
   // A prose answer is the text view; a structured or missing one leaves the
-  // JSON view, and the sources stay inline when they fit.
-  agent: { textPaths: ['answer'], truncate: true, keep: ['evidence', 'search_results', 'provenance'] },
+  // JSON view, and the sources stay inline when they fit. `reason` too: a
+  // degraded run's reason runs past the 200 characters kept unasked, and a
+  // degraded:true with no reason says nothing (R25 follow-up, 2026-10-20).
+  agent: { textPaths: ['answer'], truncate: true, keep: ['reason', 'evidence', 'search_results', 'provenance'] },
   // llms-full.txt is the large file and the view; llms.txt stays inline under
   // `files` when it fits. Neither key can be named in a json_path (the "."),
   // so the view is the way to read the long one.

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.22.2
+  version: 6.22.3
   source: crawlforge-mcp-server
 ---