crawlforge-mcp-server 6.23.0 → 6.23.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.23.0",
+  "version": "6.23.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,14 +114,14 @@
   ],
   "dependencies": {
     "@duckduckgo/autoconsent": "^16.43.0",
-    "@modelcontextprotocol/core": "^2.0.0",
-    "@modelcontextprotocol/node": "^2.0.0",
-    "@modelcontextprotocol/server": "^2.0.0",
+    "@modelcontextprotocol/core": "^2.3.1",
+    "@modelcontextprotocol/node": "^2.1.1",
+    "@modelcontextprotocol/server": "^2.3.1",
     "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.18.2",
+    "crawlforge-extractors": "^1.18.3",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
@@ -145,7 +145,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^30.3.0",
-    "@modelcontextprotocol/client": "^2.0.0",
+    "@modelcontextprotocol/client": "^2.3.1",
     "c8": "^11.0.0",
     "cross-env": "^10.0.0",
     "jest": "^30.3.0"

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.23.0",
+  version: "6.23.2",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/LLMsTxtAnalyzer.js

@@ -276,7 +276,10 @@ export class LLMsTxtAnalyzer {
             apis.push({
               url: new URL(href, baseUrl).toString(),
               type: 'documentation',
-              description: text.trim()
+              description: text.trim(),
+              // Link text as written, the label when the page itself was not
+              // classified (live sweep 7.4: it was labelled by its slug).
+              title: $(element).text().replace(/\s+/g, ' ').trim()
             });
           }
         });

package/src/core/StealthBrowserManager.js

@@ -279,7 +279,7 @@ export async function resolveStealthEngine(requested) {
     return { engine: 'chromium', fallbackWarning: null };
   }
 
-  let reason = 'camoufox is not installed (npm install camoufox)';
+  let reason = 'camoufox is not installed (npm install camoufox, then npx camoufox fetch)';
   try {
     if (await new CamoufoxAdapter().isAvailable()) {
       return { engine: 'camoufox', fallbackWarning: null };
@@ -606,7 +606,7 @@ export class StealthBrowserManager {
       const available = await adapter.isAvailable();
       if (!available) {
         throw new Error(
-          'camoufox is not installed. Run: npm install camoufox to use the Firefox-based stealth engine.'
+          'camoufox is not installed. Run: npm install camoufox, then npx camoufox fetch, to use the Firefox-based stealth engine.'
         );
       }
       // camoufox fixes its fingerprint — and, with geoip, its geolocation,
@@ -3612,7 +3612,15 @@ export class CamoufoxAdapter extends BrowserEngine {
 
   async isAvailable() {
     try {
-      await this._load();
+      const camoufox = await this._load();
+      // The npm package ships without the Firefox binary, which comes from a
+      // separate `npx camoufox fetch`. getLaunchPath() throws when it is
+      // missing, and "auto" must then pick Chromium, not fail at launch.
+      try {
+        camoufox.getLaunchPath();
+      } catch {
+        return false;
+      }
       return true;
     } catch (err) {
       if (this._isNotInstalled(err)) {

package/src/core/browser/guardedProxy.js

@@ -38,7 +38,7 @@ import tls from 'node:tls';
 import { config } from '../../constants/config.js';
 import { assertUrlAllowed, isSsrfError, ssrfLookup } from '../../utils/ssrfGuard.js';
 
-/** upstream key → Promise<{ server: string }> */
+/** upstream key → Promise<{ server: string, bypass: string }> */
 const servers = new Map();
 
 const hostForUrl = (host) => (net.isIPv6(host) ? `[${host}]` : host);
@@ -261,7 +261,12 @@ function upstreamUrl(proxy) {
  * proxy, chained to `upstream` when one is given.
  * @param {{ server: string, username?: string, password?: string }|null} [upstream]
  *   the proxy the context would otherwise have used, in Playwright's shape
- * @returns {Promise<{ server: string }>}
+ * @returns {Promise<{ server: string, bypass: string }>} `bypass` is Chromium's
+ *   `<-loopback>` (send loopback through the proxy), stated here rather than
+ *   left to Playwright: Playwright adds it only while
+ *   PLAYWRIGHT_DISABLE_FORCED_CHROMIUM_PROXIED_LOOPBACK is unset, and with that
+ *   set every Chromium request to 127.0.0.1 skipped this proxy (measured
+ *   2026-10-21). Named in the list, it is applied whatever the env says.
  */
 export function guardedProxyFor(upstream = null) {
   const up = upstreamUrl(upstream);
@@ -274,7 +279,7 @@ export function guardedProxyFor(upstream = null) {
       server.once('error', reject);
       server.listen(0, '127.0.0.1', () => {
         server.unref();
-        resolve({ server: `http://127.0.0.1:${server.address().port}` });
+        resolve({ server: `http://127.0.0.1:${server.address().port}`, bypass: '<-loopback>' });
       });
     });
     started.catch(() => servers.delete(key));

package/src/schemas/toolOutputSchemas.js

@@ -423,6 +423,7 @@ const crawlDeepShape = {
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
   redaction: redactionShape,
+  warnings: z.array(z.string()).optional().describe('Notes on this result; over max_inline_chars, where the full result is kept and how to read it'),
   ...resultHandleShape,
   _cost: costShape,
   next_step: nextStepShape

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.23.0
+  version: 6.23.2
   source: crawlforge-mcp-server
 ---
 

package/src/tools/extract/processDocument.js

@@ -187,15 +187,22 @@ const MAX_TITLE_CHARS = 200;
 /**
  * A title for a document with none in its metadata: the first non-empty line
  * of its text when that line is short enough to be one, else null. arXiv
- * stamps and page numbers before it are skipped.
+ * stamps, page numbers and lines of a prose paragraph before it are skipped:
+ * a line ending a sentence, or one the next line continues in lower case
+ * (the arXiv "Attention Is All You Need" PDF opens with a permission notice,
+ * live sweep 7.3).
  * @param {string} text
  * @returns {string|null}
  */
 export function titleFromText(text) {
-  for (const line of String(text || '').split(/\r?\n/)) {
-    const candidate = line.replace(/\s+/g, ' ').trim();
-    if (candidate === '' || NON_TITLE_LINE_RE.test(candidate)) continue;
-    return candidate.length <= MAX_TITLE_CHARS ? candidate : null;
+  const lines = String(text || '')
+    .split(/\r?\n/)
+    .map((line) => line.replace(/\s+/g, ' ').trim())
+    .filter((line) => line !== '' && !NON_TITLE_LINE_RE.test(line));
+  for (const [i, candidate] of lines.entries()) {
+    if (candidate.length > MAX_TITLE_CHARS) return null;
+    const isProse = /[.!?]$/.test(candidate) || /^\p{Ll}/u.test(lines[i + 1] || '');
+    if (!isProse) return candidate;
   }
   return null;
 }

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -218,18 +218,21 @@ export class GenerateLLMsTxtTool {
       listed.add(key);
       return true;
     };
-    const emitSection = (heading, urls) => {
-      // Coerce to an array: sitemap/sections may arrive as a flat array, a
-      // grouped object ({path: [...]}), or a single value.
+    // Coerce to a URL array: sitemap/sections may arrive as a flat array, a
+    // grouped object ({path: [...]}), or a single value.
+    const toUrlList = (urls) => {
       let arr = [];
       if (Array.isArray(urls)) {
         arr = urls;
       } else if (urls && typeof urls === 'object') {
         arr = Object.values(urls).flat();
       }
-      const list = arr
+      return arr
         .map((u) => (typeof u === 'string' ? u : (u?.url || u?.loc)))
-        .filter(Boolean)
+        .filter(Boolean);
+    };
+    const emitSection = (heading, urls) => {
+      const list = toUrlList(urls)
         .filter(firstListing)
         .slice(0, 25);
       if (list.length === 0) return;
@@ -262,19 +265,22 @@ export class GenerateLLMsTxtTool {
     // listed only when no named section matched at all, which silently dropped
     // the homepage and whole sections (e.g. /specification) from sites that
     // happen to also have a /docs.
-    emitSection('Pages', flatten('other'));
+    // Only endpoints observed answering JSON/XML are APIs. Links to API
+    // documentation are HTML pages ("Python's C API" on docs.python.org was
+    // listed under APIs, live sweep 7.4), so they are listed as pages.
+    const allApis = (Array.isArray(analysis.apis) ? analysis.apis : []).filter((api) => api?.url);
+    const isMachineReadable = (api) => /json|xml/i.test(api.contentType || '');
+    const docLinks = allApis.filter((api) => !isMachineReadable(api)).map((api) => api.url);
 
     // Fallback: if no categorized section produced output, list the raw
-    // sitemap so llms.txt always carries a URL inventory. Must run BEFORE the
-    // APIs section — an APIs entry alone used to set hasBody and suppress it.
-    const hasBody = lines.some((l) => l.startsWith('## '));
-    if (!hasBody) {
-      emitSection('Pages', analysis.structure?.sitemap || []);
-    }
+    // sitemap so llms.txt always carries a URL inventory. Decided BEFORE the
+    // linked pages and APIs are added — those alone used to suppress it.
+    const pageUrls = flatten('other');
+    const hasBody = lines.some((l) => l.startsWith('## ')) || pageUrls.length > 0;
+    emitSection('Pages', [...(hasBody ? pageUrls : toUrlList(analysis.structure?.sitemap)), ...docLinks]);
 
     // APIs as their own section.
-    const apis = (Array.isArray(analysis.apis) ? analysis.apis : [])
-      .filter((api) => api?.url && firstListing(api.url));
+    const apis = allApis.filter((api) => isMachineReadable(api) && firstListing(api.url));
     if (apis.length > 0) {
       lines.push('## APIs');
       lines.push('');
@@ -310,6 +316,11 @@ export class GenerateLLMsTxtTool {
       counts.set(clean, (counts.get(clean) || 0) + 1);
     };
 
+    // Link text of API documentation links first: a title captured from the
+    // page itself replaces it below.
+    for (const api of Array.isArray(analysis.apis) ? analysis.apis : []) {
+      if (api?.type === 'documentation') add(api.url, api.title);
+    }
     for (const entries of Object.values(analysis.contentTypes || {})) {
       if (!Array.isArray(entries)) continue;
       for (const entry of entries) {

package/src/utils/ssrfGuard.js

@@ -342,7 +342,8 @@ export async function safeFetch(url, options = {}) {
     return await fetchResigned(url, { ...options, ...guard });
   } catch (err) {
     if (isSsrfError(err)) {
-      throw new Error(err.cause?.message || err.message);
+      // Keep the code: callers tell a refusal from a fetch failure by it.
+      throw Object.assign(new Error(err.cause?.message || err.message), { code: 'SSRF_BLOCKED' });
     }
     throw withFetchCause(err, url);
   }