crawlforge-mcp-server 5.0.1 → 5.0.2

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.1
+**Current Version:** 5.0.2
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.1",
+  "version": "5.0.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -99,7 +99,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.0.1",
+  version: "5.0.2",
   description: "Production-ready MCP server with 27 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/LLMsTxtAnalyzer.js

@@ -228,7 +228,9 @@ export class LLMsTxtAnalyzer {
         $('a[href*="api"], a[href*="developer"], a[href*="docs"]').each((_, element) => {
           const href = $(element).attr('href');
           const text = $(element).text().toLowerCase();
-          if (href && (text.includes('api') || text.includes('developer'))) {
+          // Word-boundary match: substring checks flagged "Sapiens"/"rapid"
+          // style words as API links.
+          if (href && /\b(api|developer)s?\b/.test(text)) {
             apis.push({
               url: new URL(href, baseUrl).toString(),
               type: 'documentation',

package/src/core/analysis/ContentAnalyzer.js

@@ -546,7 +546,9 @@ export class ContentAnalyzer {
       const people = doc.people().out('array');
       const places = doc.places().out('array');
       const organizations = doc.organizations().out('array');
-      const dates = doc.dates().out('array');
+      // .dates() needs the compromise-dates plugin (not installed) and threw,
+      // aborting ALL entity extraction; #Date+ tag matching is core compromise.
+      const dates = doc.match('#Date+').out('array');
       const money = doc.money().out('array');
       let other = doc.topics().out('array').slice(0, 10);
 

package/src/tools/extract/extractStructured.js

@@ -28,7 +28,8 @@ const SEMANTIC_FIELD_SELECTORS = {
   summary: ['article p', 'main p', 'p'],
   author: ['[rel="author"]', '.author', '.byline'],
   date: ['time', '.date'],
-  published: ['time', '.published', '.date']
+  published: ['time', '.published', '.date'],
+  price: ['[itemprop="price"]', '[class*="price"]']
 };
 
 const ExtractStructuredSchema = z.object({

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -225,6 +225,14 @@ export class GenerateLLMsTxtTool {
     emitSection('Tools', flatten('tools'));
     emitSection('Navigation', flatten('navigation'));
 
+    // Fallback: if no categorized section produced output, list the raw
+    // sitemap so llms.txt always carries a URL inventory. Must run BEFORE the
+    // APIs section — an APIs entry alone used to set hasBody and suppress it.
+    const hasBody = lines.some((l) => l.startsWith('## '));
+    if (!hasBody) {
+      emitSection('Pages', analysis.structure?.sitemap || []);
+    }
+
     // APIs as their own section.
     if (Array.isArray(analysis.apis) && analysis.apis.length > 0) {
       lines.push('## APIs');
@@ -236,12 +244,6 @@ export class GenerateLLMsTxtTool {
       lines.push('');
     }
 
-    // Fallback: if no categorized sections produced output, list the raw sitemap.
-    const hasBody = lines.some((l) => l.startsWith('## '));
-    if (!hasBody) {
-      emitSection('Pages', analysis.structure?.sitemap || []);
-    }
-
     return lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
   }
 

package/src/tools/templates/TemplateRegistry.js

@@ -87,9 +87,13 @@ const TEMPLATES = [
         description: attr($, 'meta[property="og:description"]', 'content') || text($, 'p.f4.my-3'),
         stars: text($, '#repo-stars-counter-star') || text($, '[aria-label*="stargazers"]'),
         forks: text($, '#repo-network-counter') || text($, '[aria-label*="forks"]'),
-        watchers: text($, '[aria-label*="watchers"]'),
+        // React (logged-out) layout has no watchers aria-label; the count is
+        // the <strong> right after the single octicon-eye. Language is a
+        // client-side skeleton on that layout — unrecoverable from static
+        // HTML, so it stays null there (itemprop still works on classic).
+        watchers: text($, '.octicon-eye + strong') || text($, '[aria-label*="watchers"]'),
         language: text($, 'span[itemprop="programmingLanguage"]') || text($, '.d-inline-flex[class*="language"]'),
-        topics: list($, 'a.topic-tag'),
+        topics: list($, 'a.topic-tag, a[href^="/topics/"]'),
         license: text($, 'a[href*="blob/"][href*="LICENSE"]') || text($, '.octicon-law ~ span'),
         last_updated: attr($, 'relative-time', 'datetime'),
         homepage: attr($, 'a[href][rel="noopener noreferrer"]', 'href'),