crawlforge-mcp-server 5.0.0 → 5.0.2

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.0
+**Current Version:** 5.0.2
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.0",
+  "version": "5.0.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -99,7 +99,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.0.0",
+  version: "5.0.2",
   description: "Production-ready MCP server with 27 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/cli/commands/init.js

@@ -4,6 +4,7 @@
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { install, installHook } from '../../skills/installer.js';
+import { isCreatorModeVerified } from '../../core/creatorMode.js';
 
 const HOME = process.env.HOME || process.env.USERPROFILE || '';
 
@@ -16,8 +17,8 @@ function loadStoredApiKey() {
   }
 }
 
-function mcpStanza(apiKey) {
-  const stanza = { command: 'npx', args: ['-y', 'crawlforge@latest', 'mcp'] };
+export function mcpStanza(apiKey) {
+  const stanza = { command: 'npx', args: ['-y', 'crawlforge-mcp-server@latest', 'mcp'] };
   if (apiKey) stanza.env = { CRAWLFORGE_API_KEY: apiKey };
   return stanza;
 }
@@ -66,15 +67,20 @@ export function register(program) {
     .action(async (opts) => {
       const out = (msg) => process.stderr.write(msg + '\n');
 
-      // 1. API key check
+      // 1. API key check (creator mode proceeds keyless — the registered
+      // server re-derives creator mode from its own environment, and the
+      // secret must never be written into client configs)
       const apiKey = loadStoredApiKey() || process.env.CRAWLFORGE_API_KEY;
-      if (!apiKey) {
+      if (apiKey) {
+        out('API key: found (' + apiKey.slice(0, 8) + '...)');
+      } else if (isCreatorModeVerified()) {
+        out('API key: none (creator mode active — proceeding without a key)');
+      } else {
         out('No CrawlForge API key found.');
         out('Run: npx crawlforge-setup');
         out('Then re-run: crawlforge init');
         process.exit(1);
       }
-      out('API key: found (' + apiKey.slice(0, 8) + '...)');
 
       // 2. Install skills
       const skillTarget = opts.all ? 'all' : 'claude-code';

package/src/cli/commands/stealth.js

@@ -9,7 +9,7 @@ export function register(program) {
   program
     .command('stealth <url>')
     .description('Scrape a URL using stealth/anti-bot browser mode')
-    .option('--engine <engine>', 'Browser engine: playwright or camoufox', 'playwright')
+    .option('--engine <engine>', 'Browser engine: chromium or camoufox', 'chromium')
     .option('--wait <ms>', 'Wait time after page load in milliseconds', '2000')
     .option('--screenshot', 'Capture a screenshot')
     .action(async (url, opts, cmd) => {
@@ -21,7 +21,8 @@ export function register(program) {
       };
       await runTool(wrapperTool, {
         url,
-        engine: opts.engine,
+        // "playwright" was the pre-v4.0.0 name for the chromium engine
+        engine: opts.engine === 'playwright' ? 'chromium' : opts.engine,
         wait_for: parseInt(opts.wait, 10),
         screenshot: !!opts.screenshot
       }, cliFlags);

package/src/core/LLMsTxtAnalyzer.js

@@ -228,7 +228,9 @@ export class LLMsTxtAnalyzer {
         $('a[href*="api"], a[href*="developer"], a[href*="docs"]').each((_, element) => {
           const href = $(element).attr('href');
           const text = $(element).text().toLowerCase();
-          if (href && (text.includes('api') || text.includes('developer'))) {
+          // Word-boundary match: substring checks flagged "Sapiens"/"rapid"
+          // style words as API links.
+          if (href && /\b(api|developer)s?\b/.test(text)) {
             apis.push({
               url: new URL(href, baseUrl).toString(),
               type: 'documentation',

package/src/core/MonitorScheduler.js

@@ -111,6 +111,7 @@ export class MonitorScheduler {
   }
 
   async stopMonitor(id) {
+    if (!this.store._loaded) await this.store.load();
     this._clearTimer(id);
     const existed = !!this.store.get(id);
     await this.store.remove(id);
@@ -118,6 +119,7 @@ export class MonitorScheduler {
   }
 
   async stopByUrl(url) {
+    if (!this.store._loaded) await this.store.load();
     let count = 0;
     for (const def of this.store.list()) {
       if (def.url === url) {
@@ -162,10 +164,17 @@ export class MonitorScheduler {
     const ct = this.tool.changeTracker;
     if (ct?.snapshots?.has(def.url)) return;
     try {
-      const q = await this.tool.snapshotManager.querySnapshots({ url: def.url, limit: 1, includeContent: true });
-      const content = q?.snapshots?.[0]?.content;
-      if (content && typeof content === 'string') {
-        await ct.createBaseline(def.url, content, def.trackingOptions);
+      // limit > 1 skips legacy empty junk snapshots; Buffer normalization is
+      // required because querySnapshots returns stored content as a Buffer —
+      // the old string guard silently disabled rehydration after restart.
+      const q = await this.tool.snapshotManager.querySnapshots({ url: def.url, limit: 5, includeContent: true });
+      for (const snap of q?.snapshots ?? []) {
+        let content = snap?.content;
+        if (Buffer.isBuffer(content)) content = content.toString('utf8');
+        if (content && typeof content === 'string') {
+          await ct.createBaseline(def.url, content, def.trackingOptions);
+          return;
+        }
       }
     } catch {
       /* no usable snapshot — first fire will create the baseline */

package/src/core/SnapshotManager.js

@@ -84,10 +84,15 @@ export class SnapshotManager extends EventEmitter {
     // the server with a cwd the process cannot write to (e.g. '/').
     const defaultBaseDir = path.join(os.homedir(), '.crawlforge', 'snapshots');
 
+    // metadata/temp must live under the EFFECTIVE storage dir: anchoring them
+    // to defaultBaseDir when a custom storageDir is passed split content from
+    // metadata and shared one metadata dir across all custom-dir instances.
+    const baseDir = options.storageDir || defaultBaseDir;
+
     this.options = {
-      storageDir: options.storageDir || defaultBaseDir,
-      metadataDir: options.metadataDir || path.join(defaultBaseDir, 'metadata'),
-      tempDir: options.tempDir || path.join(defaultBaseDir, 'temp'),
+      storageDir: baseDir,
+      metadataDir: options.metadataDir || path.join(baseDir, 'metadata'),
+      tempDir: options.tempDir || path.join(baseDir, 'temp'),
       enableCompression: options.enableCompression !== false,
       enableDeltaStorage: options.enableDeltaStorage !== false,
       enableEncryption: options.enableEncryption || false,

package/src/core/analysis/ContentAnalyzer.js

@@ -546,7 +546,9 @@ export class ContentAnalyzer {
       const people = doc.people().out('array');
       const places = doc.places().out('array');
       const organizations = doc.organizations().out('array');
-      const dates = doc.dates().out('array');
+      // .dates() needs the compromise-dates plugin (not installed) and threw,
+      // aborting ALL entity extraction; #Date+ tag matching is core compromise.
+      const dates = doc.match('#Date+').out('array');
       const money = doc.money().out('array');
       let other = doc.topics().out('array').slice(0, 10);
 

package/src/core/crawlers/BFSCrawler.js

@@ -168,7 +168,13 @@ export class BFSCrawler {
       }
     }
 
-    // Mark as visited
+    // Mark as visited. Re-check the cap and dedupe first: the checks at the
+    // top of this task ran before the awaited robots lookup, so concurrent
+    // queue tasks may have filled the budget (or claimed this URL) since.
+    // This block is synchronous, so the cap is exact.
+    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
+      return;
+    }
     this.visited.add(normalizedUrl);
 
     try {
@@ -204,11 +210,15 @@ export class BFSCrawler {
 
       // Process links for analysis
       if (this.enableLinkAnalysis && this.linkAnalyzer && pageData.links) {
+        // Parse the page once and reuse for every link; re-parsing per link is
+        // O(links × page size) and starves the event loop on link-dense pages.
+        const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
+        const pageBodyText = $page ? $page('body').text() : '';
         for (const link of pageData.links) {
           const absoluteUrl = this.resolveUrl(link, normalizedUrl);
           if (absoluteUrl) {
             // Extract anchor text and context from link
-            const linkMetadata = this.extractLinkMetadata(link, pageData.originalHtml, normalizedUrl);
+            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
             this.linkAnalyzer.addLink(normalizedUrl, absoluteUrl, linkMetadata);
           }
         }
@@ -548,19 +558,18 @@ export class BFSCrawler {
   }
 
   /**
-   * Extract link metadata from HTML
+   * Extract link metadata from a pre-parsed page
    * @param {string} href - The href attribute value
-   * @param {string} html - Original HTML content
-   * @param {string} baseUrl - Base URL for context
+   * @param {Object} $ - Cheerio document for the page (parsed once per page)
+   * @param {string} bodyText - Pre-computed body text of the page
    * @returns {Object} Link metadata
    */
-  extractLinkMetadata(href, html, baseUrl) {
-    if (!html) return {};
+  extractLinkMetadata(href, $, bodyText = '') {
+    if (!$) return {};
 
     try {
-      const $ = load(html);
       const linkElement = $(`a[href="${href}"]`).first();
-      
+
       if (linkElement.length === 0) {
         return { href };
       }
@@ -569,10 +578,8 @@ export class BFSCrawler {
       const title = linkElement.attr('title');
       const rel = linkElement.attr('rel');
       const className = linkElement.attr('class');
-      
+
       // Get surrounding context (up to 100 characters before and after)
-      const linkHtml = linkElement.prop('outerHTML');
-      const bodyText = $('body').text();
       const linkTextIndex = bodyText.indexOf(anchorText);
       let context = '';
       

package/src/schemas/toolOutputSchemas.js

@@ -122,7 +122,7 @@ const serpRankResultShape = z.object({
   domain: z.string().optional(),
   url: z.string().nullable().optional(),
   title: z.string().nullable().optional(),
-  snippet: z.string().optional()
+  snippet: z.string().nullable().optional()
 }).passthrough();
 
 const serpRankShape = {

package/src/tools/extract/extractStructured.js

@@ -28,7 +28,8 @@ const SEMANTIC_FIELD_SELECTORS = {
   summary: ['article p', 'main p', 'p'],
   author: ['[rel="author"]', '.author', '.byline'],
   date: ['time', '.date'],
-  published: ['time', '.published', '.date']
+  published: ['time', '.published', '.date'],
+  price: ['[itemprop="price"]', '[class*="price"]']
 };
 
 const ExtractStructuredSchema = z.object({

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -70,9 +70,12 @@ export class GenerateLLMsTxtTool {
       // per-analysis state on `this.analysis`, so a shared instance would let
       // concurrent (or successive) calls cross-contaminate results.
       logger.info(`Analyzing website: ${baseUrl}`);
+      // analysisOptions must reach the constructor: the analyzer reads crawl
+      // caps (maxPages/maxDepth/…) from this.options, not the per-call arg.
       const analyzer = new LLMsTxtAnalyzer({
         timeout: this.options.timeout,
-        userAgent: this.options.userAgent
+        userAgent: this.options.userAgent,
+        ...analysisOptions
       });
       const analysis = await analyzer.analyzeWebsite(url, analysisOptions);
 
@@ -222,6 +225,14 @@ export class GenerateLLMsTxtTool {
     emitSection('Tools', flatten('tools'));
     emitSection('Navigation', flatten('navigation'));
 
+    // Fallback: if no categorized section produced output, list the raw
+    // sitemap so llms.txt always carries a URL inventory. Must run BEFORE the
+    // APIs section — an APIs entry alone used to set hasBody and suppress it.
+    const hasBody = lines.some((l) => l.startsWith('## '));
+    if (!hasBody) {
+      emitSection('Pages', analysis.structure?.sitemap || []);
+    }
+
     // APIs as their own section.
     if (Array.isArray(analysis.apis) && analysis.apis.length > 0) {
       lines.push('## APIs');
@@ -233,12 +244,6 @@ export class GenerateLLMsTxtTool {
       lines.push('');
     }
 
-    // Fallback: if no categorized sections produced output, list the raw sitemap.
-    const hasBody = lines.some((l) => l.startsWith('## '));
-    if (!hasBody) {
-      emitSection('Pages', analysis.structure?.sitemap || []);
-    }
-
     return lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
   }
 

package/src/tools/templates/TemplateRegistry.js

@@ -87,9 +87,13 @@ const TEMPLATES = [
         description: attr($, 'meta[property="og:description"]', 'content') || text($, 'p.f4.my-3'),
         stars: text($, '#repo-stars-counter-star') || text($, '[aria-label*="stargazers"]'),
         forks: text($, '#repo-network-counter') || text($, '[aria-label*="forks"]'),
-        watchers: text($, '[aria-label*="watchers"]'),
+        // React (logged-out) layout has no watchers aria-label; the count is
+        // the <strong> right after the single octicon-eye. Language is a
+        // client-side skeleton on that layout — unrecoverable from static
+        // HTML, so it stays null there (itemprop still works on classic).
+        watchers: text($, '.octicon-eye + strong') || text($, '[aria-label*="watchers"]'),
         language: text($, 'span[itemprop="programmingLanguage"]') || text($, '.d-inline-flex[class*="language"]'),
-        topics: list($, 'a.topic-tag'),
+        topics: list($, 'a.topic-tag, a[href^="/topics/"]'),
         license: text($, 'a[href*="blob/"][href*="LICENSE"]') || text($, '.octicon-law ~ span'),
         last_updated: attr($, 'relative-time', 'datetime'),
         homepage: attr($, 'a[href][rel="noopener noreferrer"]', 'href'),

package/src/tools/tracking/trackChanges/index.js

@@ -130,11 +130,16 @@ export class TrackChangesTool extends EventEmitter {
 
   _setupEventHandlers() {
     this.changeTracker.on('changeDetected', async (changeRecord) => {
-      if (changeRecord.significance !== 'none') {
+      // changeRecord.details is diff analysis and carries no page content —
+      // storing `details.current || ''` wrote an empty junk snapshot on every
+      // significant change (compareWithBaseline already snapshots the real
+      // current content). Only store if a future record ever carries content.
+      const current = changeRecord.details?.current;
+      if (changeRecord.significance !== 'none' && current) {
         try {
           await this.snapshotManager.storeSnapshot(
             changeRecord.url,
-            changeRecord.details.current || '',
+            current,
             { changes: changeRecord.details, significance: changeRecord.significance, changeType: changeRecord.changeType }
           );
         } catch (error) {
@@ -209,9 +214,34 @@ export class TrackChangesTool extends EventEmitter {
     };
   }
 
+  // Rebuild the in-memory baseline from the newest persisted snapshot, so
+  // compare works across processes (fresh CLI runs, server restarts). Same
+  // fail-soft mechanism as MonitorScheduler._ensureBaseline: no usable
+  // snapshot means compare still reports "No baseline" for a genuine first run.
+  async rehydrateBaseline(url, trackingOptions = {}) {
+    if (this.changeTracker?.snapshots?.has(url)) return;
+    try {
+      // limit > 1: existing stores contain empty junk snapshots (from the old
+      // changeDetected handler) that can tie on timestamp with the real one —
+      // take the newest snapshot that actually has content.
+      const q = await this.snapshotManager.querySnapshots({ url, limit: 5, includeContent: true });
+      for (const snap of q?.snapshots ?? []) {
+        let content = snap?.content;
+        if (Buffer.isBuffer(content)) content = content.toString('utf8');
+        if (content && typeof content === 'string') {
+          await this.changeTracker.createBaseline(url, content, trackingOptions);
+          return;
+        }
+      }
+    } catch {
+      /* no usable snapshot — caller's compare will surface "No baseline" */
+    }
+  }
+
   async compareWithBaseline(params) {
     const { url, content, html, trackingOptions, storageOptions = {}, notificationOptions } = params;
     const enableSnapshots = storageOptions.enableSnapshots !== false;
+    await this.rehydrateBaseline(url, trackingOptions);
 
     let currentContent = content || html;
     let fetchMeta = {};
@@ -362,7 +392,10 @@ export class TrackChangesTool extends EventEmitter {
     const monitorId = scheduledMonitorOptions?.monitorId;
     if (monitorId) {
       const result = await this.scheduler.stopMonitor(monitorId);
-      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: result.stopped, timestamp: Date.now() };
+      if (!result.stopped) {
+        return { success: false, operation: 'stop_scheduled_monitor', monitorId, stopped: false, error: `No scheduled monitor found with id ${monitorId}`, timestamp: Date.now() };
+      }
+      return { success: true, operation: 'stop_scheduled_monitor', monitorId, stopped: true, timestamp: Date.now() };
     }
     if (!url) throw new Error('stop_scheduled_monitor requires a url or scheduledMonitorOptions.monitorId');
     const result = await this.scheduler.stopByUrl(url);