crawlforge-mcp-server 4.2.3 → 4.2.5

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 23 web scraping, crawling, and content processing tools (5 inline + 18 advanced).
 
-**Current Version:** 4.2.2
+**Current Version:** 4.2.4
 
 ## Development Commands
 
@@ -92,8 +92,11 @@ npm run dev
 # Test MCP protocol compliance
 npm test
 
-# Unit tests (131 tests across 17 tools, no live network)
+# Unit tests (262 tests, no live network)
 npm run test:unit
+# Note: add --test-force-exit if the run appears to hang at the end — importing
+# StealthBrowserManager (d2-reliability.test.js) leaves a Playwright handle that
+# otherwise delays process exit ~100s. Tests themselves pass either way.
 
 # Integration tests
 npm run test:integration
@@ -120,7 +123,7 @@ npm run docker:prod         # Run production container
 
 ### Debugging Tips
 
-- Server logs via Winston logger (stderr for status, stdout for MCP protocol)
+- All diagnostic output (Winston logs + status banners) goes to stderr; stdout is reserved for the MCP JSON-RPC stream and CLI `--json` output (enforced in v4.2.4)
 - Set `NODE_ENV=development` for verbose logging
 - Use `--expose-gc` flag for memory profiling: `node --expose-gc server.js`
 - Check `cache/` directory for cached responses
@@ -282,4 +285,4 @@ try {
 - Each sub agent must work on their strengths; when done they report to the project manager who updates `docs/PRODUCTION_READINESS.md`
 - Whenever a phase is completed, push all changes to GitHub
 - Put all documentation md files into the `docs/` folder
-- Every time you finish a phase run `npm run build` and fix all errors before pushing
+- Every time you finish a phase run the test suites (`npm run test:unit` and `npm test`) and fix all failures before pushing

package/README.md

@@ -48,7 +48,7 @@ Add to `claude_desktop_config.json`:
   "mcpServers": {
     "crawlforge": {
       "command": "npx",
-      "args": ["crawlforge-mcp-server"]
+      "args": ["-y", "crawlforge-mcp-server"]
     }
   }
 }
@@ -71,7 +71,7 @@ The setup wizard automatically configures Claude Code by adding to `~/.claude.js
   "mcpServers": {
     "crawlforge": {
       "type": "stdio",
-      "command": "crawlforge"
+      "command": "crawlforge-mcp"
     }
   }
 }
@@ -89,7 +89,7 @@ The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.jso
   "mcpServers": {
     "crawlforge": {
       "type": "stdio",
-      "command": "crawlforge"
+      "command": "crawlforge-mcp"
     }
   }
 }
@@ -98,6 +98,8 @@ The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.jso
 Restart Cursor to activate.
 </details>
 
+> **Which launch command?** `npx -y crawlforge-mcp-server` needs no global install and always runs the published version (recommended for Claude Desktop). For a global install (`npm i -g crawlforge-mcp-server`), use the dedicated `crawlforge-mcp` bin — it resolves on your `PATH`, so it survives Node/nvm version switches. The bare `crawlforge` command still launches the server when an MCP client spawns it over stdio (backward compatibility for configs created before v4.2.5); interactively it's the CLI — run `crawlforge mcp` to start the server by hand.
+
 ## 📊 Available Tools
 
 ### Basic Tools (1 credit each)

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "4.2.3",
+  "version": "4.2.5",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 23 web scraping, crawling, and content processing tools. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. v4.0 adds Markdown-first output, pre-built site templates, Camoufox stealth engine, and cost transparency.",
   "main": "server.js",
   "bin": {
     "crawlforge": "src/cli/index.js",
-    "crawlforge-setup": "setup.js"
+    "crawlforge-setup": "setup.js",
+    "crawlforge-mcp-server": "server.js",
+    "crawlforge-mcp": "server.js"
   },
   "scripts": {
     "start": "node server.js",

package/server.js

@@ -57,7 +57,8 @@ if (!AuthManager.isAuthenticated() && !AuthManager.isCreatorMode()) {
   const apiKey = process.env.CRAWLFORGE_API_KEY;
   if (apiKey) {
     // Auto-setup if API key is provided via environment
-    console.log('🔧 Auto-configuring CrawlForge with provided API key...');
+    // Status → stderr; stdout is reserved for the MCP JSON-RPC stream.
+    console.error('🔧 Auto-configuring CrawlForge with provided API key...');
     const success = await AuthManager.runSetup(apiKey);
     if (!success) {
       console.error('❌ Failed to authenticate with provided API key');
@@ -95,7 +96,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "4.2.2",
+  version: "4.2.5",
   description: "Production-ready MCP server with 23 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, change tracking, and local-LLM extraction via Ollama.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png"

package/setup.js

@@ -38,9 +38,16 @@ function addToMcpConfig(configPath, clientName, apiKey) {
       config.mcpServers = {};
     }
 
-    // Check if crawlforge is already configured with the correct API key
+    // Check if crawlforge is already configured with the correct API key AND the
+    // current launch command. The `crawlforge-mcp` bin (added in v4.2.5) is the
+    // dedicated MCP server launcher; older configs used the bare `crawlforge` bin,
+    // which became the CLI in v4.1.0 — re-running setup migrates them.
     const existingConfig = config.mcpServers.crawlforge;
-    if (existingConfig && existingConfig.env?.CRAWLFORGE_API_KEY === apiKey) {
+    if (
+      existingConfig &&
+      existingConfig.env?.CRAWLFORGE_API_KEY === apiKey &&
+      existingConfig.command === 'crawlforge-mcp'
+    ) {
       return {
         success: true,
         message: `CrawlForge already configured in ${clientName}`,
@@ -48,10 +55,13 @@ function addToMcpConfig(configPath, clientName, apiKey) {
       };
     }
 
-    // Add or update crawlforge MCP server configuration with API key
+    // Add or update crawlforge MCP server configuration with API key.
+    // `crawlforge-mcp` is the dedicated stdio MCP-server bin (PATH-resolved, so it
+    // survives Node/nvm version switches). The bare `crawlforge` bin also still
+    // launches the server when spawned over stdio, for backward compatibility.
     config.mcpServers.crawlforge = {
       type: "stdio",
-      command: "crawlforge",
+      command: "crawlforge-mcp",
       args: [],
       env: {
         CRAWLFORGE_API_KEY: apiKey
@@ -253,7 +263,7 @@ async function main() {
       console.log('     "mcpServers": {');
       console.log('       "crawlforge": {');
       console.log('         "type": "stdio",');
-      console.log('         "command": "crawlforge",');
+      console.log('         "command": "crawlforge-mcp",');
       console.log('         "env": {');
       console.log(`           "CRAWLFORGE_API_KEY": "${apiKey.trim()}"`);
       console.log('         }');
@@ -266,8 +276,8 @@ async function main() {
     console.log('────────────────────────────────────────────────────────');
     console.log('');
     console.log('Quick start:');
-    console.log('  crawlforge             # Start the MCP server');
-    console.log('  npm run test           # Test your setup');
+    console.log('  crawlforge-mcp         # Start the MCP server (stdio)');
+    console.log('  crawlforge --help      # Explore the CLI commands');
     console.log('');
     console.log('Need help? Visit: https://www.crawlforge.dev/docs');
     console.log('');

package/src/cli/commands/analyze.js

@@ -1,6 +1,9 @@
 /**
  * analyze command — analyze content of a URL.
+ * Fetches and cleans the page content first (extract_content), then runs
+ * NLP analysis (analyze_content) on the extracted text.
  */
+import { ExtractContentTool } from '../../tools/extract/extractContent.js';
 import { AnalyzeContentTool } from '../../tools/extract/analyzeContent.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
@@ -13,7 +16,29 @@ export function register(program) {
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+
+      // analyze_content operates on text, so fetch & clean the page first.
+      const extractor = new ExtractContentTool(getToolConfig('extract_content'));
+      let text;
+      try {
+        const extracted = await extractor.execute({ url });
+        text = extracted?.content?.text;
+      } catch (e) {
+        process.stderr.write(`Error fetching content from ${url}: ${e.message}\n`);
+        process.exit(1);
+      }
+
+      if (!text || text.trim().length < 10) {
+        process.stderr.write(`Error: could not extract enough text from ${url} to analyze\n`);
+        process.exit(1);
+      }
+
       const tool = new AnalyzeContentTool(getToolConfig('analyze_content'));
-      await runTool(tool, { url, analysis_depth: opts.depth }, cliFlags);
+      // All analyses (language, topics, entities, sentiment, readability) default to true;
+      // --depth full additionally enables advanced metrics.
+      await runTool(tool, {
+        text,
+        options: { includeAdvancedMetrics: opts.depth === 'full' }
+      }, cliFlags);
     });
 }

package/src/cli/commands/localize.js

@@ -1,29 +1,64 @@
 /**
- * localize command — fetch content with locale/geo awareness.
+ * localize command — fetch a URL with locale/geo-aware request headers.
+ * Builds a localization config (Accept-Language, User-Agent) for the target
+ * country via LocalizationManager, then fetches the URL with those headers.
  */
 import { LocalizationManager } from '../../core/LocalizationManager.js';
+import { fetchUrlHandler } from '../../tools/basic/fetchUrl.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
 
+// Derive a 2-letter country code from a --country flag or an en-US style locale.
+function resolveCountry(country, locale) {
+  if (country) return country.toUpperCase();
+  if (locale && locale.includes('-')) return locale.split('-')[1].toUpperCase();
+  return 'US';
+}
+
 export function register(program) {
   program
     .command('localize <url>')
-    .description('Fetch URL with locale/geo-aware settings')
+    .description('Fetch URL with locale/geo-aware request headers')
     .option('--locale <locale>', 'Locale code (e.g. en-US, fr-FR)', 'en-US')
     .option('--country <code>', 'Country code for geo-targeting (e.g. US, FR)')
     .option('--currency <code>', 'Currency code (e.g. USD, EUR)')
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
-      const mgr = new LocalizationManager(getToolConfig('localization'));
+
+      const countryCode = resolveCountry(opts.country, opts.locale);
+      const language = opts.locale ? opts.locale.split('-')[0] : undefined;
+
       const wrapperTool = {
-        execute: (p) => mgr.fetchWithLocalization(p)
+        execute: async () => {
+          const mgr = new LocalizationManager(getToolConfig('localization'));
+          await mgr.initialize();
+          const config = await mgr.configureCountry(countryCode, {
+            language,
+            currency: opts.currency
+          });
+
+          const headers = {
+            'Accept-Language': config.acceptLanguage,
+            'User-Agent': mgr.generateUserAgent(countryCode),
+            ...(config.customHeaders || {})
+          };
+
+          const fetched = await fetchUrlHandler({ url, headers });
+          return {
+            localization: {
+              countryCode: config.countryCode,
+              language: config.language,
+              timezone: config.timezone,
+              currency: config.currency,
+              acceptLanguage: config.acceptLanguage
+            },
+            request_headers: headers,
+            response: fetched
+          };
+        }
       };
-      await runTool(wrapperTool, {
-        url,
-        locale: opts.locale,
-        country: opts.country,
-        currency: opts.currency
-      }, cliFlags);
+
+      await runTool(wrapperTool, {}, cliFlags);
     });
 }

package/src/cli/commands/monitor.js

@@ -17,6 +17,7 @@ export function register(program) {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      // monitor runs continuously — do not auto-exit after the first result.
       await runTool(tool, {
         url,
         scheduled: true,
@@ -24,6 +25,6 @@ export function register(program) {
         selector: opts.selector,
         webhook_url: opts.webhook,
         change_threshold: parseFloat(opts.threshold)
-      }, cliFlags);
+      }, cliFlags, { exitOnSuccess: false });
     });
 }

package/src/cli/commands/template.js

@@ -7,7 +7,7 @@ import { runTool } from '../lib/runTool.js';
 
 export function register(program) {
   program
-    .command('template <id> <target>')
+    .command('template [id] [target]')
     .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo)')
     .option('--list', 'List all available templates')
     .action(async (id, target, opts, cmd) => {
@@ -16,11 +16,16 @@ export function register(program) {
       const tool = new ScrapeTemplateTool(getToolConfig('scrape_template'));
 
       if (opts.list) {
-        const wrapperTool = { execute: () => tool.listTemplates() };
+        const wrapperTool = { execute: () => tool.execute({ template: 'list' }) };
         await runTool(wrapperTool, {}, cliFlags);
         return;
       }
 
-      await runTool(tool, { template_id: id, url: target }, cliFlags);
+      if (!id || !target) {
+        process.stderr.write('Error: template requires <id> and <target>, or use --list\n');
+        process.exit(1);
+      }
+
+      await runTool(tool, { template: id, url: target }, cliFlags);
     });
 }

package/src/cli/index.js

@@ -46,6 +46,25 @@ import { register as registerMonitor } from './commands/monitor.js';
 import { register as registerInstallSkills } from './commands/install-skills.js';
 import { register as registerUninstallSkills } from './commands/uninstall-skills.js';
 
+// ─── MCP stdio server mode (backward compatibility) ──────────────────────────
+// Before v4.1.0 the `crawlforge` bin WAS the MCP server. v4.1.0 turned it into
+// this CLI, which silently broke MCP clients still configured with
+// `command: "crawlforge"` — they received CLI help text instead of a JSON-RPC
+// stream, surfacing as a -32000 connect error. Detect that case and hand off to
+// the MCP server so existing configs keep working with no edits:
+//   • explicit:  `crawlforge mcp` / `crawlforge serve`  (registered below)
+//   • explicit:  CRAWLFORGE_MCP_STDIO=true
+//   • implicit:  no subcommand AND stdin is not a TTY (i.e. spawned by an MCP host)
+// Escape hatch: CRAWLFORGE_FORCE_CLI=true forces CLI help even over a pipe.
+const __mcpImplicit =
+  process.argv.slice(2).length === 0 &&
+  !process.stdin.isTTY &&
+  process.env.CRAWLFORGE_FORCE_CLI !== 'true';
+
+if (process.env.CRAWLFORGE_MCP_STDIO === 'true' || __mcpImplicit) {
+  await import('../../server.js');
+} else {
+
 const program = new Command();
 
 program
@@ -58,11 +77,28 @@ program
   .option('--api-key <key>', 'CrawlForge API key (overrides CRAWLFORGE_API_KEY env var)')
   .option('--timeout <ms>', 'Global request timeout in milliseconds', '30000');
 
+// Resolve the API key from (in priority order): --api-key flag, CRAWLFORGE_API_KEY env,
+// then the stored ~/.crawlforge/config.json written by `crawlforge-setup`.
+function loadStoredApiKey() {
+  try {
+    const home = process.env.HOME || process.env.USERPROFILE;
+    if (!home) return undefined;
+    const cfgPath = join(home, '.crawlforge', 'config.json');
+    const cfg = JSON.parse(readFileSync(cfgPath, 'utf8'));
+    return cfg.apiKey || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 // Apply --api-key globally before commands run
 program.hook('preAction', (thisCommand) => {
   const opts = program.opts();
   if (opts.apiKey) {
     process.env.CRAWLFORGE_API_KEY = opts.apiKey;
+  } else if (!process.env.CRAWLFORGE_API_KEY) {
+    const stored = loadStoredApiKey();
+    if (stored) process.env.CRAWLFORGE_API_KEY = stored;
   }
   if (opts.timeout) {
     process.env.CRAWLFORGE_CLI_TIMEOUT = opts.timeout;
@@ -88,7 +124,20 @@ registerMonitor(program);
 registerInstallSkills(program);
 registerUninstallSkills(program);
 
+// `crawlforge mcp` / `crawlforge serve` — explicitly start the MCP server over
+// stdio. Extra args (e.g. --http) are read directly by server.js from argv.
+program
+  .command('mcp')
+  .alias('serve')
+  .description('Start the MCP server over stdio (for MCP clients like Claude Code, Claude Desktop, Cursor)')
+  .allowUnknownOption(true)
+  .action(async () => {
+    await import('../../server.js');
+  });
+
 program.parseAsync(process.argv).catch((err) => {
   process.stderr.write(`Fatal error: ${err.message}\n`);
   process.exit(1);
 });
+
+}

package/src/cli/lib/runTool.js

@@ -16,9 +16,13 @@ import { formatResult, formatError } from '../formatter.js';
  * @param {object} cliFlags     — { json, pretty, quiet }
  * @param {object} [options]
  * @param {boolean} [options.exitOnError=true]
+ * @param {boolean} [options.exitOnSuccess=true]  Exit the process after writing
+ *        output. One-shot CLI commands need this because background timers
+ *        (metrics, cache/connection cleanup, etc.) otherwise keep the event loop
+ *        alive. Long-running commands (e.g. `monitor`) pass false.
  */
 export async function runTool(tool, params, cliFlags, options = {}) {
-  const { exitOnError = true } = options;
+  const { exitOnError = true, exitOnSuccess = true } = options;
 
   try {
     const result = await tool.execute(params);
@@ -32,7 +36,12 @@ export async function runTool(tool, params, cliFlags, options = {}) {
     }
 
     const output = formatResult(result, cliFlags);
-    if (output) process.stdout.write(output + '\n');
+    if (output) {
+      // Wait for stdout to flush (pipes/files buffer) before exiting.
+      process.stdout.write(output + '\n', () => { if (exitOnSuccess) process.exit(0); });
+    } else if (exitOnSuccess) {
+      process.exit(0);
+    }
   } catch (error) {
     process.stderr.write(formatError(error, cliFlags) + '\n');
     if (exitOnError) process.exit(1);

package/src/core/ActionExecutor.js

@@ -926,7 +926,8 @@ export class ActionExecutor extends EventEmitter {
    */
   log(level, message) {
     if (this.enableLogging) {
-      console.log('[ActionExecutor:' + level.toUpperCase() + '] ' + message);
+      // → stderr so stdout stays clean for MCP JSON-RPC / CLI --json output.
+      console.error('[ActionExecutor:' + level.toUpperCase() + '] ' + message);
     }
   }
 

package/src/core/AuthManager.js

@@ -69,7 +69,8 @@ class AuthManager {
 
     // Skip config loading in creator mode
     if (this.isCreatorMode()) {
-      console.log('🚀 Creator Mode Active - Unlimited Access Enabled');
+      // Status → stderr; stdout is reserved for MCP JSON-RPC / CLI --json output.
+      console.error('🚀 Creator Mode Active - Unlimited Access Enabled');
       this.initialized = true;
       return;
     }
@@ -78,7 +79,7 @@ class AuthManager {
       await this.loadConfig();
       this.initialized = true;
     } catch (error) {
-      console.log('No existing CrawlForge configuration found. Run setup to configure.');
+      console.error('No existing CrawlForge configuration found. Run setup to configure.');
       this.initialized = true;
     }
 

package/src/core/PerformanceManager.js

@@ -771,6 +771,9 @@ export class PerformanceManager extends EventEmitter {
     this.metricsTimer = setInterval(() => {
       this.collectMetrics();
     }, this.metricsInterval);
+    // Don't let the metrics interval keep a short-lived process (e.g. a one-shot
+    // CLI command) alive. The long-running server stays up via its stdio transport.
+    if (typeof this.metricsTimer.unref === 'function') this.metricsTimer.unref();
   }
 
   /**

package/src/core/creatorMode.js

@@ -29,7 +29,8 @@ if (process.env.CRAWLFORGE_CREATOR_SECRET) {
 
   if (crypto.timingSafeEqual(Buffer.from(providedHash, 'hex'), Buffer.from(CREATOR_SECRET_HASH, 'hex'))) {
     _creatorModeVerified = true;
-    console.log('Creator Mode Enabled - Unlimited Access');
+    // Status message → stderr so stdout stays clean (MCP JSON-RPC / CLI --json output).
+    console.error('Creator Mode Enabled - Unlimited Access');
   } else {
     console.warn('Invalid creator secret provided');
   }

package/src/tools/advanced/batchScrape/index.js

@@ -301,7 +301,8 @@ export class BatchScrapeTool extends EventEmitter {
   }
 
   _log(level, message) {
-    if (this.enableLogging) console.log(`[BatchScrapeTool:${level.toUpperCase()}] ${message}`);
+    // → stderr so stdout stays clean for MCP JSON-RPC / CLI --json output.
+    if (this.enableLogging) console.error(`[BatchScrapeTool:${level.toUpperCase()}] ${message}`);
   }
 
   _initializeJobExecutors() {

package/src/tools/search/adapters/searchProviderFactory.js

@@ -36,7 +36,8 @@ export class SearchProviderFactory {
         );
       }
 
-      console.log('🔍 Creator Mode: Using Google Search API directly');
+      // Status message → stderr so stdout stays clean (MCP JSON-RPC / CLI --json).
+      console.error('🔍 Creator Mode: Using Google Search API directly');
       return new GoogleSearchAdapter(googleApiKey, googleSearchEngineId);
     }
 

package/src/utils/Logger.js

@@ -116,6 +116,9 @@ export class Logger {
 
     if (enableConsole) {
       transports.push(new winston.transports.Console({
+        // Route ALL log levels to stderr so stdout stays reserved for structured
+        // output (MCP JSON-RPC protocol and CLI --json results).
+        stderrLevels: ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly'],
         format: winston.format.combine(
           winston.format.colorize(),
           winston.format.simple()