crawlforge-mcp-server 4.2.4 → 4.2.6

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 23 web scraping, crawling, and content processing tools (5 inline + 18 advanced).
 
-**Current Version:** 4.2.2
+**Current Version:** 4.2.4
 
 ## Development Commands
 
@@ -92,8 +92,11 @@ npm run dev
 # Test MCP protocol compliance
 npm test
 
-# Unit tests (131 tests across 17 tools, no live network)
+# Unit tests (262 tests, no live network)
 npm run test:unit
+# Note: add --test-force-exit if the run appears to hang at the end — importing
+# StealthBrowserManager (d2-reliability.test.js) leaves a Playwright handle that
+# otherwise delays process exit ~100s. Tests themselves pass either way.
 
 # Integration tests
 npm run test:integration
@@ -120,7 +123,7 @@ npm run docker:prod         # Run production container
 
 ### Debugging Tips
 
-- Server logs via Winston logger (stderr for status, stdout for MCP protocol)
+- All diagnostic output (Winston logs + status banners) goes to stderr; stdout is reserved for the MCP JSON-RPC stream and CLI `--json` output (enforced in v4.2.4)
 - Set `NODE_ENV=development` for verbose logging
 - Use `--expose-gc` flag for memory profiling: `node --expose-gc server.js`
 - Check `cache/` directory for cached responses
@@ -282,4 +285,4 @@ try {
 - Each sub agent must work on their strengths; when done they report to the project manager who updates `docs/PRODUCTION_READINESS.md`
 - Whenever a phase is completed, push all changes to GitHub
 - Put all documentation md files into the `docs/` folder
-- Every time you finish a phase run `npm run build` and fix all errors before pushing
+- Every time you finish a phase run the test suites (`npm run test:unit` and `npm test`) and fix all failures before pushing

package/README.md

@@ -48,7 +48,7 @@ Add to `claude_desktop_config.json`:
   "mcpServers": {
     "crawlforge": {
       "command": "npx",
-      "args": ["crawlforge-mcp-server"]
+      "args": ["-y", "crawlforge-mcp-server"]
     }
   }
 }
@@ -71,7 +71,7 @@ The setup wizard automatically configures Claude Code by adding to `~/.claude.js
   "mcpServers": {
     "crawlforge": {
       "type": "stdio",
-      "command": "crawlforge"
+      "command": "crawlforge-mcp"
     }
   }
 }
@@ -89,7 +89,7 @@ The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.jso
   "mcpServers": {
     "crawlforge": {
       "type": "stdio",
-      "command": "crawlforge"
+      "command": "crawlforge-mcp"
     }
   }
 }
@@ -98,6 +98,8 @@ The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.jso
 Restart Cursor to activate.
 </details>
 
+> **Which launch command?** `npx -y crawlforge-mcp-server` needs no global install and always runs the published version (recommended for Claude Desktop). For a global install (`npm i -g crawlforge-mcp-server`), use the dedicated `crawlforge-mcp` bin — it resolves on your `PATH`, so it survives Node/nvm version switches. The bare `crawlforge` command still launches the server when an MCP client spawns it over stdio (backward compatibility for configs created before v4.2.5); interactively it's the CLI — run `crawlforge mcp` to start the server by hand.
+
 ## 📊 Available Tools
 
 ### Basic Tools (1 credit each)

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "4.2.4",
+  "version": "4.2.6",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 23 web scraping, crawling, and content processing tools. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. v4.0 adds Markdown-first output, pre-built site templates, Camoufox stealth engine, and cost transparency.",
   "main": "server.js",
   "bin": {
     "crawlforge": "src/cli/index.js",
-    "crawlforge-setup": "setup.js"
+    "crawlforge-setup": "setup.js",
+    "crawlforge-mcp-server": "src/cli/index.js",
+    "crawlforge-mcp": "server.js"
   },
   "scripts": {
     "start": "node server.js",

package/server.js

@@ -96,7 +96,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "4.2.2",
+  version: "4.2.6",
   description: "Production-ready MCP server with 23 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, change tracking, and local-LLM extraction via Ollama.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png"

package/setup.js

@@ -38,9 +38,16 @@ function addToMcpConfig(configPath, clientName, apiKey) {
       config.mcpServers = {};
     }
 
-    // Check if crawlforge is already configured with the correct API key
+    // Check if crawlforge is already configured with the correct API key AND the
+    // current launch command. The `crawlforge-mcp` bin (added in v4.2.5) is the
+    // dedicated MCP server launcher; older configs used the bare `crawlforge` bin,
+    // which became the CLI in v4.1.0 — re-running setup migrates them.
     const existingConfig = config.mcpServers.crawlforge;
-    if (existingConfig && existingConfig.env?.CRAWLFORGE_API_KEY === apiKey) {
+    if (
+      existingConfig &&
+      existingConfig.env?.CRAWLFORGE_API_KEY === apiKey &&
+      existingConfig.command === 'crawlforge-mcp'
+    ) {
       return {
         success: true,
         message: `CrawlForge already configured in ${clientName}`,
@@ -48,10 +55,13 @@ function addToMcpConfig(configPath, clientName, apiKey) {
       };
     }
 
-    // Add or update crawlforge MCP server configuration with API key
+    // Add or update crawlforge MCP server configuration with API key.
+    // `crawlforge-mcp` is the dedicated stdio MCP-server bin (PATH-resolved, so it
+    // survives Node/nvm version switches). The bare `crawlforge` bin also still
+    // launches the server when spawned over stdio, for backward compatibility.
     config.mcpServers.crawlforge = {
       type: "stdio",
-      command: "crawlforge",
+      command: "crawlforge-mcp",
       args: [],
       env: {
         CRAWLFORGE_API_KEY: apiKey
@@ -253,7 +263,7 @@ async function main() {
       console.log('     "mcpServers": {');
       console.log('       "crawlforge": {');
       console.log('         "type": "stdio",');
-      console.log('         "command": "crawlforge",');
+      console.log('         "command": "crawlforge-mcp",');
       console.log('         "env": {');
       console.log(`           "CRAWLFORGE_API_KEY": "${apiKey.trim()}"`);
       console.log('         }');
@@ -266,8 +276,8 @@ async function main() {
     console.log('────────────────────────────────────────────────────────');
     console.log('');
     console.log('Quick start:');
-    console.log('  crawlforge             # Start the MCP server');
-    console.log('  npm run test           # Test your setup');
+    console.log('  crawlforge-mcp         # Start the MCP server (stdio)');
+    console.log('  crawlforge --help      # Explore the CLI commands');
     console.log('');
     console.log('Need help? Visit: https://www.crawlforge.dev/docs');
     console.log('');

package/src/cli/index.js

@@ -46,6 +46,25 @@ import { register as registerMonitor } from './commands/monitor.js';
 import { register as registerInstallSkills } from './commands/install-skills.js';
 import { register as registerUninstallSkills } from './commands/uninstall-skills.js';
 
+// ─── MCP stdio server mode (backward compatibility) ──────────────────────────
+// Before v4.1.0 the `crawlforge` bin WAS the MCP server. v4.1.0 turned it into
+// this CLI, which silently broke MCP clients still configured with
+// `command: "crawlforge"` — they received CLI help text instead of a JSON-RPC
+// stream, surfacing as a -32000 connect error. Detect that case and hand off to
+// the MCP server so existing configs keep working with no edits:
+//   • explicit:  `crawlforge mcp` / `crawlforge serve`  (registered below)
+//   • explicit:  CRAWLFORGE_MCP_STDIO=true
+//   • implicit:  no subcommand AND stdin is not a TTY (i.e. spawned by an MCP host)
+// Escape hatch: CRAWLFORGE_FORCE_CLI=true forces CLI help even over a pipe.
+const __mcpImplicit =
+  process.argv.slice(2).length === 0 &&
+  !process.stdin.isTTY &&
+  process.env.CRAWLFORGE_FORCE_CLI !== 'true';
+
+if (process.env.CRAWLFORGE_MCP_STDIO === 'true' || __mcpImplicit) {
+  await import('../../server.js');
+} else {
+
 const program = new Command();
 
 program
@@ -105,7 +124,20 @@ registerMonitor(program);
 registerInstallSkills(program);
 registerUninstallSkills(program);
 
+// `crawlforge mcp` / `crawlforge serve` — explicitly start the MCP server over
+// stdio. Extra args (e.g. --http) are read directly by server.js from argv.
+program
+  .command('mcp')
+  .alias('serve')
+  .description('Start the MCP server over stdio (for MCP clients like Claude Code, Claude Desktop, Cursor)')
+  .allowUnknownOption(true)
+  .action(async () => {
+    await import('../../server.js');
+  });
+
 program.parseAsync(process.argv).catch((err) => {
   process.stderr.write(`Fatal error: ${err.message}\n`);
   process.exit(1);
 });
+
+}