crawlforge-mcp-server 3.0.9 → 3.0.11

package/README.md

@@ -32,11 +32,12 @@ npx crawlforge-setup
 This will:
 - Guide you through getting your free API key
 - Configure your credentials securely
+- **Auto-configure Claude Code and Cursor** (if installed)
 - Verify your setup is working
 
 **Don't have an API key?** Get one free at [https://www.crawlforge.dev/signup](https://www.crawlforge.dev/signup)
 
-### 3. Configure Your IDE
+### 3. Configure Your IDE (if not auto-configured)
 
 <details>
 <summary>🤖 For Claude Desktop</summary>
@@ -62,17 +63,39 @@ Restart Claude Desktop to activate.
 </details>
 
 <details>
-<summary>💻 For Cursor IDE</summary>
+<summary>🖥️ For Claude Code CLI (Auto-configured)</summary>
 
-Add to `.cursorrules` in your project:
-```bash
-mcp_servers:
-  crawlforge:
-    command: npx
-    args: ["crawlforge-mcp-server"]
+The setup wizard automatically configures Claude Code by adding to `~/.claude.json`:
+```json
+{
+  "mcpServers": {
+    "crawlforge": {
+      "type": "stdio",
+      "command": "crawlforge"
+    }
+  }
+}
+```
+
+After setup, restart Claude Code to activate.
+</details>
+
+<details>
+<summary>💻 For Cursor IDE (Auto-configured)</summary>
+
+The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.json`:
+```json
+{
+  "mcpServers": {
+    "crawlforge": {
+      "type": "stdio",
+      "command": "crawlforge"
+    }
+  }
+}
 ```
 
-Or use the MCP plugin in Cursor settings.
+Restart Cursor to activate.
 </details>
 
 ## 📊 Available Tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "3.0.9",
+  "version": "3.0.11",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 19 comprehensive web scraping, crawling, and content processing tools.",
   "main": "server.js",
   "bin": {

package/server.js

@@ -97,7 +97,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 }
 
 // Create the server
-const server = new McpServer({ name: "crawlforge", version: "3.0.4" });
+const server = new McpServer({ name: "crawlforge", version: "3.0.10" });
 
 // Helper function to wrap tool handlers with authentication and credit tracking
 function withAuth(toolName, handler) {

package/setup.js

@@ -6,16 +6,143 @@
  */
 
 import readline from 'readline';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import AuthManager from './src/core/AuthManager.js';
 
-const rl = readline.createInterface({
-  input: process.stdin,
-  output: process.stdout
-});
+/**
+ * Add CrawlForge to an MCP client configuration file
+ * @param {string} configPath - Path to the config file
+ * @param {string} clientName - Name of the client (for messages)
+ * @param {string} apiKey - The CrawlForge API key to include in env
+ * @returns {object} Result with success status and message
+ */
+function addToMcpConfig(configPath, clientName, apiKey) {
+  // Check if config exists
+  if (!fs.existsSync(configPath)) {
+    return {
+      success: false,
+      message: `${clientName} config not found (${configPath}). You may need to configure manually.`,
+      notInstalled: true
+    };
+  }
+
+  try {
+    // Read existing config
+    const configContent = fs.readFileSync(configPath, 'utf8');
+    const config = JSON.parse(configContent);
+
+    // Ensure mcpServers object exists
+    if (!config.mcpServers) {
+      config.mcpServers = {};
+    }
+
+    // Check if crawlforge is already configured with the correct API key
+    const existingConfig = config.mcpServers.crawlforge;
+    if (existingConfig && existingConfig.env?.CRAWLFORGE_API_KEY === apiKey) {
+      return {
+        success: true,
+        message: `CrawlForge already configured in ${clientName}`,
+        alreadyConfigured: true
+      };
+    }
+
+    // Add or update crawlforge MCP server configuration with API key
+    config.mcpServers.crawlforge = {
+      type: "stdio",
+      command: "crawlforge",
+      args: [],
+      env: {
+        CRAWLFORGE_API_KEY: apiKey
+      }
+    };
+
+    // Write updated config
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+
+    return {
+      success: true,
+      message: existingConfig
+        ? `CrawlForge API key updated in ${clientName} MCP config`
+        : `CrawlForge added to ${clientName} MCP config`
+    };
+  } catch (error) {
+    return {
+      success: false,
+      message: `Failed to update ${clientName} config: ${error.message}`
+    };
+  }
+}
+
+/**
+ * Configure all detected MCP clients
+ * @param {string} apiKey - The CrawlForge API key to include in env
+ * @returns {object} Results for each client
+ */
+function configureMcpClients(apiKey) {
+  const results = {
+    claudeCode: null,
+    cursor: null
+  };
+
+  // Claude Code config path
+  const claudeConfigPath = path.join(os.homedir(), '.claude.json');
+  results.claudeCode = addToMcpConfig(claudeConfigPath, 'Claude Code', apiKey);
+
+  // Cursor config path (macOS)
+  const cursorConfigPath = path.join(os.homedir(), '.cursor', 'mcp.json');
+  if (fs.existsSync(path.join(os.homedir(), '.cursor'))) {
+    // Cursor directory exists, try to configure
+    if (!fs.existsSync(cursorConfigPath)) {
+      // Create mcp.json if it doesn't exist
+      try {
+        fs.writeFileSync(cursorConfigPath, JSON.stringify({ mcpServers: {} }, null, 2));
+      } catch (e) {
+        // Ignore creation errors
+      }
+    }
+    results.cursor = addToMcpConfig(cursorConfigPath, 'Cursor', apiKey);
+  }
+
+  return results;
+}
+
+let rl = null;
+
+function getReadline() {
+  if (!rl) {
+    rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout
+    });
+  }
+  return rl;
+}
 
-const question = (query) => new Promise((resolve) => rl.question(query, resolve));
+const question = (query) => new Promise((resolve) => getReadline().question(query, resolve));
+
+function closeReadline() {
+  if (rl) {
+    rl.close();
+    rl = null;
+  }
+}
 
 async function main() {
+  // Check if running interactively
+  const isInteractive = process.stdin.isTTY;
+
+  if (!isInteractive) {
+    console.log('');
+    console.log('❌ Setup requires an interactive terminal.');
+    console.log('');
+    console.log('Please run this command directly in your terminal:');
+    console.log('  npx crawlforge-setup');
+    console.log('');
+    process.exit(1);
+  }
+
   console.log('');
   console.log('╔═══════════════════════════════════════════════════════╗');
   console.log('║           CrawlForge MCP Server Setup Wizard          ║');
@@ -45,7 +172,7 @@ async function main() {
     
     if (overwrite.toLowerCase() !== 'y') {
       console.log('Setup cancelled.');
-      rl.close();
+      closeReadline();
       process.exit(0);
     }
     console.log('');
@@ -58,7 +185,7 @@ async function main() {
     console.log('');
     console.log('❌ API key is required');
     console.log('Get your free API key at: https://www.crawlforge.dev/signup');
-    rl.close();
+    closeReadline();
     process.exit(1);
   }
 
@@ -74,8 +201,72 @@ async function main() {
     console.log('');
     console.log('🎉 Setup complete! You can now use CrawlForge MCP Server.');
     console.log('');
+
+    // Auto-configure MCP clients (Claude Code, Cursor) with API key
+    console.log('🔧 Configuring MCP client integrations...');
+    const clientResults = configureMcpClients(apiKey.trim());
+    let anyConfigured = false;
+    let needsRestart = false;
+
+    // Report Claude Code status
+    if (clientResults.claudeCode) {
+      if (clientResults.claudeCode.success) {
+        anyConfigured = true;
+        if (clientResults.claudeCode.alreadyConfigured) {
+          console.log('✅ Claude Code: Already configured');
+        } else {
+          console.log('✅ Claude Code: Added to MCP config (~/.claude.json)');
+          needsRestart = true;
+        }
+      } else if (!clientResults.claudeCode.notInstalled) {
+        console.log(`⚠️  Claude Code: ${clientResults.claudeCode.message}`);
+      }
+    }
+
+    // Report Cursor status
+    if (clientResults.cursor) {
+      if (clientResults.cursor.success) {
+        anyConfigured = true;
+        if (clientResults.cursor.alreadyConfigured) {
+          console.log('✅ Cursor: Already configured');
+        } else {
+          console.log('✅ Cursor: Added to MCP config (~/.cursor/mcp.json)');
+          needsRestart = true;
+        }
+      } else if (!clientResults.cursor.notInstalled) {
+        console.log(`⚠️  Cursor: ${clientResults.cursor.message}`);
+      }
+    }
+
+    // Show restart warning if any client was configured
+    if (needsRestart) {
+      console.log('');
+      console.log('⚠️  IMPORTANT: Restart your IDE to load the new MCP server');
+    }
+
+    // Show manual config instructions if no clients detected
+    if (!anyConfigured && clientResults.claudeCode?.notInstalled && !clientResults.cursor) {
+      console.log('ℹ️  No MCP clients detected. Manual configuration needed:');
+      console.log('');
+      console.log('   Add this to your MCP client config:');
+      console.log('   {');
+      console.log('     "mcpServers": {');
+      console.log('       "crawlforge": {');
+      console.log('         "type": "stdio",');
+      console.log('         "command": "crawlforge",');
+      console.log('         "env": {');
+      console.log(`           "CRAWLFORGE_API_KEY": "${apiKey.trim()}"`);
+      console.log('         }');
+      console.log('       }');
+      console.log('     }');
+      console.log('   }');
+    }
+
+    console.log('');
+    console.log('────────────────────────────────────────────────────────');
+    console.log('');
     console.log('Quick start:');
-    console.log('  npm start              # Start the MCP server');
+    console.log('  crawlforge             # Start the MCP server');
     console.log('  npm run test           # Test your setup');
     console.log('');
     console.log('Need help? Visit: https://www.crawlforge.dev/docs');
@@ -88,11 +279,11 @@ async function main() {
     console.log('  • Documentation: https://www.crawlforge.dev/docs');
     console.log('  • Support: support@crawlforge.dev');
     console.log('');
-    rl.close();
+    closeReadline();
     process.exit(1);
   }
 
-  rl.close();
+  closeReadline();
 }
 
 // Handle errors
@@ -100,13 +291,13 @@ process.on('unhandledRejection', (error) => {
   console.error('');
   console.error('❌ Setup error:', error.message);
   console.error('');
-  rl.close();
+  closeReadline();
   process.exit(1);
 });
 
 // Run setup
 main().catch((error) => {
   console.error('❌ Fatal error:', error);
-  rl.close();
+  closeReadline();
   process.exit(1);
 });

package/src/constants/config.js

@@ -11,7 +11,7 @@ export const config = {
   // CrawlForge API Configuration
   crawlforge: {
     apiKey: process.env.CRAWLFORGE_API_KEY || '',
-    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://api.crawlforge.dev'
+    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev'
   },
 
   // Performance

package/src/core/AuthManager.js

@@ -9,7 +9,7 @@ import path from 'path';
 
 class AuthManager {
   constructor() {
-    this.apiEndpoint = process.env.CRAWLFORGE_API_URL || 'https://api.crawlforge.dev';
+    this.apiEndpoint = process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev';
     this.configPath = path.join(process.env.HOME || process.env.USERPROFILE, '.crawlforge', 'config.json');
     this.config = null;
     this.creditCache = new Map();

package/src/tools/search/adapters/crawlforgeSearch.js

@@ -8,7 +8,7 @@
  */
 
 export class CrawlForgeSearchAdapter {
-  constructor(apiKey, apiBaseUrl = 'https://api.crawlforge.dev') {
+  constructor(apiKey, apiBaseUrl = 'https://www.crawlforge.dev') {
     if (!apiKey) {
       throw new Error('CrawlForge API key is required for search functionality');
     }