crawlforge-mcp-server 3.0.8 → 3.0.10

package/CLAUDE.md

@@ -418,21 +418,13 @@ export class ToolName {
 
 ### Search Provider Architecture
 
-Search uses a factory pattern with two providers (Google Search only):
+All search requests are proxied through the CrawlForge.dev API:
 
-**Production (users with CrawlForge API key):**
 - `crawlforgeSearch.js` - Proxies through CrawlForge.dev API (Google Search backend)
 - No Google API credentials needed from users
+- Users only need their CrawlForge API key
 - Credit cost: 2 credits per search
 
-**Creator Mode (development with Google API credentials):**
-- `googleSearch.js` - Direct Google Custom Search API
-- Requires: `GOOGLE_API_KEY` and `GOOGLE_SEARCH_ENGINE_ID` environment variables
-- Console shows: "🔍 Creator Mode: Using Google Search API directly"
-- Get credentials at:
-  - API Key: https://console.cloud.google.com/apis/credentials
-  - Search Engine ID: https://programmablesearchengine.google.com/
-
 Factory in `src/tools/search/adapters/searchProviderFactory.js`
 
 ### Browser Management

package/README.md

@@ -32,11 +32,12 @@ npx crawlforge-setup
 This will:
 - Guide you through getting your free API key
 - Configure your credentials securely
+- **Auto-configure Claude Code and Cursor** (if installed)
 - Verify your setup is working
 
 **Don't have an API key?** Get one free at [https://www.crawlforge.dev/signup](https://www.crawlforge.dev/signup)
 
-### 3. Configure Your IDE
+### 3. Configure Your IDE (if not auto-configured)
 
 <details>
 <summary>🤖 For Claude Desktop</summary>
@@ -62,17 +63,39 @@ Restart Claude Desktop to activate.
 </details>
 
 <details>
-<summary>💻 For Cursor IDE</summary>
+<summary>🖥️ For Claude Code CLI (Auto-configured)</summary>
 
-Add to `.cursorrules` in your project:
-```bash
-mcp_servers:
-  crawlforge:
-    command: npx
-    args: ["crawlforge-mcp-server"]
+The setup wizard automatically configures Claude Code by adding to `~/.claude.json`:
+```json
+{
+  "mcpServers": {
+    "crawlforge": {
+      "type": "stdio",
+      "command": "crawlforge"
+    }
+  }
+}
+```
+
+After setup, restart Claude Code to activate.
+</details>
+
+<details>
+<summary>💻 For Cursor IDE (Auto-configured)</summary>
+
+The setup wizard automatically configures Cursor by adding to `~/.cursor/mcp.json`:
+```json
+{
+  "mcpServers": {
+    "crawlforge": {
+      "type": "stdio",
+      "command": "crawlforge"
+    }
+  }
+}
 ```
 
-Or use the MCP plugin in Cursor settings.
+Restart Cursor to activate.
 </details>
 
 ## 📊 Available Tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "3.0.8",
+  "version": "3.0.10",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 19 comprehensive web scraping, crawling, and content processing tools.",
   "main": "server.js",
   "bin": {
@@ -95,7 +95,6 @@
     "compromise": "^14.14.4",
     "diff": "^8.0.2",
     "dotenv": "^17.2.1",
-    "duck-duck-scrape": "^2.2.7",
     "franc": "^6.2.0",
     "isomorphic-dompurify": "^2.26.0",
     "jsdom": "^26.1.0",

package/server.js

@@ -97,7 +97,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 }
 
 // Create the server
-const server = new McpServer({ name: "crawlforge", version: "3.0.4" });
+const server = new McpServer({ name: "crawlforge", version: "3.0.10" });
 
 // Helper function to wrap tool handlers with authentication and credit tracking
 function withAuth(toolName, handler) {

package/setup.js

@@ -6,8 +6,101 @@
  */
 
 import readline from 'readline';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import AuthManager from './src/core/AuthManager.js';
 
+/**
+ * Add CrawlForge to an MCP client configuration file
+ * @param {string} configPath - Path to the config file
+ * @param {string} clientName - Name of the client (for messages)
+ * @returns {object} Result with success status and message
+ */
+function addToMcpConfig(configPath, clientName) {
+  // Check if config exists
+  if (!fs.existsSync(configPath)) {
+    return {
+      success: false,
+      message: `${clientName} config not found (${configPath}). You may need to configure manually.`,
+      notInstalled: true
+    };
+  }
+
+  try {
+    // Read existing config
+    const configContent = fs.readFileSync(configPath, 'utf8');
+    const config = JSON.parse(configContent);
+
+    // Ensure mcpServers object exists
+    if (!config.mcpServers) {
+      config.mcpServers = {};
+    }
+
+    // Check if crawlforge is already configured
+    if (config.mcpServers.crawlforge) {
+      return {
+        success: true,
+        message: `CrawlForge already configured in ${clientName}`,
+        alreadyConfigured: true
+      };
+    }
+
+    // Add crawlforge MCP server configuration
+    config.mcpServers.crawlforge = {
+      type: "stdio",
+      command: "crawlforge",
+      args: [],
+      env: {}
+    };
+
+    // Write updated config
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+
+    return {
+      success: true,
+      message: `CrawlForge added to ${clientName} MCP config`
+    };
+  } catch (error) {
+    return {
+      success: false,
+      message: `Failed to update ${clientName} config: ${error.message}`
+    };
+  }
+}
+
+/**
+ * Configure all detected MCP clients
+ * @returns {object} Results for each client
+ */
+function configureMcpClients() {
+  const results = {
+    claudeCode: null,
+    cursor: null
+  };
+
+  // Claude Code config path
+  const claudeConfigPath = path.join(os.homedir(), '.claude.json');
+  results.claudeCode = addToMcpConfig(claudeConfigPath, 'Claude Code');
+
+  // Cursor config path (macOS)
+  const cursorConfigPath = path.join(os.homedir(), '.cursor', 'mcp.json');
+  if (fs.existsSync(path.join(os.homedir(), '.cursor'))) {
+    // Cursor directory exists, try to configure
+    if (!fs.existsSync(cursorConfigPath)) {
+      // Create mcp.json if it doesn't exist
+      try {
+        fs.writeFileSync(cursorConfigPath, JSON.stringify({ mcpServers: {} }, null, 2));
+      } catch (e) {
+        // Ignore creation errors
+      }
+    }
+    results.cursor = addToMcpConfig(cursorConfigPath, 'Cursor');
+  }
+
+  return results;
+}
+
 const rl = readline.createInterface({
   input: process.stdin,
   output: process.stdout
@@ -74,8 +167,69 @@ async function main() {
     console.log('');
     console.log('🎉 Setup complete! You can now use CrawlForge MCP Server.');
     console.log('');
+
+    // Auto-configure MCP clients (Claude Code, Cursor)
+    console.log('🔧 Configuring MCP client integrations...');
+    const clientResults = configureMcpClients();
+    let anyConfigured = false;
+    let needsRestart = false;
+
+    // Report Claude Code status
+    if (clientResults.claudeCode) {
+      if (clientResults.claudeCode.success) {
+        anyConfigured = true;
+        if (clientResults.claudeCode.alreadyConfigured) {
+          console.log('✅ Claude Code: Already configured');
+        } else {
+          console.log('✅ Claude Code: Added to MCP config (~/.claude.json)');
+          needsRestart = true;
+        }
+      } else if (!clientResults.claudeCode.notInstalled) {
+        console.log(`⚠️  Claude Code: ${clientResults.claudeCode.message}`);
+      }
+    }
+
+    // Report Cursor status
+    if (clientResults.cursor) {
+      if (clientResults.cursor.success) {
+        anyConfigured = true;
+        if (clientResults.cursor.alreadyConfigured) {
+          console.log('✅ Cursor: Already configured');
+        } else {
+          console.log('✅ Cursor: Added to MCP config (~/.cursor/mcp.json)');
+          needsRestart = true;
+        }
+      } else if (!clientResults.cursor.notInstalled) {
+        console.log(`⚠️  Cursor: ${clientResults.cursor.message}`);
+      }
+    }
+
+    // Show restart warning if any client was configured
+    if (needsRestart) {
+      console.log('');
+      console.log('⚠️  IMPORTANT: Restart your IDE to load the new MCP server');
+    }
+
+    // Show manual config instructions if no clients detected
+    if (!anyConfigured && clientResults.claudeCode?.notInstalled && !clientResults.cursor) {
+      console.log('ℹ️  No MCP clients detected. Manual configuration needed:');
+      console.log('');
+      console.log('   Add this to your MCP client config:');
+      console.log('   {');
+      console.log('     "mcpServers": {');
+      console.log('       "crawlforge": {');
+      console.log('         "type": "stdio",');
+      console.log('         "command": "crawlforge"');
+      console.log('       }');
+      console.log('     }');
+      console.log('   }');
+    }
+
+    console.log('');
+    console.log('────────────────────────────────────────────────────────');
+    console.log('');
     console.log('Quick start:');
-    console.log('  npm start              # Start the MCP server');
+    console.log('  crawlforge             # Start the MCP server');
     console.log('  npm run test           # Test your setup');
     console.log('');
     console.log('Need help? Visit: https://www.crawlforge.dev/docs');

package/src/constants/config.js

@@ -11,7 +11,7 @@ export const config = {
   // CrawlForge API Configuration
   crawlforge: {
     apiKey: process.env.CRAWLFORGE_API_KEY || '',
-    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://api.crawlforge.dev'
+    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev'
   },
 
   // Performance

package/src/core/AuthManager.js

@@ -9,7 +9,7 @@ import path from 'path';
 
 class AuthManager {
   constructor() {
-    this.apiEndpoint = process.env.CRAWLFORGE_API_URL || 'https://api.crawlforge.dev';
+    this.apiEndpoint = process.env.CRAWLFORGE_API_URL || 'https://www.crawlforge.dev';
     this.configPath = path.join(process.env.HOME || process.env.USERPROFILE, '.crawlforge', 'config.json');
     this.config = null;
     this.creditCache = new Map();

package/src/tools/search/adapters/crawlforgeSearch.js

@@ -8,7 +8,7 @@
  */
 
 export class CrawlForgeSearchAdapter {
-  constructor(apiKey, apiBaseUrl = 'https://api.crawlforge.dev') {
+  constructor(apiKey, apiBaseUrl = 'https://www.crawlforge.dev') {
     if (!apiKey) {
       throw new Error('CrawlForge API key is required for search functionality');
     }

package/src/tools/search/searchWeb.js

@@ -79,7 +79,7 @@ export class SearchWebTool {
       throw new Error('CrawlForge API key is required for search functionality');
     }
 
-    // Create the search adapter (CrawlForge API or DuckDuckGo fallback for Creator Mode)
+    // Create the search adapter (CrawlForge API proxy or Google Search API direct in Creator Mode)
     try {
       this.searchAdapter = SearchProviderFactory.createAdapter(apiKey, {
         apiBaseUrl,