crawlforge-mcp-server 3.0.7 → 3.0.9

package/CLAUDE.md

@@ -91,7 +91,7 @@ Tools are organized in subdirectories by category:
 - `crawl/` - crawlDeep, mapSite
 - `extract/` - analyzeContent, extractContent, processDocument, summarizeContent
 - `research/` - deepResearch
-- `search/` - searchWeb and provider adapters (Google, DuckDuckGo)
+- `search/` - searchWeb (uses CrawlForge proxy for Google Search)
 - `tracking/` - trackChanges
 - `llmstxt/` - generateLLMsTxt
 
@@ -103,7 +103,7 @@ Tools are organized in subdirectories by category:
 
 **Advanced Tools:**
 
-- search_web (conditional - requires search provider), crawl_deep, map_site
+- search_web, crawl_deep, map_site
 - extract_content, process_document, summarize_content, analyze_content
 - batch_scrape, scrape_with_actions, deep_research
 - track_changes, generate_llms_txt, stealth_mode, localization
@@ -154,12 +154,6 @@ CRAWLFORGE_API_KEY=your_api_key_here
 # CRAWLFORGE_CREATOR_SECRET=your-uuid-secret
 # Enables unlimited access for development/testing
 
-# Search Provider (auto, google, duckduckgo)
-SEARCH_PROVIDER=auto
-
-# Google API (optional, only if using Google)
-GOOGLE_API_KEY=your_key
-GOOGLE_SEARCH_ENGINE_ID=your_id
 
 # Performance Settings
 MAX_WORKERS=10
@@ -424,18 +418,16 @@ export class ToolName {
 
 ### Search Provider Architecture
 
-Search providers implement a factory pattern:
+All search requests are proxied through the CrawlForge.dev API:
 
-- `searchProviderFactory.js` selects provider based on config
-- Providers implement common interface: `search(query, options)`
-- Auto-fallback: Google → DuckDuckGo if Google credentials missing
-- Each provider in `src/tools/search/adapters/`
+- `crawlforgeSearch.js` - Proxies through CrawlForge.dev API (Google Search backend)
+- No Google API credentials needed from users
+- Users only need their CrawlForge API key
+- Credit cost: 2 credits per search
 
-### Browser Management
+Factory in `src/tools/search/adapters/searchProviderFactory.js`
 
-- Playwright used for browser automation (ActionExecutor, ScrapeWithActionsTool)
-- Stealth features in StealthBrowserManager
-- Always cleanup browsers in error handlers
+### Browser Management
 - Context isolation per operation for security
 
 ### Memory Management

package/README.md

@@ -85,7 +85,7 @@ Or use the MCP plugin in Cursor settings.
 
 ### Advanced Tools (2-3 credits)
 - `scrape_structured` - Extract structured data with CSS selectors
-- `search_web` - Search the web with Google/DuckDuckGo
+- `search_web` - Search the web using Google Search API
 - `summarize_content` - Generate intelligent summaries
 - `analyze_content` - Comprehensive content analysis
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 19 comprehensive web scraping, crawling, and content processing tools.",
   "main": "server.js",
   "bin": {
@@ -95,7 +95,6 @@
     "compromise": "^14.14.4",
     "diff": "^8.0.2",
     "dotenv": "^17.2.1",
-    "duck-duck-scrape": "^2.2.7",
     "franc": "^6.2.0",
     "isomorphic-dompurify": "^2.26.0",
     "jsdom": "^26.1.0",

package/server.js

@@ -49,7 +49,7 @@ import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
 import { StealthBrowserManager } from "./src/core/StealthBrowserManager.js";
 import { LocalizationManager } from "./src/core/LocalizationManager.js";
 import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
-import { config, validateConfig, isSearchConfigured, getToolConfig, getActiveSearchProvider } from "./src/constants/config.js";
+import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 // Authentication Manager
 import AuthManager from "./src/core/AuthManager.js";
 
@@ -160,11 +160,8 @@ function withAuth(toolName, handler) {
   };
 }
 
-// Initialize tools
-let searchWebTool = null;
-if (isSearchConfigured()) {
-  searchWebTool = new SearchWebTool(getToolConfig('search_web'));
-}
+// Initialize Search Web Tool - always available with CrawlForge API key
+const searchWebTool = new SearchWebTool(getToolConfig("search_web"));
 const crawlDeepTool = new CrawlDeepTool(getToolConfig('crawl_deep'));
 const mapSiteTool = new MapSiteTool(getToolConfig('map_site'));
 
@@ -946,60 +943,48 @@ server.registerTool("scrape_structured", {
 }));
 
 // Tool: search_web - Web search with configurable providers
-if (searchWebTool) {
-  const activeProvider = getActiveSearchProvider();
-  const providerName = activeProvider === 'google' ? 'Google Custom Search API' : 
-                      activeProvider === 'duckduckgo' ? 'DuckDuckGo' : 'Auto-selected provider';
-  
-  server.registerTool("search_web", {
-    description: `Search the web using ${providerName}`,
-    inputSchema: {
-      query: z.string(),
-      limit: z.number().min(1).max(100).optional(),
-      offset: z.number().min(0).optional(),
-      lang: z.string().optional(),
-      safe_search: z.boolean().optional(),
-      time_range: z.enum(['day', 'week', 'month', 'year', 'all']).optional(),
-      site: z.string().optional(),
-      file_type: z.string().optional()
-    }
-  }, withAuth("search_web", async ({ query, limit, offset, lang, safe_search, time_range, site, file_type }) => {
-    try {
-      if (!query) {
-        return {
-          content: [{
-            type: "text",
-            text: "Query parameter is required"
-          }],
-          isError: true
-        };
-      }
-      
-      const result = await searchWebTool.execute({ query, limit, offset, lang, safe_search, time_range, site, file_type });
-      return {
-        content: [{
-          type: "text",
-          text: JSON.stringify(result, null, 2)
-        }]
-      };
-    } catch (error) {
+// Tool: search_web - Search the web using Google Search via CrawlForge proxy
+server.registerTool("search_web", {
+  description: "Search the web using Google Search API (proxied through CrawlForge)",
+  inputSchema: {
+    query: z.string(),
+    limit: z.number().min(1).max(100).optional(),
+    offset: z.number().min(0).optional(),
+    lang: z.string().optional(),
+    safe_search: z.boolean().optional(),
+    time_range: z.enum(["day", "week", "month", "year", "all"]).optional(),
+    site: z.string().optional(),
+    file_type: z.string().optional()
+  }
+}, withAuth("search_web", async ({ query, limit, offset, lang, safe_search, time_range, site, file_type }) => {
+  try {
+    if (!query) {
       return {
         content: [{
           type: "text",
-          text: `Search failed: ${error.message}`
+          text: "Query parameter is required"
         }],
         isError: true
       };
     }
-  }));
-} else {
-  const activeProvider = getActiveSearchProvider();
-  if (activeProvider === 'google') {
-    console.error("Warning: search_web tool not configured. Set GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID to enable Google search.");
-  } else {
-    console.error("Warning: search_web tool initialization failed. Check your SEARCH_PROVIDER configuration.");
+    
+    const result = await searchWebTool.execute({ query, limit, offset, lang, safe_search, time_range, site, file_type });
+    return {
+      content: [{
+        type: "text",
+        text: JSON.stringify(result, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{
+        type: "text",
+        text: `Search failed: ${error.message}`
+      }],
+      isError: true
+    };
   }
-}
+}));
 
 // Tool: crawl_deep - Deep crawl websites with BFS algorithm
 server.registerTool("crawl_deep", {
@@ -1859,34 +1844,19 @@ async function runServer() {
   console.error("CrawlForge MCP Server v3.0 running on stdio");
   console.error(`Environment: ${config.server.nodeEnv}`);
   
-  if (isSearchConfigured()) {
-    const activeProvider = getActiveSearchProvider();
-    console.error(`Search enabled: ${isSearchConfigured()} (provider: ${activeProvider})`);
-  } else {
-    console.error(`Search enabled: ${isSearchConfigured()}`);
-  }
+  console.error("Search enabled: true (via CrawlForge proxy)");
   
-  const baseTools = 'fetch_url, extract_text, extract_links, extract_metadata, scrape_structured, crawl_deep, map_site';
-  const searchTool = isSearchConfigured() ? ', search_web' : '';
-  const phase3Tools = ', extract_content, process_document, summarize_content, analyze_content';
-  const wave2Tools = ', batch_scrape, scrape_with_actions';
-  const researchTools = ', deep_research';
-  const trackingTools = ', track_changes';
-  const llmsTxtTools = ', generate_llms_txt';
-  const wave3Tools = ', stealth_mode, localization';
+  const baseTools = "fetch_url, extract_text, extract_links, extract_metadata, scrape_structured, crawl_deep, map_site";
+  const searchTool = ", search_web";
+  const phase3Tools = ", extract_content, process_document, summarize_content, analyze_content";
+  const wave2Tools = ", batch_scrape, scrape_with_actions";
+  const researchTools = ", deep_research";
+  const trackingTools = ", track_changes";
+  const llmsTxtTools = ", generate_llms_txt";
+  const wave3Tools = ", stealth_mode, localization";
   console.error(`Tools available: ${baseTools}${searchTool}${phase3Tools}${wave2Tools}${researchTools}${trackingTools}${llmsTxtTools}${wave3Tools}`);
 
-  // Start memory monitoring in development
-  if (config.server.nodeEnv === "development") {
-    memoryMonitor.start();
-    console.error("Memory monitoring started");
-  }
-}
 
-runServer().catch((error) => {
-  console.error("Server error:", error);
-  process.exit(1);
-});
 // === MEMORY LEAK PREVENTION ===
 // Add graceful shutdown handling to prevent memory leaks
 
@@ -1978,3 +1948,15 @@ if (config.server.nodeEnv === 'development') {
     }
   }, 60000); // Check every minute
 }
+
+  // Start memory monitoring in development
+  if (config.server.nodeEnv === "development") {
+    memoryMonitor.start();
+    console.error("Memory monitoring started");
+  }
+}
+
+runServer().catch((error) => {
+  console.error("Server error:", error);
+  process.exit(1);
+});

package/src/constants/config.js

@@ -8,23 +8,10 @@ const __dirname = dirname(__filename);
 dotenv.config({ path: join(__dirname, '../../.env'), quiet: true });
 
 export const config = {
-  // Search Provider Configuration
-  search: {
-    provider: process.env.SEARCH_PROVIDER || 'auto', // 'google', 'duckduckgo', or 'auto'
-    
-    // Google Search API
-    google: {
-      apiKey: process.env.GOOGLE_API_KEY || '',
-      searchEngineId: process.env.GOOGLE_SEARCH_ENGINE_ID || ''
-    },
-    
-    // DuckDuckGo Configuration
-    duckduckgo: {
-      timeout: parseInt(process.env.DUCKDUCKGO_TIMEOUT || '30000'),
-      maxRetries: parseInt(process.env.DUCKDUCKGO_MAX_RETRIES || '3'),
-      retryDelay: parseInt(process.env.DUCKDUCKGO_RETRY_DELAY || '1000'),
-      userAgent: process.env.DUCKDUCKGO_USER_AGENT || process.env.USER_AGENT || 'CrawlForge/1.0'
-    }
+  // CrawlForge API Configuration
+  crawlforge: {
+    apiKey: process.env.CRAWLFORGE_API_KEY || '',
+    apiBaseUrl: process.env.CRAWLFORGE_API_URL || 'https://api.crawlforge.dev'
   },
 
   // Performance
@@ -286,30 +273,6 @@ export const config = {
 export function validateConfig() {
   const errors = [];
 
-  // Check search provider configuration
-  const provider = getActiveSearchProvider();
-  
-  if (config.server.nodeEnv === 'production') {
-    if (provider === 'google') {
-      if (!config.search.google.apiKey) {
-        errors.push('GOOGLE_API_KEY is required when using Google search provider in production');
-      }
-      if (!config.search.google.searchEngineId) {
-        errors.push('GOOGLE_SEARCH_ENGINE_ID is required when using Google search provider in production');
-      }
-    }
-    
-    if (!isSearchConfigured()) {
-      errors.push('Search provider is not properly configured');
-    }
-  }
-  
-  // Validate search provider setting
-  const validProviders = ['google', 'duckduckgo', 'auto'];
-  if (!validProviders.includes(config.search.provider.toLowerCase())) {
-    errors.push(`Invalid SEARCH_PROVIDER value. Must be one of: ${validProviders.join(', ')}`);
-  }
-
   // Validate numeric ranges
   if (config.crawling.maxDepth > 10) {
     errors.push('MAX_CRAWL_DEPTH should not exceed 10 for performance reasons');
@@ -330,60 +293,12 @@ export function validateConfig() {
   return errors;
 }
 
-// Check if search is properly configured
-export function isSearchConfigured() {
-  const provider = getActiveSearchProvider();
-  
-  switch (provider) {
-    case 'google':
-      return !!(config.search.google.apiKey && config.search.google.searchEngineId);
-    case 'duckduckgo':
-      return true; // DuckDuckGo doesn't require API credentials
-    default:
-      return false;
-  }
-}
-
-// Get the active search provider based on configuration and availability
-export function getActiveSearchProvider() {
-  const configuredProvider = config.search.provider.toLowerCase();
-  
-  switch (configuredProvider) {
-    case 'google':
-      return 'google';
-    case 'duckduckgo':
-      return 'duckduckgo';
-    case 'auto':
-    default:
-      // Auto mode: prefer Google if credentials available, otherwise use DuckDuckGo
-      if (config.search.google.apiKey && config.search.google.searchEngineId) {
-        return 'google';
-      }
-      return 'duckduckgo';
-  }
-}
-
 // Get configuration for a specific tool
 export function getToolConfig(toolName) {
-  const provider = getActiveSearchProvider();
-  
   const toolConfigs = {
     search_web: {
-      provider: provider,
-      
-      // Google-specific configuration
-      google: {
-        apiKey: config.search.google.apiKey,
-        searchEngineId: config.search.google.searchEngineId
-      },
-      
-      // DuckDuckGo-specific configuration
-      duckduckgo: {
-        timeout: config.search.duckduckgo.timeout,
-        maxRetries: config.search.duckduckgo.maxRetries,
-        retryDelay: config.search.duckduckgo.retryDelay,
-        userAgent: config.search.duckduckgo.userAgent
-      },
+      apiKey: config.crawlforge.apiKey,
+      apiBaseUrl: config.crawlforge.apiBaseUrl,
       
       // Common configuration
       cacheEnabled: config.performance.cacheEnableDisk,
@@ -593,7 +508,8 @@ export function validateLocalizationConfig() {
     if (localizationConfig.translation.enabled) {
       const validProviders = ['google', 'azure', 'libre'];
       if (!validProviders.includes(localizationConfig.translation.defaultProvider)) {
-        errors.push(`TRANSLATION_PROVIDER must be one of: ${validProviders.join(', ')}`);
+        const providersString = validProviders.join(', ');
+        errors.push('TRANSLATION_PROVIDER must be one of: ' + providersString);
       }
     }
     
@@ -612,4 +528,4 @@ export function validateLocalizationConfig() {
   return errors;
 }
 
-export default config;
+export default config;

package/src/tools/search/adapters/crawlforgeSearch.js

@@ -0,0 +1,107 @@
+/**
+ * CrawlForge Search Adapter
+ * 
+ * Proxies search requests through CrawlForge.dev API which uses Google Search.
+ * Users only need their CrawlForge API key - no Google credentials required.
+ * 
+ * Credit Cost: 2 credits per search
+ */
+
+export class CrawlForgeSearchAdapter {
+  constructor(apiKey, apiBaseUrl = 'https://api.crawlforge.dev') {
+    if (!apiKey) {
+      throw new Error('CrawlForge API key is required for search functionality');
+    }
+    
+    this.apiKey = apiKey;
+    this.apiBaseUrl = apiBaseUrl;
+  }
+
+  /**
+   * Perform a web search via CrawlForge API
+   * @param {Object} params - Search parameters
+   * @param {string} params.query - Search query
+   * @param {number} params.num - Number of results
+   * @param {number} params.start - Starting position
+   * @param {string} params.lr - Language restriction
+   * @param {string} params.safe - Safe search setting
+   * @param {string} params.dateRestrict - Date restriction
+   * @param {string} params.cr - Country restriction
+   * @param {string} params.uule - Location encoding
+   * @returns {Promise<Object>} Search results in Google Search API format
+   */
+  async search(params) {
+    try {
+      const response = await fetch(`${this.apiBaseUrl}/api/v1/search`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'X-API-Key': this.apiKey
+        },
+        body: JSON.stringify({
+          query: params.query,
+          num: params.num || 10,
+          start: params.start || 1,
+          lr: params.lr,
+          safe: params.safe,
+          dateRestrict: params.dateRestrict,
+          cr: params.cr,
+          uule: params.uule,
+          // Forward any additional localization headers
+          headers: params.headers
+        })
+      });
+
+      if (!response.ok) {
+        let errorMessage = 'Search request failed';
+        
+        try {
+          const errorData = await response.json();
+          errorMessage = errorData.message || errorData.error || errorMessage;
+          
+          // Handle specific error cases
+          if (response.status === 401) {
+            errorMessage = 'Invalid API key. Please check your CrawlForge API key.';
+          } else if (response.status === 402) {
+            errorMessage = 'Insufficient credits. Please upgrade your plan at https://www.crawlforge.dev/pricing';
+          } else if (response.status === 429) {
+            errorMessage = 'Rate limit exceeded. Please try again later.';
+          }
+        } catch (parseError) {
+          // If we can't parse the error response, use the status text
+          errorMessage = `Search failed with status ${response.status}: ${response.statusText}`;
+        }
+        
+        throw new Error(errorMessage);
+      }
+
+      const data = await response.json();
+      
+      // Validate response format
+      if (!data || typeof data !== 'object') {
+        throw new Error('Invalid response format from search API');
+      }
+      
+      // Return data in Google Search API compatible format
+      return {
+        items: data.items || [],
+        searchInformation: data.searchInformation || {
+          totalResults: '0',
+          searchTime: 0
+        },
+        queries: data.queries || {},
+        context: data.context || {}
+      };
+    } catch (error) {
+      // Network errors or fetch failures
+      if (error.name === 'TypeError' || error.message.includes('fetch')) {
+        throw new Error(`Network error connecting to CrawlForge API: ${error.message}`);
+      }
+      
+      // Re-throw our formatted errors
+      throw error;
+    }
+  }
+}
+
+export default CrawlForgeSearchAdapter;