crawlforge-mcp-server 3.0.7 → 3.0.9

package/src/tools/search/adapters/googleSearch.js

@@ -1,236 +1,131 @@
-import { customsearch } from '@googleapis/customsearch';
-import { RetryManager } from '../../../utils/RetryManager.js';
-import { createCircuitBreaker } from '../../../utils/CircuitBreaker.js';
-import { logger } from '../../../utils/Logger.js';
+/**
+ * Google Custom Search API Adapter
+ *
+ * Direct integration with Google Custom Search API.
+ * Used by Creator Mode when GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID are configured.
+ *
+ * Requirements:
+ * - GOOGLE_API_KEY: Your Google Cloud API key
+ * - GOOGLE_SEARCH_ENGINE_ID: Your Custom Search Engine ID (cx)
+ *
+ * Get credentials at:
+ * - API Key: https://console.cloud.google.com/apis/credentials
+ * - Search Engine ID: https://programmablesearchengine.google.com/
+ */
 
 export class GoogleSearchAdapter {
-  constructor(apiKey, searchEngineId, options = {}) {
-    if (!apiKey || !searchEngineId) {
-      throw new Error('Google API key and Search Engine ID are required');
+  constructor(apiKey, searchEngineId) {
+    if (!apiKey) {
+      throw new Error('Google API key is required');
+    }
+    if (!searchEngineId) {
+      throw new Error('Google Search Engine ID (cx) is required');
     }
 
     this.apiKey = apiKey;
     this.searchEngineId = searchEngineId;
-    this.customsearch = customsearch('v1');
-
-    // Initialize error handling components
-    this.retryManager = options.retryManager || RetryManager.createPreset('api');
-    this.circuitBreaker = options.circuitBreaker || createCircuitBreaker('api');
-    this.logger = logger.child({ component: 'GoogleSearchAdapter' });
-
-    // Service identifier for circuit breaker
-    this.serviceId = 'google-search-api';
+    this.baseUrl = 'https://www.googleapis.com/customsearch/v1';
   }
 
+  /**
+   * Perform a web search via Google Custom Search API
+   * @param {Object} params - Search parameters
+   * @param {string} params.query - Search query
+   * @param {number} params.num - Number of results (1-10 per request)
+   * @param {number} params.start - Starting position (1-based)
+   * @param {string} params.lr - Language restriction (e.g., 'lang_en')
+   * @param {string} params.safe - Safe search setting ('active' or 'off')
+   * @param {string} params.dateRestrict - Date restriction (e.g., 'd1', 'w1', 'm1', 'y1')
+   * @param {string} params.cr - Country restriction
+   * @param {string} params.siteSearch - Restrict to specific site
+   * @param {string} params.fileType - Restrict to file type
+   * @returns {Promise<Object>} Search results
+   */
   async search(params) {
-    const requestId = this.logger.startRequest({ 
-      operation: 'search',
-      query: params.query,
-      parameters: { ...params, query: '[REDACTED]' } // Don't log sensitive query data
-    });
-
     try {
-      // Simplified execution without circuit breaker for now
-      const executeSearch = async () => {
-        this.logger.debug('Executing Google search API call', { params }, requestId);
-        
-        const response = await this.customsearch.cse.list({
-          auth: this.apiKey,
-          cx: this.searchEngineId,
-          q: params.query,
-          num: params.num || 10,
-          start: params.start || 1,
-          lr: params.lr,
-          safe: params.safe,
-          dateRestrict: params.dateRestrict,
-          siteSearch: params.siteSearch,
-          siteSearchFilter: params.siteSearchFilter,
-          fileType: params.fileType,
-          rights: params.rights,
-          imgSize: params.imgSize,
-          imgType: params.imgType,
-          imgColorType: params.imgColorType,
-          imgDominantColor: params.imgDominantColor
-        });
-
-        this.logger.info('Google search API call successful', {
-          resultsCount: response.data?.items?.length || 0,
-          searchTime: response.data?.searchInformation?.searchTime
-        }, requestId);
-
-        return response.data;
-      };
-
-      // Try to use retry manager if available, otherwise execute directly
-      let result;
-      try {
-        result = await this.retryManager.execute(executeSearch, { operation: 'search', query: params.query });
-      } catch (retryError) {
-        // If retry manager fails, try direct execution
-        this.logger.warn('Retry manager failed, executing directly', { error: retryError.message }, requestId);
-        result = await executeSearch();
-      }
-
-      this.logger.endRequest(requestId, { 
-        success: true,
-        resultsCount: result?.items?.length || 0 
-      });
-
-      return result;
-    } catch (error) {
-      this.logger.requestError(requestId, error, { 
-        operation: 'search',
-        query: params.query 
+      // Build query parameters
+      const searchParams = new URLSearchParams({
+        key: this.apiKey,
+        cx: this.searchEngineId,
+        q: params.query,
+        num: Math.min(params.num || 10, 10), // Google API max is 10 per request
+        start: params.start || 1,
       });
 
-      // Enhanced error handling with detailed logging
-      if (error.response) {
-        const status = error.response.status;
-        const message = error.response.data?.error?.message || error.message;
-        
-        this.logger.warn('Google Search API error response', {
-          status,
-          message,
-          query: params.query
-        }, requestId);
-        
-        if (status === 429) {
-          throw new Error('API rate limit exceeded. Please try again later.');
-        } else if (status === 403) {
-          throw new Error('API access forbidden. Check your API key and permissions.');
-        } else if (status === 400) {
-          throw new Error(`Invalid search parameters: ${message}`);
-        } else if (status >= 500) {
-          throw new Error(`Google Search API server error (${status}): ${message}`);
-        }
+      // Add optional parameters
+      if (params.lr) {
+        searchParams.set('lr', params.lr);
       }
-      
-      throw new Error(`Google Search API error: ${error.message}`);
-    }
-  }
-
-  async getSuggestions(query) {
-    // Google doesn't provide suggestions through the Custom Search API
-    // This could be implemented with a separate API or service
-    return [];
-  }
-
-  async getRelatedSearches(query) {
-    try {
-      // Perform a search and extract related searches from the response
-      const response = await this.search({
-        query,
-        num: 1
-      });
-
-      if (response.queries && response.queries.related) {
-        return response.queries.related.map(r => r.searchTerms);
+      if (params.safe) {
+        searchParams.set('safe', params.safe);
+      }
+      if (params.dateRestrict) {
+        searchParams.set('dateRestrict', params.dateRestrict);
+      }
+      if (params.cr) {
+        searchParams.set('cr', params.cr);
+      }
+      if (params.siteSearch) {
+        searchParams.set('siteSearch', params.siteSearch);
+      }
+      if (params.fileType) {
+        searchParams.set('fileType', params.fileType);
       }
 
-      return [];
-    } catch {
-      return [];
-    }
-  }
-
-  async validateApiKey() {
-    const requestId = this.logger.startRequest({ operation: 'validateApiKey' });
-    
-    try {
-      await this.search({
-        query: 'test',
-        num: 1
+      const response = await fetch(`${this.baseUrl}?${searchParams.toString()}`, {
+        method: 'GET',
+        headers: {
+          'Accept': 'application/json',
+        }
       });
-      
-      this.logger.endRequest(requestId, { success: true, valid: true });
-      return true;
-    } catch (error) {
-      this.logger.requestError(requestId, error, { operation: 'validateApiKey' });
-      return false;
-    }
-  }
-
-  /**
-   * Get error handling statistics
-   * @returns {Object} Statistics from retry manager and circuit breaker
-   */
-  getStats() {
-    return {
-      retryStats: this.retryManager.getStats(),
-      circuitBreakerStats: this.circuitBreaker.getStats(),
-      loggerStats: this.logger.getStats()
-    };
-  }
-
-  /**
-   * Reset error handling statistics
-   */
-  resetStats() {
-    this.retryManager.resetStats();
-    this.circuitBreaker.reset(this.serviceId);
-  }
 
-  /**
-   * Get health status of the service
-   * @returns {Object} Health status information
-   */
-  getHealthStatus() {
-    const circuitStats = this.circuitBreaker.getServiceMetrics(this.serviceId);
-    const retryStats = this.retryManager.getStats();
-    
-    return {
-      status: circuitStats.state === 'CLOSED' ? 'healthy' : 'degraded',
-      circuitState: circuitStats.state,
-      errorRate: circuitStats.errorRate,
-      successRate: retryStats.successRate,
-      lastFailure: circuitStats.lastFailure,
-      nextAttempt: circuitStats.nextAttempt
-    };
-  }
-}
-
-export class MockSearchAdapter {
-  // Mock adapter for testing without API keys
-  async search(params) {
-    return {
-      kind: 'customsearch#search',
-      searchInformation: {
-        searchTime: 0.123,
-        formattedSearchTime: '0.12',
-        totalResults: '1000',
-        formattedTotalResults: '1,000'
-      },
-      items: [
-        {
-          title: `Mock result for: ${params.query}`,
-          link: `https://example.com/mock/${params.query.replace(/\s+/g, '-')}`,
-          displayLink: 'example.com',
-          snippet: `This is a mock search result for the query "${params.query}". It demonstrates the search functionality without requiring API credentials.`,
-          htmlSnippet: `This is a mock search result for the query "<b>${params.query}</b>". It demonstrates the search functionality without requiring API credentials.`,
-          formattedUrl: 'https://example.com/mock',
-          pagemap: {
-            metatags: [{
-              'og:title': `Mock: ${params.query}`,
-              'og:description': 'Mock search result description',
-              'og:image': 'https://example.com/image.jpg'
-            }]
+      if (!response.ok) {
+        let errorMessage = 'Google Search API request failed';
+
+        try {
+          const errorData = await response.json();
+          if (errorData.error) {
+            errorMessage = errorData.error.message || errorMessage;
+
+            // Handle specific error cases
+            if (response.status === 400) {
+              errorMessage = `Invalid request: ${errorMessage}`;
+            } else if (response.status === 401) {
+              errorMessage = 'Invalid Google API key';
+            } else if (response.status === 403) {
+              errorMessage = 'Google API access forbidden. Check API key permissions or quota.';
+            } else if (response.status === 429) {
+              errorMessage = 'Google API quota exceeded. Try again later.';
+            }
           }
+        } catch (parseError) {
+          errorMessage = `Google Search failed with status ${response.status}: ${response.statusText}`;
         }
-      ]
-    };
-  }
 
-  async getSuggestions(query) {
-    return [`${query} tutorial`, `${query} examples`, `${query} documentation`];
-  }
+        throw new Error(errorMessage);
+      }
 
-  async getRelatedSearches(query) {
-    return [`${query} best practices`, `${query} alternatives`, `how to ${query}`];
-  }
+      const data = await response.json();
+
+      // Return in standard format
+      return {
+        items: data.items || [],
+        searchInformation: data.searchInformation || {
+          totalResults: '0',
+          searchTime: 0
+        },
+        queries: data.queries || {},
+        context: data.context || {}
+      };
+    } catch (error) {
+      // Network errors
+      if (error.name === 'TypeError' || error.message.includes('fetch')) {
+        throw new Error(`Network error connecting to Google Search API: ${error.message}`);
+      }
 
-  async validateApiKey() {
-    return true;
+      throw error;
+    }
   }
 }
 
-export default GoogleSearchAdapter;
+export default GoogleSearchAdapter;

package/src/tools/search/adapters/searchProviderFactory.js

@@ -1,96 +1,186 @@
+/**
+ * Search Provider Factory
+ *
+ * Creates search adapter instances.
+ * - Production: Uses CrawlForge proxy for Google Search (users with CrawlForge API key)
+ * - Creator Mode: Uses Google Search API directly (requires GOOGLE_API_KEY & GOOGLE_SEARCH_ENGINE_ID)
+ */
+
+import { CrawlForgeSearchAdapter } from './crawlforgeSearch.js';
 import { GoogleSearchAdapter } from './googleSearch.js';
-import { DuckDuckGoSearchAdapter } from './duckduckgoSearch.js';
 
 export class SearchProviderFactory {
-  static createAdapter(provider, options = {}) {
-    switch (provider.toLowerCase()) {
-      case 'google':
-        if (!options.google?.apiKey || !options.google?.searchEngineId) {
-          throw new Error('Google Search adapter requires apiKey and searchEngineId');
-        }
-        return new GoogleSearchAdapter(
-          options.google.apiKey,
-          options.google.searchEngineId
+  /**
+   * Create a search adapter
+   * @param {string} apiKey - CrawlForge API key (optional for Creator Mode)
+   * @param {Object} options - Configuration options
+   * @param {string} options.apiBaseUrl - Custom API base URL (optional)
+   * @param {boolean} options.creatorMode - Whether Creator Mode is enabled
+   * @param {string} options.googleApiKey - Google API key (for Creator Mode)
+   * @param {string} options.googleSearchEngineId - Google Search Engine ID (for Creator Mode)
+   * @returns {CrawlForgeSearchAdapter|GoogleSearchAdapter} Search adapter instance
+   */
+  static createAdapter(apiKey, options = {}) {
+    // In Creator Mode without CrawlForge API key, use Google Search API directly
+    if (!apiKey && options.creatorMode) {
+      const googleApiKey = options.googleApiKey || process.env.GOOGLE_API_KEY;
+      const googleSearchEngineId = options.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;
+
+      if (!googleApiKey || !googleSearchEngineId) {
+        throw new Error(
+          'Creator Mode requires Google Search API credentials. ' +
+          'Set GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID environment variables.\n' +
+          'Get credentials at:\n' +
+          '  - API Key: https://console.cloud.google.com/apis/credentials\n' +
+          '  - Search Engine ID: https://programmablesearchengine.google.com/'
         );
-        
-      case 'duckduckgo':
-        return new DuckDuckGoSearchAdapter(options.duckduckgo || {});
-        
-      default:
-        throw new Error(`Unsupported search provider: ${provider}`);
+      }
+
+      console.log('🔍 Creator Mode: Using Google Search API directly');
+      return new GoogleSearchAdapter(googleApiKey, googleSearchEngineId);
+    }
+
+    // Production mode requires CrawlForge API key
+    if (!apiKey) {
+      throw new Error('CrawlForge API key is required for search functionality');
     }
+
+    return new CrawlForgeSearchAdapter(
+      apiKey,
+      options.apiBaseUrl || 'https://api.crawlforge.dev'
+    );
   }
 
+  /**
+   * Get supported providers
+   * @returns {Array<string>} List of supported providers
+   */
   static getSupportedProviders() {
-    return ['google', 'duckduckgo'];
+    return ['crawlforge', 'google'];
   }
 
-  static isProviderAvailable(provider, options = {}) {
-    try {
-      SearchProviderFactory.createAdapter(provider, options);
-      return true;
-    } catch {
-      return false;
+  /**
+   * Check if provider is available
+   * @param {string} apiKey - CrawlForge API key
+   * @param {Object} options - Configuration options
+   * @returns {boolean} True if a provider is available
+   */
+  static isProviderAvailable(apiKey, options = {}) {
+    // CrawlForge API key available
+    if (apiKey) return true;
+
+    // Creator Mode with Google credentials
+    if (options.creatorMode) {
+      const googleApiKey = options.googleApiKey || process.env.GOOGLE_API_KEY;
+      const googleSearchEngineId = options.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;
+      return !!(googleApiKey && googleSearchEngineId);
     }
+
+    return false;
   }
 
-  static getProviderCapabilities(provider) {
-    const capabilities = {
-      google: {
+  /**
+   * Get provider capabilities
+   * @param {string} provider - Provider name ('crawlforge' or 'google')
+   * @returns {Object} Provider capabilities
+   */
+  static getProviderCapabilities(provider = 'crawlforge') {
+    if (provider === 'google') {
+      return {
         requiresApiKey: true,
+        apiKeyType: 'Google API key + Search Engine ID',
         supportsPagination: true,
         supportsLanguageFilter: true,
         supportsDateFilter: true,
         supportsSiteFilter: true,
         supportsFileTypeFilter: true,
         supportsSafeSearch: true,
-        maxResultsPerRequest: 100,
-        rateLimit: '100 queries per day (free tier)',
+        supportsLocalization: true,
+        supportsCountryTargeting: true,
+        maxResultsPerRequest: 10,
+        rateLimit: 'Based on Google API quota',
+        creditCost: 'N/A (direct API)',
         features: [
-          'Web search',
-          'Image search',
+          'Google Custom Search API',
+          'Full text search',
+          'Image metadata',
           'Exact phrase matching',
           'Boolean operators',
           'Site-specific search',
           'File type filtering',
           'Date range filtering',
           'Language filtering',
+          'Country targeting',
           'Safe search',
-          'Related searches'
+          'Rich snippets'
+        ],
+        benefits: [
+          'Direct Google API access',
+          'No proxy overhead',
+          'Full Google Search capabilities',
+          'Creator Mode only'
         ]
-      },
-      duckduckgo: {
-        requiresApiKey: false,
-        supportsPagination: false, // Limited by API
-        supportsLanguageFilter: true,
-        supportsDateFilter: true,
-        supportsSiteFilter: false, // Not directly supported
-        supportsFileTypeFilter: false, // Not directly supported
-        supportsSafeSearch: true,
-        maxResultsPerRequest: 10, // Limited by instant answer API
-        rateLimit: 'No explicit limit (be respectful)',
-        features: [
-          'Privacy-focused search',
-          'Instant answers',
-          'No tracking',
-          'Language filtering',
-          'Date filtering',
-          'Safe search',
-          'Autocomplete suggestions'
-        ]
-      }
-    };
+      };
+    }
 
-    return capabilities[provider.toLowerCase()] || null;
+    return {
+      requiresApiKey: true,
+      apiKeyType: 'CrawlForge API key',
+      supportsPagination: true,
+      supportsLanguageFilter: true,
+      supportsDateFilter: true,
+      supportsSiteFilter: true,
+      supportsFileTypeFilter: true,
+      supportsSafeSearch: true,
+      supportsLocalization: true,
+      supportsCountryTargeting: true,
+      maxResultsPerRequest: 100,
+      rateLimit: 'Based on your CrawlForge plan',
+      creditCost: '2 credits per search',
+      features: [
+        'Google Search results via CrawlForge proxy',
+        'No Google API credentials needed',
+        'Full text search',
+        'Image metadata',
+        'Exact phrase matching',
+        'Boolean operators',
+        'Site-specific search',
+        'File type filtering',
+        'Date range filtering',
+        'Language filtering',
+        'Country targeting',
+        'Safe search',
+        'Localization support',
+        'Related searches',
+        'Rich snippets'
+      ],
+      benefits: [
+        'Simplified authentication (one API key)',
+        'Unified billing through CrawlForge',
+        'Enterprise-grade reliability',
+        'No Google API quota management',
+        'Built-in rate limiting',
+        'Credit-based pricing'
+      ]
+    };
   }
 
+  /**
+   * Compare providers
+   * @returns {Array<Object>} Provider comparison
+   */
   static compareProviders() {
-    const providers = SearchProviderFactory.getSupportedProviders();
-    return providers.map(provider => ({
-      name: provider,
-      ...SearchProviderFactory.getProviderCapabilities(provider)
-    }));
+    return [
+      {
+        name: 'crawlforge',
+        ...SearchProviderFactory.getProviderCapabilities('crawlforge')
+      },
+      {
+        name: 'google',
+        ...SearchProviderFactory.getProviderCapabilities('google')
+      }
+    ];
   }
 }
 
-export default SearchProviderFactory;
+export default SearchProviderFactory;