@oevortex/ddg_search 1.1.9 → 1.2.1

package/src/utils/search.js

@@ -5,16 +5,17 @@ import { getRandomUserAgent } from './user_agents.js';
 
 // Constants
 const MAX_CACHE_PAGES = 5;
+const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes
+const REQUEST_TIMEOUT = 10000; // 10 seconds
 
 // Cache results to avoid repeated requests
 const resultsCache = new Map();
-const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes
 
 // HTTPS agent configuration to handle certificate chain issues
 const httpsAgent = new https.Agent({
   rejectUnauthorized: true, // Keep security enabled
   keepAlive: true,
-  timeout: 10000,
+  timeout: REQUEST_TIMEOUT,
   // Provide fallback for certificate issues while maintaining security
   secureProtocol: 'TLSv1_2_method'
 });
@@ -107,15 +108,42 @@ function getFaviconUrl(url) {
   }
 }
 
+/**
+ * Generate a Jina AI URL for a given website URL
+ * @param {string} url - The website URL
+ * @returns {string} The Jina AI URL
+ */
+function getJinaAiUrl(url) {
+  try {
+    const urlObj = new URL(url);
+    return `https://r.jina.ai/${urlObj.href}`;
+  } catch {
+    return '';
+  }
+}
 
 /**
  * Scrapes search results from DuckDuckGo HTML
  * @param {string} query - The search query
  * @param {number} numResults - Number of results to return (default: 10)
+ * @param {string} mode - 'short' or 'detailed' mode (default: 'short')
  * @returns {Promise<Array>} - Array of search results
  */
 async function searchDuckDuckGo(query, numResults = 10, mode = 'short') {
   try {
+    // Input validation
+    if (!query || typeof query !== 'string') {
+      throw new Error('Invalid query: query must be a non-empty string');
+    }
+    
+    if (!Number.isInteger(numResults) || numResults < 1 || numResults > 20) {
+      throw new Error('Invalid numResults: must be an integer between 1 and 20');
+    }
+    
+    if (!['short', 'detailed'].includes(mode)) {
+      throw new Error('Invalid mode: must be "short" or "detailed"');
+    }
+
     // Clear old cache entries
     clearOldCache();
 
@@ -124,145 +152,172 @@ async function searchDuckDuckGo(query, numResults = 10, mode = 'short') {
     const cachedResults = resultsCache.get(cacheKey);
 
     if (cachedResults && Date.now() - cachedResults.timestamp < CACHE_DURATION) {
+      console.log(`Cache hit for query: "${query}"`);
       return cachedResults.results.slice(0, numResults);
     }
 
     // Get a random user agent
     const userAgent = getRandomUserAgent();
 
-    // Fetch results
-    const response = await axios.get(
-      `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
-      {
-        headers: {
-          'User-Agent': userAgent
-        },
-        httpsAgent: httpsAgent
-      }
-    );
+    console.log(`Searching DuckDuckGo for: "${query}" (${numResults} results, mode: ${mode})`);
+
+    // Fetch results with timeout
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT);
+
+    try {
+      const response = await axios.get(
+        `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
+        {
+          signal: controller.signal,
+          headers: {
+            'User-Agent': userAgent
+          },
+          httpsAgent: httpsAgent,
+          timeout: REQUEST_TIMEOUT
+        }
+      );
 
-    if (response.status !== 200) {
-      throw new Error('Failed to fetch search results');
-    }
+      clearTimeout(timeoutId);
 
-    const html = response.data;
-
-    // Parse results using cheerio
-    const $ = cheerio.load(html);
-
-    const results = [];
-    const jinaFetchPromises = [];
-    $('.result').each((i, result) => {
-      const $result = $(result);
-      const titleEl = $result.find('.result__title a');
-      const linkEl = $result.find('.result__url');
-      const snippetEl = $result.find('.result__snippet');
-
-      const title = titleEl.text()?.trim();
-      const rawLink = titleEl.attr('href');
-      const description = snippetEl.text()?.trim();
-      const displayUrl = linkEl.text()?.trim();
-
-      const directLink = extractDirectUrl(rawLink || '');
-      const favicon = getFaviconUrl(directLink);
-      const jinaUrl = getJinaAiUrl(directLink);
-
-      if (title && directLink) {
-        if (mode === 'detailed') {
-          jinaFetchPromises.push(
-            axios.get(jinaUrl, {
-              headers: {
-                'User-Agent': getRandomUserAgent()
-              },
-              httpsAgent: httpsAgent,
-              timeout: 10000
-            })
-            .then(jinaRes => {
-              let jinaContent = '';
-              if (jinaRes.status === 200 && typeof jinaRes.data === 'string') {
-                const $jina = cheerio.load(jinaRes.data);
-                jinaContent = $jina('body').text()
-              }
-              return {
-                title,
-                url: directLink,
-                snippet: description || '',
-                favicon: favicon,
-                displayUrl: displayUrl || '',
-                Description: jinaContent
-              };
-            })
-            .catch(() => {
-              return {
+      if (response.status !== 200) {
+        throw new Error(`HTTP ${response.status}: Failed to fetch search results`);
+      }
+
+      const html = response.data;
+
+      // Parse results using cheerio
+      const $ = cheerio.load(html);
+
+      const results = [];
+      const jinaFetchPromises = [];
+
+      $('.result').each((i, result) => {
+        const $result = $(result);
+        const titleEl = $result.find('.result__title a');
+        const linkEl = $result.find('.result__url');
+        const snippetEl = $result.find('.result__snippet');
+
+        const title = titleEl.text()?.trim();
+        const rawLink = titleEl.attr('href');
+        const description = snippetEl.text()?.trim();
+        const displayUrl = linkEl.text()?.trim();
+
+        const directLink = extractDirectUrl(rawLink || '');
+        const favicon = getFaviconUrl(directLink);
+        const jinaUrl = getJinaAiUrl(directLink);
+
+        if (title && directLink) {
+          if (mode === 'detailed') {
+            jinaFetchPromises.push(
+              axios.get(jinaUrl, {
+                headers: {
+                  'User-Agent': getRandomUserAgent()
+                },
+                httpsAgent: httpsAgent,
+                timeout: 8000
+              })
+                .then(jinaRes => {
+                  let jinaContent = '';
+                  if (jinaRes.status === 200 && typeof jinaRes.data === 'string') {
+                    const $jina = cheerio.load(jinaRes.data);
+                    jinaContent = $jina('body').text();
+                  }
+                  return {
+                    title,
+                    url: directLink,
+                    snippet: description || '',
+                    favicon: favicon,
+                    displayUrl: displayUrl || '',
+                    description: jinaContent
+                  };
+                })
+                .catch(() => {
+                  // Return fallback without content
+                  return {
+                    title,
+                    url: directLink,
+                    snippet: description || '',
+                    favicon: favicon,
+                    displayUrl: displayUrl || '',
+                    description: ''
+                  };
+                })
+            );
+          } else {
+            // short mode: omit description
+            jinaFetchPromises.push(
+              Promise.resolve({
                 title,
                 url: directLink,
                 snippet: description || '',
                 favicon: favicon,
-                displayUrl: displayUrl || '',
-                Description: ''
-              };
-            })
-          );
-        } else {
-          // short mode: omit Description
-          jinaFetchPromises.push(
-            Promise.resolve({
-              title,
-              url: directLink,
-              snippet: description || '',
-              favicon: favicon,
-              displayUrl: displayUrl || ''
-            })
-          );
+                displayUrl: displayUrl || ''
+              })
+            );
+          }
         }
+      });
+
+      // Wait for all Jina AI fetches to complete with timeout
+      const jinaResults = await Promise.race([
+        Promise.all(jinaFetchPromises),
+        new Promise((_, reject) => 
+          setTimeout(() => reject(new Error('Content fetch timeout')), 15000)
+        )
+      ]);
+
+      results.push(...jinaResults);
+
+      // Get limited results
+      const limitedResults = results.slice(0, numResults);
+
+      // Cache the results
+      resultsCache.set(cacheKey, {
+        results: limitedResults,
+        timestamp: Date.now()
+      });
+
+      // If cache is too big, remove oldest entries
+      if (resultsCache.size > MAX_CACHE_PAGES) {
+        const oldestKey = Array.from(resultsCache.keys())[0];
+        resultsCache.delete(oldestKey);
       }
-    });
-
-    // Wait for all Jina AI fetches to complete
-    const jinaResults = await Promise.all(jinaFetchPromises);
-    results.push(...jinaResults);
-
-    // Get limited results
-    const limitedResults = results.slice(0, numResults);
 
-    // Cache the results
-    resultsCache.set(cacheKey, {
-      results: limitedResults,
-      timestamp: Date.now()
-    });
-
-    // If cache is too big, remove oldest entries
-    if (resultsCache.size > MAX_CACHE_PAGES) {
-      const oldestKey = Array.from(resultsCache.keys())[0];
-      resultsCache.delete(oldestKey);
+      console.log(`Found ${limitedResults.length} results for query: "${query}"`);
+      return limitedResults;
+    } catch (fetchError) {
+      clearTimeout(timeoutId);
+      
+      if (fetchError.name === 'AbortError') {
+        throw new Error('Search request timeout: took longer than 10 seconds');
+      }
+      
+      if (fetchError.code === 'ENOTFOUND') {
+        throw new Error('Network error: unable to resolve host');
+      }
+      
+      if (fetchError.code === 'ECONNREFUSED') {
+        throw new Error('Network error: connection refused');
+      }
+      
+      throw fetchError;
     }
-
-    return limitedResults;
   } catch (error) {
     console.error('Error searching DuckDuckGo:', error.message);
-    throw error;
+    
+    // Enhanced error reporting
+    if (error.message.includes('Invalid')) {
+      throw error; // Re-throw validation errors as-is
+    }
+    
+    throw new Error(`Search failed for "${query}": ${error.message}`);
   }
 }
 
-
 export {
   searchDuckDuckGo,
   extractDirectUrl,
-  getFaviconUrl
-};
-
-/**
- * Generate a Jina AI URL for a given website URL
- * @param {string} url - The website URL
- * @returns {string} The Jina AI URL
- */
-function getJinaAiUrl(url) {
-  try {
-    const urlObj = new URL(url);
-    return `https://r.jina.ai/${urlObj.href}`;
-  } catch {
-    return '';
-  }
-}
-
-export { getJinaAiUrl };
+  getFaviconUrl,
+  getJinaAiUrl
+};

package/src/utils/search_brave_ai.js

@@ -0,0 +1,167 @@
+import axios from 'axios';
+import { randomBytes } from 'crypto';
+import { getRandomUserAgent } from './user_agents.js';
+
+const BASE_URL = 'https://search.brave.com/api/tap/v1';
+const DEFAULT_TIMEOUT = 30000;
+
+function generateKeyB64() {
+  const key = randomBytes(32);
+  const k = key.toString('base64url');
+  const jwk = {
+    alg: 'A256GCM',
+    ext: true,
+    k,
+    key_ops: ['encrypt', 'decrypt'],
+    kty: 'oct'
+  };
+  return Buffer.from(JSON.stringify(jwk)).toString('base64');
+}
+
+function buildHeaders() {
+  return {
+    accept: 'application/json',
+    'accept-language': 'en-US,en;q=0.9',
+    'user-agent': getRandomUserAgent(),
+    'sec-ch-ua': '"Chromium";v="127", "Not)A;Brand";v="99"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-origin',
+    referer: 'https://search.brave.com/ask'
+  };
+}
+
+function parseStream(stream) {
+  return new Promise((resolve, reject) => {
+    let buffer = '';
+    let text = '';
+
+    stream.on('data', (chunk) => {
+      buffer += chunk.toString();
+      const lines = buffer.split('\n');
+      buffer = lines.pop() ?? '';
+
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+          continue;
+        }
+
+        try {
+          const payload = JSON.parse(trimmed);
+          if (payload?.type === 'text_delta') {
+            text += payload.delta ?? '';
+          }
+        } catch (error) {
+          // Ignore malformed lines
+        }
+      }
+    });
+
+    stream.on('end', () => resolve(text));
+    stream.on('error', (error) => reject(error));
+  });
+}
+
+/**
+ * Search using Brave AI Search.
+ * @param {string} prompt - The search query.
+ * @param {object} [options] - Search options.
+ * @param {boolean} [options.enableResearch=false] - Enable deep research mode.
+ * @param {number} [options.timeout=30000] - Request timeout in ms.
+ * @param {string} [options.language='en'] - Language code.
+ * @param {string} [options.country='US'] - Country code.
+ * @param {string} [options.uiLang='en-us'] - UI language.
+ * @param {string|null} [options.geoloc=null] - Geolocation coordinates.
+ * @returns {Promise<string>} AI-generated response text.
+ */
+export async function searchBraveAI(
+  prompt,
+  {
+    enableResearch = false,
+    timeout = DEFAULT_TIMEOUT,
+    language = 'en',
+    country = 'US',
+    uiLang = 'en-us',
+    geoloc = null
+  } = {}
+) {
+  if (!prompt || typeof prompt !== 'string') {
+    throw new Error('Invalid prompt: must be a non-empty string');
+  }
+
+  if (prompt.length > 5000) {
+    throw new Error('Invalid prompt: too long (maximum 5000 characters)');
+  }
+
+  const symmetricKey = generateKeyB64();
+  const client = axios.create({
+    timeout,
+    headers: buildHeaders(),
+    validateStatus: (status) => status >= 200 && status < 500
+  });
+
+  const newParams = {
+    language,
+    country,
+    ui_lang: uiLang,
+    symmetric_key: symmetricKey,
+    source: enableResearch ? 'home' : 'llmSuggest',
+    query: prompt,
+    enable_research: enableResearch ? 'true' : 'false'
+  };
+
+  if (geoloc) {
+    newParams.geoloc = geoloc;
+  }
+
+  try {
+    const newResponse = await client.get(`${BASE_URL}/new`, { params: newParams });
+    if (newResponse.status !== 200) {
+      throw new Error(`Brave AI failed to initialize chat: HTTP ${newResponse.status}`);
+    }
+
+    const chatId = newResponse.data?.id;
+    if (!chatId) {
+      throw new Error('Brave AI failed to initialize chat: missing conversation id');
+    }
+
+    const streamParams = {
+      id: chatId,
+      query: prompt,
+      symmetric_key: symmetricKey,
+      language,
+      country,
+      ui_lang: uiLang,
+      enable_research: enableResearch ? 'true' : 'false',
+      enable_followups: enableResearch ? 'true' : 'false'
+    };
+
+    const referer = `https://search.brave.com/ask?q=${encodeURIComponent(prompt)}&conversation=${chatId}`;
+    const streamResponse = await client.get(`${BASE_URL}/stream`, {
+      params: streamParams,
+      responseType: 'stream',
+      headers: {
+        referer
+      }
+    });
+
+    if (streamResponse.status !== 200) {
+      throw new Error(`Brave AI stream failed: HTTP ${streamResponse.status}`);
+    }
+
+    return await parseStream(streamResponse.data);
+  } catch (error) {
+    if (error.response?.status === 429) {
+      throw new Error('Brave AI rate limit: too many requests');
+    }
+
+    if (error.code === 'ECONNABORTED') {
+      throw new Error('Brave AI request timeout: took too long');
+    }
+
+    throw new Error(`Brave AI search failed for "${prompt}": ${error.message}`);
+  }
+}