@oevortex/ddg_search 1.2.0 → 1.2.1

package/src/utils/search_brave_ai.js

@@ -0,0 +1,167 @@
+import axios from 'axios';
+import { randomBytes } from 'crypto';
+import { getRandomUserAgent } from './user_agents.js';
+
+const BASE_URL = 'https://search.brave.com/api/tap/v1';
+const DEFAULT_TIMEOUT = 30000;
+
+function generateKeyB64() {
+  const key = randomBytes(32);
+  const k = key.toString('base64url');
+  const jwk = {
+    alg: 'A256GCM',
+    ext: true,
+    k,
+    key_ops: ['encrypt', 'decrypt'],
+    kty: 'oct'
+  };
+  return Buffer.from(JSON.stringify(jwk)).toString('base64');
+}
+
+function buildHeaders() {
+  return {
+    accept: 'application/json',
+    'accept-language': 'en-US,en;q=0.9',
+    'user-agent': getRandomUserAgent(),
+    'sec-ch-ua': '"Chromium";v="127", "Not)A;Brand";v="99"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-origin',
+    referer: 'https://search.brave.com/ask'
+  };
+}
+
+function parseStream(stream) {
+  return new Promise((resolve, reject) => {
+    let buffer = '';
+    let text = '';
+
+    stream.on('data', (chunk) => {
+      buffer += chunk.toString();
+      const lines = buffer.split('\n');
+      buffer = lines.pop() ?? '';
+
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+          continue;
+        }
+
+        try {
+          const payload = JSON.parse(trimmed);
+          if (payload?.type === 'text_delta') {
+            text += payload.delta ?? '';
+          }
+        } catch (error) {
+          // Ignore malformed lines
+        }
+      }
+    });
+
+    stream.on('end', () => resolve(text));
+    stream.on('error', (error) => reject(error));
+  });
+}
+
+/**
+ * Search using Brave AI Search.
+ * @param {string} prompt - The search query.
+ * @param {object} [options] - Search options.
+ * @param {boolean} [options.enableResearch=false] - Enable deep research mode.
+ * @param {number} [options.timeout=30000] - Request timeout in ms.
+ * @param {string} [options.language='en'] - Language code.
+ * @param {string} [options.country='US'] - Country code.
+ * @param {string} [options.uiLang='en-us'] - UI language.
+ * @param {string|null} [options.geoloc=null] - Geolocation coordinates.
+ * @returns {Promise<string>} AI-generated response text.
+ */
+export async function searchBraveAI(
+  prompt,
+  {
+    enableResearch = false,
+    timeout = DEFAULT_TIMEOUT,
+    language = 'en',
+    country = 'US',
+    uiLang = 'en-us',
+    geoloc = null
+  } = {}
+) {
+  if (!prompt || typeof prompt !== 'string') {
+    throw new Error('Invalid prompt: must be a non-empty string');
+  }
+
+  if (prompt.length > 5000) {
+    throw new Error('Invalid prompt: too long (maximum 5000 characters)');
+  }
+
+  const symmetricKey = generateKeyB64();
+  const client = axios.create({
+    timeout,
+    headers: buildHeaders(),
+    validateStatus: (status) => status >= 200 && status < 500
+  });
+
+  const newParams = {
+    language,
+    country,
+    ui_lang: uiLang,
+    symmetric_key: symmetricKey,
+    source: enableResearch ? 'home' : 'llmSuggest',
+    query: prompt,
+    enable_research: enableResearch ? 'true' : 'false'
+  };
+
+  if (geoloc) {
+    newParams.geoloc = geoloc;
+  }
+
+  try {
+    const newResponse = await client.get(`${BASE_URL}/new`, { params: newParams });
+    if (newResponse.status !== 200) {
+      throw new Error(`Brave AI failed to initialize chat: HTTP ${newResponse.status}`);
+    }
+
+    const chatId = newResponse.data?.id;
+    if (!chatId) {
+      throw new Error('Brave AI failed to initialize chat: missing conversation id');
+    }
+
+    const streamParams = {
+      id: chatId,
+      query: prompt,
+      symmetric_key: symmetricKey,
+      language,
+      country,
+      ui_lang: uiLang,
+      enable_research: enableResearch ? 'true' : 'false',
+      enable_followups: enableResearch ? 'true' : 'false'
+    };
+
+    const referer = `https://search.brave.com/ask?q=${encodeURIComponent(prompt)}&conversation=${chatId}`;
+    const streamResponse = await client.get(`${BASE_URL}/stream`, {
+      params: streamParams,
+      responseType: 'stream',
+      headers: {
+        referer
+      }
+    });
+
+    if (streamResponse.status !== 200) {
+      throw new Error(`Brave AI stream failed: HTTP ${streamResponse.status}`);
+    }
+
+    return await parseStream(streamResponse.data);
+  } catch (error) {
+    if (error.response?.status === 429) {
+      throw new Error('Brave AI rate limit: too many requests');
+    }
+
+    if (error.code === 'ECONNABORTED') {
+      throw new Error('Brave AI request timeout: took too long');
+    }
+
+    throw new Error(`Brave AI search failed for "${prompt}": ${error.message}`);
+  }
+}

package/src/utils/search_iask.js

@@ -107,233 +107,233 @@ function formatHtml(htmlContent) {
   return outputLines.join('');
 }
 
-/**
- * Search using IAsk AI via WebSocket (Phoenix LiveView)
- * @param {string} prompt - The search query or prompt
- * @param {string} mode - Search mode: 'question', 'academic', 'forums', 'wiki', 'thinking'
- * @param {string|null} detailLevel - Detail level: 'concise', 'detailed', 'comprehensive'
- * @returns {Promise<string>} The search results
- */
-async function searchIAsk(prompt, mode = 'thinking', detailLevel = null) {
-  // Input validation
-  if (!prompt || typeof prompt !== 'string') {
-    throw new Error('Invalid prompt: prompt must be a non-empty string');
-  }
-
-  // Validate mode
-  if (!VALID_MODES.includes(mode)) {
-    throw new Error(`Invalid mode: ${mode}. Valid modes are: ${VALID_MODES.join(', ')}`);
-  }
-
-  // Validate detail level
-  if (detailLevel && !VALID_DETAIL_LEVELS.includes(detailLevel)) {
-    throw new Error(`Invalid detail level: ${detailLevel}. Valid levels are: ${VALID_DETAIL_LEVELS.join(', ')}`);
-  }
-
-  console.log(`IAsk search starting: "${prompt}" (mode: ${mode}, detailLevel: ${detailLevel || 'default'})`);
-
-  // Clear old cache entries
-  clearOldCache();
-
-  const cacheKey = getCacheKey(prompt, mode, detailLevel);
-  const cachedResults = resultsCache.get(cacheKey);
-
-  if (cachedResults && Date.now() - cachedResults.timestamp < CACHE_DURATION) {
-    console.log(`Cache hit for IAsk query: "${prompt}"`);
-    return cachedResults.results;
-  }
-
-  // Build URL parameters
-  const params = new URLSearchParams({ mode, q: prompt });
-  if (detailLevel) {
-    params.append('options[detail_level]', detailLevel);
-  }
-
-  // Create a cookie jar for session management
-  const jar = new CookieJar();
-  const client = wrapper(axios.create({ jar }));
-
-  try {
-    // Get initial page and extract tokens
-    console.log('Fetching IAsk AI initial page...');
-    const response = await client.get(API_ENDPOINT, {
-      params: Object.fromEntries(params),
-      timeout: DEFAULT_TIMEOUT,
-      headers: {
-        'User-Agent': getRandomUserAgent()
-      }
-    });
-
-    const $ = cheerio.load(response.data);
-    
-    const phxNode = $('[id^="phx-"]').first();
-    const csrfToken = $('[name="csrf-token"]').attr('content');
-    const phxId = phxNode.attr('id');
-    const phxSession = phxNode.attr('data-phx-session');
-
-    if (!phxId || !csrfToken) {
-      throw new Error('Failed to extract required tokens from IAsk AI page');
-    }
-
-    // Get the actual response URL (after any redirects)
-    const responseUrl = response.request.res?.responseUrl || response.config.url;
-    
-    // Get cookies from the jar for WebSocket connection
-    const cookies = await jar.getCookies(API_ENDPOINT);
-    const cookieString = cookies.map(c => `${c.key}=${c.value}`).join('; ');
-    
-    // Build WebSocket URL
-    const wsParams = new URLSearchParams({
-      '_csrf_token': csrfToken,
-      'vsn': '2.0.0'
-    });
-    const wsUrl = `wss://iask.ai/live/websocket?${wsParams.toString()}`;
-
-    return new Promise((resolve, reject) => {
-      const ws = new WebSocket(wsUrl, {
-        headers: {
-          'Cookie': cookieString,
-          'User-Agent': getRandomUserAgent(),
-          'Origin': 'https://iask.ai'
-        }
-      });
-      
-      let buffer = '';
-      let timeoutId;
-      let connectionTimeoutId;
-
-      // Set connection timeout
-      connectionTimeoutId = setTimeout(() => {
-        ws.close();
-        reject(new Error('IAsk connection timeout: unable to establish WebSocket connection'));
-      }, 15000);
-
-      ws.on('open', () => {
-        clearTimeout(connectionTimeoutId);
-        console.log('IAsk WebSocket connection established');
-        
-        // Send phx_join message
-        ws.send(JSON.stringify([
-          null,
-          null,
-          `lv:${phxId}`,
-          'phx_join',
-          {
-            params: { _csrf_token: csrfToken },
-            url: responseUrl,
-            session: phxSession
-          }
-        ]));
-
-        // Set message timeout
-        timeoutId = setTimeout(() => {
-          ws.close();
-          if (buffer) {
-            resolve(buffer || 'No results found.');
-          } else {
-            reject(new Error('IAsk response timeout: no response received'));
-          }
-        }, DEFAULT_TIMEOUT);
-      });
-
-      ws.on('message', (data) => {
-        try {
-          const msg = JSON.parse(data.toString());
-          if (!msg) return;
-
-          const diff = msg[4];
-          if (!diff) return;
-
-          let chunk = null;
-
-          try {
-            // Try to get chunk from diff.e[0][1].data
-            if (diff.e) {
-              chunk = diff.e[0][1].data;
-              
-              if (chunk) {
-                let formatted;
-                if (/<[^>]+>/.test(chunk)) {
-                  formatted = formatHtml(chunk);
-                } else {
-                  formatted = chunk.replace(/<br\/>/g, '\n');
-                }
-                
-                buffer += formatted;
-              }
-            } else {
-              throw new Error('No diff.e');
-            }
-          } catch {
-            // Fallback to cacheFind
-            const cache = cacheFind(diff);
-            if (cache) {
-              let formatted;
-              if (/<[^>]+>/.test(cache)) {
-                formatted = formatHtml(cache);
-              } else {
-                formatted = cache;
-              }
-              buffer += formatted;
-              // Close after cache find
-              ws.close();
-              return;
-            }
-          }
-        } catch (err) {
-          console.error('Error parsing IAsk message:', err.message);
-        }
-      });
-
-      ws.on('close', () => {
-        clearTimeout(timeoutId);
-        clearTimeout(connectionTimeoutId);
-        
-        console.log(`IAsk search completed: ${buffer.length} characters received`);
-        
-        // Cache the result
-        if (buffer) {
-          resultsCache.set(cacheKey, {
-            results: buffer,
-            timestamp: Date.now()
-          });
-        }
-        
-        resolve(buffer || 'No results found.');
-      });
-
-      ws.on('error', (err) => {
-        clearTimeout(timeoutId);
-        clearTimeout(connectionTimeoutId);
-        console.error('IAsk WebSocket error:', err.message);
-        
-        if (err.message.includes('timeout')) {
-          reject(new Error('IAsk WebSocket timeout: connection took too long'));
-        } else if (err.message.includes('connection refused')) {
-          reject(new Error('IAsk connection refused: service may be unavailable'));
-        } else {
-          reject(new Error(`IAsk WebSocket error: ${err.message}`));
-        }
-      });
-    });
-  } catch (error) {
-    console.error('Error in IAsk search:', error.message);
-    
-    // Enhanced error handling
-    if (error.code === 'ENOTFOUND') {
-      throw new Error('IAsk network error: unable to resolve host');
-    }
-    
-    if (error.code === 'ECONNREFUSED') {
-      throw new Error('IAsk network error: connection refused');
-    }
-    
-    if (error.message.includes('timeout')) {
-      throw new Error(`IAsk timeout: ${error.message}`);
-    }
-    
-    throw new Error(`IAsk search failed for "${prompt}": ${error.message}`);
-  }
-}
+/**
+ * Search using IAsk AI via WebSocket (Phoenix LiveView)
+ * @param {string} prompt - The search query or prompt
+ * @param {string} mode - Search mode: 'question', 'academic', 'forums', 'wiki', 'thinking'
+ * @param {string|null} detailLevel - Detail level: 'concise', 'detailed', 'comprehensive'
+ * @returns {Promise<string>} The search results
+ */
+async function searchIAsk(prompt, mode = 'thinking', detailLevel = null) {
+  // Input validation
+  if (!prompt || typeof prompt !== 'string') {
+    throw new Error('Invalid prompt: prompt must be a non-empty string');
+  }
+
+  // Validate mode
+  if (!VALID_MODES.includes(mode)) {
+    throw new Error(`Invalid mode: ${mode}. Valid modes are: ${VALID_MODES.join(', ')}`);
+  }
+
+  // Validate detail level
+  if (detailLevel && !VALID_DETAIL_LEVELS.includes(detailLevel)) {
+    throw new Error(`Invalid detail level: ${detailLevel}. Valid levels are: ${VALID_DETAIL_LEVELS.join(', ')}`);
+  }
+
+  console.log(`IAsk search starting: "${prompt}" (mode: ${mode}, detailLevel: ${detailLevel || 'default'})`);
+
+  // Clear old cache entries
+  clearOldCache();
+
+  const cacheKey = getCacheKey(prompt, mode, detailLevel);
+  const cachedResults = resultsCache.get(cacheKey);
+
+  if (cachedResults && Date.now() - cachedResults.timestamp < CACHE_DURATION) {
+    console.log(`Cache hit for IAsk query: "${prompt}"`);
+    return cachedResults.results;
+  }
+
+  // Build URL parameters
+  const params = new URLSearchParams({ mode, q: prompt });
+  if (detailLevel) {
+    params.append('options[detail_level]', detailLevel);
+  }
+
+  // Create a cookie jar for session management
+  const jar = new CookieJar();
+  const client = wrapper(axios.create({ jar }));
+
+  try {
+    // Get initial page and extract tokens
+    console.log('Fetching IAsk AI initial page...');
+    const response = await client.get(API_ENDPOINT, {
+      params: Object.fromEntries(params),
+      timeout: DEFAULT_TIMEOUT,
+      headers: {
+        'User-Agent': getRandomUserAgent()
+      }
+    });
+
+    const $ = cheerio.load(response.data);
+    
+    const phxNode = $('[id^="phx-"]').first();
+    const csrfToken = $('[name="csrf-token"]').attr('content');
+    const phxId = phxNode.attr('id');
+    const phxSession = phxNode.attr('data-phx-session');
+
+    if (!phxId || !csrfToken) {
+      throw new Error('Failed to extract required tokens from IAsk AI page');
+    }
+
+    // Get the actual response URL (after any redirects)
+    const responseUrl = response.request.res?.responseUrl || response.config.url;
+    
+    // Get cookies from the jar for WebSocket connection
+    const cookies = await jar.getCookies(API_ENDPOINT);
+    const cookieString = cookies.map(c => `${c.key}=${c.value}`).join('; ');
+    
+    // Build WebSocket URL
+    const wsParams = new URLSearchParams({
+      '_csrf_token': csrfToken,
+      'vsn': '2.0.0'
+    });
+    const wsUrl = `wss://iask.ai/live/websocket?${wsParams.toString()}`;
+
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(wsUrl, {
+        headers: {
+          'Cookie': cookieString,
+          'User-Agent': getRandomUserAgent(),
+          'Origin': 'https://iask.ai'
+        }
+      });
+      
+      let buffer = '';
+      let timeoutId;
+      let connectionTimeoutId;
+
+      // Set connection timeout
+      connectionTimeoutId = setTimeout(() => {
+        ws.close();
+        reject(new Error('IAsk connection timeout: unable to establish WebSocket connection'));
+      }, 15000);
+
+      ws.on('open', () => {
+        clearTimeout(connectionTimeoutId);
+        console.log('IAsk WebSocket connection established');
+        
+        // Send phx_join message
+        ws.send(JSON.stringify([
+          null,
+          null,
+          `lv:${phxId}`,
+          'phx_join',
+          {
+            params: { _csrf_token: csrfToken },
+            url: responseUrl,
+            session: phxSession
+          }
+        ]));
+
+        // Set message timeout
+        timeoutId = setTimeout(() => {
+          ws.close();
+          if (buffer) {
+            resolve(buffer || 'No results found.');
+          } else {
+            reject(new Error('IAsk response timeout: no response received'));
+          }
+        }, DEFAULT_TIMEOUT);
+      });
+
+      ws.on('message', (data) => {
+        try {
+          const msg = JSON.parse(data.toString());
+          if (!msg) return;
+
+          const diff = msg[4];
+          if (!diff) return;
+
+          let chunk = null;
+
+          try {
+            // Try to get chunk from diff.e[0][1].data
+            if (diff.e) {
+              chunk = diff.e[0][1].data;
+              
+              if (chunk) {
+                let formatted;
+                if (/<[^>]+>/.test(chunk)) {
+                  formatted = formatHtml(chunk);
+                } else {
+                  formatted = chunk.replace(/<br\/>/g, '\n');
+                }
+                
+                buffer += formatted;
+              }
+            } else {
+              throw new Error('No diff.e');
+            }
+          } catch {
+            // Fallback to cacheFind
+            const cache = cacheFind(diff);
+            if (cache) {
+              let formatted;
+              if (/<[^>]+>/.test(cache)) {
+                formatted = formatHtml(cache);
+              } else {
+                formatted = cache;
+              }
+              buffer += formatted;
+              // Close after cache find
+              ws.close();
+              return;
+            }
+          }
+        } catch (err) {
+          console.error('Error parsing IAsk message:', err.message);
+        }
+      });
+
+      ws.on('close', () => {
+        clearTimeout(timeoutId);
+        clearTimeout(connectionTimeoutId);
+        
+        console.log(`IAsk search completed: ${buffer.length} characters received`);
+        
+        // Cache the result
+        if (buffer) {
+          resultsCache.set(cacheKey, {
+            results: buffer,
+            timestamp: Date.now()
+          });
+        }
+        
+        resolve(buffer || 'No results found.');
+      });
+
+      ws.on('error', (err) => {
+        clearTimeout(timeoutId);
+        clearTimeout(connectionTimeoutId);
+        console.error('IAsk WebSocket error:', err.message);
+        
+        if (err.message.includes('timeout')) {
+          reject(new Error('IAsk WebSocket timeout: connection took too long'));
+        } else if (err.message.includes('connection refused')) {
+          reject(new Error('IAsk connection refused: service may be unavailable'));
+        } else {
+          reject(new Error(`IAsk WebSocket error: ${err.message}`));
+        }
+      });
+    });
+  } catch (error) {
+    console.error('Error in IAsk search:', error.message);
+    
+    // Enhanced error handling
+    if (error.code === 'ENOTFOUND') {
+      throw new Error('IAsk network error: unable to resolve host');
+    }
+    
+    if (error.code === 'ECONNREFUSED') {
+      throw new Error('IAsk network error: connection refused');
+    }
+    
+    if (error.message.includes('timeout')) {
+      throw new Error(`IAsk timeout: ${error.message}`);
+    }
+    
+    throw new Error(`IAsk search failed for "${prompt}": ${error.message}`);
+  }
+}
 
 export { searchIAsk, VALID_MODES, VALID_DETAIL_LEVELS };