intelwatch 1.3.0 → 1.5.0

package/src/utils/fetcher.js

@@ -1,5 +1,11 @@
 import axios from 'axios';
 
+// ── Debug stub (remplacer par logger réel en prod) ──────────────────────────
+const debug = (...args) => {
+  if (process.env.DEBUG_FETCHER) console.log('[fetcher]', ...args);
+};
+
+// ── User-Agent rotation ────────────────────────────────────────────────────
 const USER_AGENTS = [
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
@@ -12,18 +18,108 @@ function randomUserAgent() {
   return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
 }
 
-function sleep(ms) {
+// ── Utilitaires ────────────────────────────────────────────────────────────
+export function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+// ── Domaines protégés (Cloudflare / anti-bot lourd) ────────────────────────
+const PROTECTED_DOMAINS = ['pappers.fr', 'societe.com', 'verif.com', 'score3.fr', 'manageo.fr'];
+
+export function isProtectedDomain(url) {
+  try {
+    const { hostname } = new URL(url);
+    return PROTECTED_DOMAINS.some(d => hostname === d || hostname.endsWith(`.${d}`));
+  } catch {
+    return false;
+  }
+}
+
+// ── Camofox (anti-bot bypass local) ────────────────────────────────────────
+const CAMOFOX_BASE = 'http://localhost:9377';
+const CAMOFOX_USER_ID = 'intelwatch';
+const CAMOFOX_SESSION_KEY = 'default';
+const CAMOFOX_WAIT_MS = 7000;
+
+export async function camofoxFetch(url, options = {}) {
+  const { timeout = 30000 } = options;
+
+  // Vérifier disponibilité Camofox
+  let healthCheck;
+  try {
+    healthCheck = await axios.get(`${CAMOFOX_BASE}/health`, { timeout: 2000 });
+  } catch {
+    debug('camofox indisponible sur', CAMOFOX_BASE);
+    throw new Error(`Camofox unavailable at ${CAMOFOX_BASE} — cannot bypass protection for ${url}`);
+  }
+
+  let tabId;
+  try {
+    // POST /tabs — ouvrir onglet navigateur
+    const createRes = await axios.post(`${CAMOFOX_BASE}/tabs`, {
+      userId: CAMOFOX_USER_ID,
+      sessionKey: CAMOFOX_SESSION_KEY,
+      url,
+    }, { timeout });
+
+    tabId = createRes.data?.tabId || createRes.data?.id;
+    if (!tabId) throw new Error('Camofox: no tabId returned from POST /tabs');
+
+    debug('camofox tab created:', tabId, '— waiting', CAMOFOX_WAIT_MS, 'ms');
+
+    // Attente résolution challenge CF
+    await sleep(CAMOFOX_WAIT_MS);
+
+    // GET /tabs/{tabId}/snapshot — récupérer HTML rendu
+    const snapRes = await axios.get(`${CAMOFOX_BASE}/tabs/${tabId}/snapshot`, {
+      params: { userId: CAMOFOX_USER_ID },
+      timeout,
+    });
+
+    // Wrapper dans un format compatible response Axios
+    return {
+      status: snapRes.status,
+      statusText: snapRes.statusText,
+      headers: snapRes.headers,
+      data: snapRes.data,
+      config: snapRes.config,
+      request: snapRes.request,
+      _camofox: true,
+    };
+  } finally {
+    // Toujours cleanup, même en cas d'erreur
+    if (tabId) {
+      try {
+        await axios.delete(`${CAMOFOX_BASE}/tabs/${tabId}`, {
+          params: { userId: CAMOFOX_USER_ID },
+          timeout: 5000,
+        });
+        debug('camofox tab cleaned up:', tabId);
+      } catch (err) {
+        debug('camofox cleanup failed for tab', tabId, err.message);
+      }
+    }
+  }
+}
+
+// ── Fetch principal (Axios + fallback Camofox) ─────────────────────────────
 export async function fetch(url, options = {}) {
   const {
     retries = 3,
     delay = 1500,
     timeout = 15000,
     headers = {},
+    forceCamofox = false,
   } = options;
 
+  // Mode force : court-circuiter Axios, aller direct Camofox
+  if (forceCamofox) {
+    return camofoxFetch(url, options);
+  }
+
+  // Domaine protégé connu : tentative Axios puis fallback si 403
+  const protected_ = isProtectedDomain(url);
+
   const config = {
     url,
     method: options.method || 'GET',
@@ -41,6 +137,8 @@ export async function fetch(url, options = {}) {
   };
 
   let lastError;
+  let needsCamofox = false;
+
   for (let attempt = 1; attempt <= retries; attempt++) {
     try {
       if (attempt > 1) {
@@ -59,6 +157,13 @@ export async function fetch(url, options = {}) {
         throw new Error(`Rate limited (429) after ${retries} attempts`);
       }
 
+      // 403 = signature Cloudflare → fallback Camofox
+      if (response.status === 403) {
+        debug('403 détecté pour', url, '— fallback camofox');
+        needsCamofox = true;
+        break;
+      }
+
       return response;
     } catch (err) {
       lastError = err;
@@ -68,9 +173,25 @@ export async function fetch(url, options = {}) {
     }
   }
 
-  throw lastError;
+  // Fallback Camofox si 403 ou domaine protégé (et Axios a échoué)
+  if (needsCamofox || (protected_ && lastError)) {
+    try {
+      return await camofoxFetch(url, options);
+    } catch (camofoxErr) {
+      // Camofox indisponible → propager l'erreur Axios originale
+      debug('camofox fallback échoué:', camofoxErr.message);
+      if (lastError) throw lastError;
+      throw camofoxErr;
+    }
+  }
+
+  if (lastError) throw lastError;
+
+  // Ne devrait jamais arriver, mais sécurité
+  throw new Error(`fetch failed for ${url}`);
 }
 
+// ── Fetch avec jitter ───────────────────────────────────────────────────────
 export async function fetchWithDelay(url, options = {}) {
   const minDelay = options.minDelay ?? 1000;
   const maxDelay = options.maxDelay ?? 2000;
@@ -78,5 +199,3 @@ export async function fetchWithDelay(url, options = {}) {
   await sleep(jitter);
   return fetch(url, options);
 }
-
-export { sleep };

package/src/utils/parser.js

@@ -89,9 +89,11 @@ export function extractPricing($, html) {
   const planKeywords = ['starter', 'basic', 'pro', 'professional', 'business', 'enterprise', 'free', 'premium', 'plus'];
   const plans = [];
   for (const kw of planKeywords) {
-    const regex = new RegExp(`${kw}[^\\n]*?\\$[\\d,]+`, 'gi');
-    const matches = html.match(regex) || [];
-    plans.push(...matches.slice(0, 2));
+    // Rechercher dans le texte propre (au lieu du code HTML raw) pour éviter de capturer du code source
+    const textContent = $.text().replace(/\s+/g, ' ');
+    const regex = new RegExp(`(?:^|\\s)${kw}\\s[^$€£]{0,50}?[$€£][\\d,.]+`, 'gi');
+    const matches = textContent.match(regex) || [];
+    plans.push(...matches.slice(0, 2).map(m => m.trim()));
   }
 
   return {

package/src/scrapers/brave-search.js

@@ -1,281 +0,0 @@
-import axios from 'axios';
-import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
-
-const BRAVE_API = 'https://api.search.brave.com/res/v1';
-
-/**
- * Search via Brave Search API — reliable, no rate limiting issues.
- * Uses BRAVE_API_KEY env var or falls back to config.
- */
-function getApiKey() {
-  return process.env.BRAVE_API_KEY || process.env.BRAVE_SEARCH_API_KEY || null;
-}
-
-/**
- * Web search via Brave
- */
-export async function braveWebSearch(query, options = {}) {
-  const apiKey = getApiKey();
-  if (!apiKey) return { results: [], error: 'No BRAVE_API_KEY set' };
-
-  try {
-    const params = {
-      q: query,
-      count: options.count || 20,
-      country: options.country || 'FR',
-      search_lang: options.lang || 'fr',
-      freshness: options.freshness || undefined, // 'pd' (day), 'pw' (week), 'pm' (month)
-    };
-
-    const resp = await axios.get(`${BRAVE_API}/web/search`, {
-      headers: { 'X-Subscription-Token': apiKey, 'Accept': 'application/json' },
-      params,
-      timeout: 15000,
-    });
-
-    const results = (resp.data.web?.results || []).map(r => ({
-      title: r.title,
-      url: r.url,
-      domain: r.meta_url?.hostname?.replace('www.', '') || new URL(r.url).hostname.replace('www.', ''),
-      snippet: r.description || '',
-      age: r.age || null,
-    }));
-
-    return { results, error: null };
-  } catch (err) {
-    return { results: [], error: err.message };
-  }
-}
-
-/**
- * News search via Brave
- */
-export async function braveNewsSearch(query, options = {}) {
-  const apiKey = getApiKey();
-  if (!apiKey) return { results: [], error: 'No BRAVE_API_KEY set' };
-
-  try {
-    const params = {
-      q: query,
-      count: options.count || 20,
-      country: options.country || 'FR',
-      search_lang: options.lang || 'fr',
-      freshness: options.freshness || 'pm', // last month by default
-    };
-
-    const resp = await axios.get(`${BRAVE_API}/news/search`, {
-      headers: { 'X-Subscription-Token': apiKey, 'Accept': 'application/json' },
-      params,
-      timeout: 15000,
-    });
-
-    const results = (resp.data.results || []).map(r => ({
-      title: r.title,
-      url: r.url,
-      domain: r.meta_url?.hostname?.replace('www.', '') || '',
-      snippet: r.description || '',
-      age: r.age || null,
-      source: r.meta_url?.hostname || '',
-    }));
-
-    return { results, error: null };
-  } catch (err) {
-    return { results: [], error: err.message };
-  }
-}
-
-/**
- * Full press & mentions search for a brand/company.
- * Combines news + web results, analyzes sentiment, categorizes.
- */
-export async function searchPressMentions(brandName, options = {}) {
-  const mentions = [];
-
-  // 1. News search
-  const news = await braveNewsSearch(brandName, { freshness: 'pm', ...options });
-  for (const r of news.results) {
-    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
-    mentions.push({
-      source: 'news',
-      url: r.url,
-      domain: r.domain || r.source,
-      title: r.title,
-      snippet: r.snippet?.substring(0, 300),
-      age: r.age,
-      sentiment: sentiment.label,
-      sentimentScore: sentiment.score,
-      category: categorizeMention(r.url, r.title, r.snippet),
-    });
-  }
-
-  // 2. Web search for recent mentions
-  await new Promise(r => setTimeout(r, 500));
-  const web = await braveWebSearch(`"${brandName}" avis OR actualité OR news`, { freshness: 'pw', ...options });
-  for (const r of web.results) {
-    if (mentions.some(m => m.url === r.url)) continue; // dedupe
-    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
-    mentions.push({
-      source: 'web',
-      url: r.url,
-      domain: r.domain,
-      title: r.title,
-      snippet: r.snippet?.substring(0, 300),
-      age: r.age,
-      sentiment: sentiment.label,
-      sentimentScore: sentiment.score,
-      category: categorizeMention(r.url, r.title, r.snippet),
-    });
-  }
-
-  // 3. Search for reviews specifically
-  await new Promise(r => setTimeout(r, 500));
-  const reviews = await braveWebSearch(`"${brandName}" avis clients trustpilot`, { count: 10, ...options });
-  for (const r of reviews.results) {
-    if (mentions.some(m => m.url === r.url)) continue;
-    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
-    if (/trustpilot|avis|review|capterra|g2\.com|glassdoor/.test(r.url + r.title)) {
-      mentions.push({
-        source: 'review',
-        url: r.url,
-        domain: r.domain,
-        title: r.title,
-        snippet: r.snippet?.substring(0, 300),
-        age: r.age,
-        sentiment: sentiment.label,
-        sentimentScore: sentiment.score,
-        category: 'review',
-      });
-    }
-  }
-
-  // ── Relevance filter: drop results that don't actually mention the brand ──
-  const brandLower = brandName.toLowerCase().trim();
-  const brandWords = brandLower.split(/\s+/);
-  const filtered = mentions.filter(m => {
-    const text = ((m.title || '') + ' ' + (m.snippet || '') + ' ' + (m.domain || '')).toLowerCase();
-    // Must contain the exact brand name OR all words of the brand
-    if (text.includes(brandLower)) return true;
-    if (brandWords.length > 1 && brandWords.every(w => text.includes(w))) return true;
-    // Fuzzy: allow 1 char difference for short names (e.g. "Endrix" vs "Endrick" should be EXCLUDED)
-    return false;
-  });
-
-  return {
-    brandName,
-    checkedAt: new Date().toISOString(),
-    mentions: filtered,
-    mentionCount: filtered.length,
-    unfilteredCount: mentions.length,
-    error: news.error || web.error || null,
-  };
-}
-
-/**
- * Search SERP rankings for a keyword
- */
-export async function searchKeywordRankings(keyword, options = {}) {
-  const search = await braveWebSearch(keyword, { count: 20, ...options });
-  
-  return search.results.map((r, i) => ({
-    position: i + 1,
-    url: r.url,
-    domain: r.domain,
-    title: r.title,
-    snippet: r.snippet,
-  }));
-}
-
-/**
- * Social media search via Brave — filters by platform.
- * platforms: array of 'twitter', 'reddit', 'linkedin'
- */
-export async function searchSocial(query, platforms = ['twitter', 'reddit', 'linkedin'], options = {}) {
-  const apiKey = getApiKey();
-  if (!apiKey) return { results: [], byPlatform: {}, error: 'No BRAVE_API_KEY set' };
-
-  const siteFilters = {
-    twitter: 'site:x.com OR site:twitter.com',
-    reddit: 'site:reddit.com',
-    linkedin: 'site:linkedin.com',
-  };
-
-  const siteQuery = platforms
-    .map(p => siteFilters[p])
-    .filter(Boolean)
-    .join(' OR ');
-
-  const fullQuery = `${query} (${siteQuery})`;
-
-  const search = await braveWebSearch(fullQuery, { count: options.count || 15, ...options });
-
-  const results = (search.results || []).map(r => {
-    let platform = 'other';
-    const urlLower = r.url.toLowerCase();
-    if (urlLower.includes('x.com') || urlLower.includes('twitter.com')) platform = 'twitter';
-    else if (urlLower.includes('reddit.com')) platform = 'reddit';
-    else if (urlLower.includes('linkedin.com')) platform = 'linkedin';
-    return { ...r, platform };
-  });
-
-  const byPlatform = {};
-  for (const r of results) {
-    if (!byPlatform[r.platform]) byPlatform[r.platform] = [];
-    byPlatform[r.platform].push(r);
-  }
-
-  return { results, byPlatform, error: search.error };
-}
-
-/**
- * Extract review ratings from search snippets
- */
-export function extractRatingsFromResults(results) {
-  const platforms = [];
-  
-  for (const r of results) {
-    const text = `${r.title} ${r.snippet}`.toLowerCase();
-    
-    // Trustpilot pattern
-    if (/trustpilot/.test(r.url) || /trustpilot/.test(text)) {
-      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|out of 5|stars?|étoiles?)/);
-      const countMatch = text.match(/([\d\s,.]+)\s*(?:avis|reviews?|évaluations?)/);
-      if (ratingMatch || countMatch) {
-        platforms.push({
-          name: 'Trustpilot',
-          url: r.url,
-          rating: ratingMatch ? parseFloat(ratingMatch[1].replace(',', '.')) : null,
-          reviewCount: countMatch ? countMatch[1].replace(/\s/g, '').replace(',', '') : null,
-        });
-      }
-    }
-
-    // Google reviews pattern
-    if (/google/.test(text) && /avis|review/.test(text)) {
-      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|stars?|étoiles?)/);
-      const countMatch = text.match(/([\d\s,.]+)\s*(?:avis|reviews?|évaluations?)/);
-      if (ratingMatch) {
-        platforms.push({
-          name: 'Google',
-          url: r.url,
-          rating: parseFloat(ratingMatch[1].replace(',', '.')),
-          reviewCount: countMatch ? countMatch[1].replace(/\s/g, '') : null,
-        });
-      }
-    }
-
-    // Glassdoor (employer reputation)
-    if (/glassdoor/.test(r.url)) {
-      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|stars?)/);
-      if (ratingMatch) {
-        platforms.push({
-          name: 'Glassdoor',
-          url: r.url,
-          rating: parseFloat(ratingMatch[1].replace(',', '.')),
-          reviewCount: null,
-        });
-      }
-    }
-  }
-
-  return platforms;
-}