intelwatch 1.0.0

package/src/report/markdown.js

@@ -0,0 +1,156 @@
+export function generateMarkdownReport(data) {
+  const lines = [];
+  const date = new Date(data.generatedAt).toLocaleString();
+
+  lines.push('# IntelWatch Intelligence Report');
+  lines.push(`\n_Generated: ${date}_\n`);
+  lines.push('---\n');
+
+  // Summary
+  const totalChanges =
+    data.competitors.reduce((s, c) => s + c.changes.length, 0) +
+    data.keywords.reduce((s, k) => s + k.changes.length, 0) +
+    data.brands.reduce((s, b) => s + b.changes.length, 0);
+
+  lines.push('## Summary\n');
+  lines.push(`| Category | Tracked | Changes |`);
+  lines.push(`|----------|---------|---------|`);
+  lines.push(`| Competitors | ${data.competitors.length} | ${data.competitors.reduce((s, c) => s + c.changes.length, 0)} |`);
+  lines.push(`| Keywords | ${data.keywords.length} | ${data.keywords.reduce((s, k) => s + k.changes.length, 0)} |`);
+  lines.push(`| Brands | ${data.brands.length} | ${data.brands.reduce((s, b) => s + b.changes.length, 0)} |`);
+  lines.push(`| **Total** | **${data.competitors.length + data.keywords.length + data.brands.length}** | **${totalChanges}** |`);
+  lines.push('');
+
+  // Competitor section
+  if (data.competitors.length > 0) {
+    lines.push('## Competitors\n');
+
+    // Threat level table
+    const sorted = [...data.competitors].sort((a, b) => b.threatScore - a.threatScore);
+    lines.push('### Threat Levels\n');
+    lines.push('| Competitor | Threat | Pages | Tech | Jobs | Changes |');
+    lines.push('|------------|--------|-------|------|------|---------|');
+    for (const { tracker, snapshot, changes, threatScore } of sorted) {
+      const threat = threatScore >= 8 ? '🔴 HIGH' : threatScore >= 4 ? '🟡 MED' : '🟢 LOW';
+      lines.push(
+        `| [${tracker.name || tracker.url}](${tracker.url}) | ${threat} (${threatScore}/10) | ${snapshot.pageCount || 0} | ${(snapshot.techStack || []).length} | ${snapshot.jobs?.estimatedOpenings || '?'} | ${changes.length} |`
+      );
+    }
+    lines.push('');
+
+    for (const { tracker, snapshot, changes, threatScore } of data.competitors) {
+      lines.push(`### ${tracker.name || tracker.url}`);
+      lines.push(`\n- **URL:** ${tracker.url}`);
+      lines.push(`- **Last checked:** ${new Date(snapshot.checkedAt).toLocaleString()}`);
+      lines.push(`- **Pages:** ${snapshot.pageCount || 0}`);
+      lines.push(`- **Threat score:** ${threatScore}/10`);
+
+      if (snapshot.techStack?.length > 0) {
+        const byCategory = {};
+        for (const t of snapshot.techStack) {
+          (byCategory[t.category] = byCategory[t.category] || []).push(t.name);
+        }
+        lines.push(`\n**Tech Stack:**`);
+        for (const [cat, names] of Object.entries(byCategory)) {
+          lines.push(`- ${cat}: ${names.join(', ')}`);
+        }
+      }
+
+      if (snapshot.pricing?.prices?.length > 0) {
+        lines.push(`\n**Pricing:** ${snapshot.pricing.prices.slice(0, 5).join(' | ')}`);
+      }
+
+      if (snapshot.jobs) {
+        lines.push(`\n**Jobs:** ~${snapshot.jobs.estimatedOpenings} open positions (${snapshot.jobs.url})`);
+      }
+
+      const metaPage = snapshot.keyPages?.['/'];
+      if (metaPage?.title) {
+        lines.push(`\n**Homepage title:** ${metaPage.title}`);
+      }
+
+      if (changes.length > 0) {
+        lines.push(`\n**Changes (${changes.length}):**`);
+        for (const c of changes) {
+          const emoji = c.type === 'new' ? '🟢' : c.type === 'removed' ? '🔴' : '🟡';
+          lines.push(`- ${emoji} [${c.field}] ${c.value}`);
+        }
+      }
+
+      lines.push('');
+    }
+  }
+
+  // Keyword section
+  if (data.keywords.length > 0) {
+    lines.push('## Keyword Rankings\n');
+
+    for (const { tracker, snapshot, changes } of data.keywords) {
+      lines.push(`### "${tracker.keyword}"`);
+      lines.push(`\n_Checked: ${new Date(snapshot.checkedAt).toLocaleString()}_\n`);
+
+      if (snapshot.results?.length > 0) {
+        lines.push('| # | Domain | Title |');
+        lines.push('|---|--------|-------|');
+        for (const r of snapshot.results.slice(0, 10)) {
+          const star = r.isFeaturedSnippet ? ' ⭐' : '';
+          lines.push(`| ${r.position} | ${r.domain}${star} | ${(r.title || '').slice(0, 60)} |`);
+        }
+        lines.push('');
+      }
+
+      if (changes.length > 0) {
+        lines.push('**Changes:**');
+        for (const c of changes) {
+          const emoji = c.type === 'new' ? '🟢' : c.type === 'removed' ? '🔴' : '🟡';
+          lines.push(`- ${emoji} ${c.value}`);
+        }
+      }
+
+      lines.push('');
+    }
+  }
+
+  // Brand section
+  if (data.brands.length > 0) {
+    lines.push('## Brand Mentions\n');
+
+    for (const { tracker, snapshot, changes } of data.brands) {
+      lines.push(`### "${tracker.brandName}"`);
+      lines.push(`\n_${snapshot.mentionCount || 0} mentions found — checked: ${new Date(snapshot.checkedAt).toLocaleString()}_\n`);
+
+      const mentions = snapshot.mentions || [];
+      const negative = mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative');
+
+      if (negative.length > 0) {
+        lines.push(`⚠️ **${negative.length} negative mention(s) detected:**`);
+        for (const m of negative.slice(0, 3)) {
+          lines.push(`- [${m.title?.slice(0, 60)}](${m.url}) — ${m.domain}`);
+        }
+        lines.push('');
+      }
+
+      if (mentions.length > 0) {
+        lines.push('**Recent mentions:**');
+        for (const m of mentions.slice(0, 5)) {
+          const sentEmoji = m.sentiment === 'positive' ? '😊' : m.sentiment === 'negative' ? '😞' : '😐';
+          lines.push(`- ${sentEmoji} [${(m.title || m.url).slice(0, 80)}](${m.url}) [${m.category}]`);
+        }
+      }
+
+      if (changes.length > 0) {
+        lines.push('\n**New mentions:**');
+        for (const c of changes.filter(c => c.field === 'mention')) {
+          lines.push(`- 🟢 ${c.value}`);
+        }
+      }
+
+      lines.push('');
+    }
+  }
+
+  lines.push('---');
+  lines.push('_Generated by [intelwatch](https://github.com/intelwatch/intelwatch) — competitive intelligence from the terminal_');
+
+  return lines.join('\n');
+}

package/src/scrapers/brave-search.js

@@ -0,0 +1,268 @@
+import axios from 'axios';
+import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
+
+const BRAVE_API = 'https://api.search.brave.com/res/v1';
+
+/**
+ * Search via Brave Search API — reliable, no rate limiting issues.
+ * Uses BRAVE_API_KEY env var or falls back to config.
+ */
+function getApiKey() {
+  return process.env.BRAVE_API_KEY || process.env.BRAVE_SEARCH_API_KEY || null;
+}
+
+/**
+ * Web search via Brave
+ */
+export async function braveWebSearch(query, options = {}) {
+  const apiKey = getApiKey();
+  if (!apiKey) return { results: [], error: 'No BRAVE_API_KEY set' };
+
+  try {
+    const params = {
+      q: query,
+      count: options.count || 20,
+      country: options.country || 'FR',
+      search_lang: options.lang || 'fr',
+      freshness: options.freshness || undefined, // 'pd' (day), 'pw' (week), 'pm' (month)
+    };
+
+    const resp = await axios.get(`${BRAVE_API}/web/search`, {
+      headers: { 'X-Subscription-Token': apiKey, 'Accept': 'application/json' },
+      params,
+      timeout: 15000,
+    });
+
+    const results = (resp.data.web?.results || []).map(r => ({
+      title: r.title,
+      url: r.url,
+      domain: r.meta_url?.hostname?.replace('www.', '') || new URL(r.url).hostname.replace('www.', ''),
+      snippet: r.description || '',
+      age: r.age || null,
+    }));
+
+    return { results, error: null };
+  } catch (err) {
+    return { results: [], error: err.message };
+  }
+}
+
+/**
+ * News search via Brave
+ */
+export async function braveNewsSearch(query, options = {}) {
+  const apiKey = getApiKey();
+  if (!apiKey) return { results: [], error: 'No BRAVE_API_KEY set' };
+
+  try {
+    const params = {
+      q: query,
+      count: options.count || 20,
+      country: options.country || 'FR',
+      search_lang: options.lang || 'fr',
+      freshness: options.freshness || 'pm', // last month by default
+    };
+
+    const resp = await axios.get(`${BRAVE_API}/news/search`, {
+      headers: { 'X-Subscription-Token': apiKey, 'Accept': 'application/json' },
+      params,
+      timeout: 15000,
+    });
+
+    const results = (resp.data.results || []).map(r => ({
+      title: r.title,
+      url: r.url,
+      domain: r.meta_url?.hostname?.replace('www.', '') || '',
+      snippet: r.description || '',
+      age: r.age || null,
+      source: r.meta_url?.hostname || '',
+    }));
+
+    return { results, error: null };
+  } catch (err) {
+    return { results: [], error: err.message };
+  }
+}
+
+/**
+ * Full press & mentions search for a brand/company.
+ * Combines news + web results, analyzes sentiment, categorizes.
+ */
+export async function searchPressMentions(brandName, options = {}) {
+  const mentions = [];
+
+  // 1. News search
+  const news = await braveNewsSearch(brandName, { freshness: 'pm', ...options });
+  for (const r of news.results) {
+    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
+    mentions.push({
+      source: 'news',
+      url: r.url,
+      domain: r.domain || r.source,
+      title: r.title,
+      snippet: r.snippet?.substring(0, 300),
+      age: r.age,
+      sentiment: sentiment.label,
+      sentimentScore: sentiment.score,
+      category: categorizeMention(r.url, r.title, r.snippet),
+    });
+  }
+
+  // 2. Web search for recent mentions
+  await new Promise(r => setTimeout(r, 500));
+  const web = await braveWebSearch(`"${brandName}" avis OR actualité OR news`, { freshness: 'pw', ...options });
+  for (const r of web.results) {
+    if (mentions.some(m => m.url === r.url)) continue; // dedupe
+    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
+    mentions.push({
+      source: 'web',
+      url: r.url,
+      domain: r.domain,
+      title: r.title,
+      snippet: r.snippet?.substring(0, 300),
+      age: r.age,
+      sentiment: sentiment.label,
+      sentimentScore: sentiment.score,
+      category: categorizeMention(r.url, r.title, r.snippet),
+    });
+  }
+
+  // 3. Search for reviews specifically
+  await new Promise(r => setTimeout(r, 500));
+  const reviews = await braveWebSearch(`"${brandName}" avis clients trustpilot`, { count: 10, ...options });
+  for (const r of reviews.results) {
+    if (mentions.some(m => m.url === r.url)) continue;
+    const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
+    if (/trustpilot|avis|review|capterra|g2\.com|glassdoor/.test(r.url + r.title)) {
+      mentions.push({
+        source: 'review',
+        url: r.url,
+        domain: r.domain,
+        title: r.title,
+        snippet: r.snippet?.substring(0, 300),
+        age: r.age,
+        sentiment: sentiment.label,
+        sentimentScore: sentiment.score,
+        category: 'review',
+      });
+    }
+  }
+
+  return {
+    brandName,
+    checkedAt: new Date().toISOString(),
+    mentions,
+    mentionCount: mentions.length,
+    error: news.error || web.error || null,
+  };
+}
+
+/**
+ * Search SERP rankings for a keyword
+ */
+export async function searchKeywordRankings(keyword, options = {}) {
+  const search = await braveWebSearch(keyword, { count: 20, ...options });
+  
+  return search.results.map((r, i) => ({
+    position: i + 1,
+    url: r.url,
+    domain: r.domain,
+    title: r.title,
+    snippet: r.snippet,
+  }));
+}
+
+/**
+ * Social media search via Brave — filters by platform.
+ * platforms: array of 'twitter', 'reddit', 'linkedin'
+ */
+export async function searchSocial(query, platforms = ['twitter', 'reddit', 'linkedin'], options = {}) {
+  const apiKey = getApiKey();
+  if (!apiKey) return { results: [], byPlatform: {}, error: 'No BRAVE_API_KEY set' };
+
+  const siteFilters = {
+    twitter: 'site:x.com OR site:twitter.com',
+    reddit: 'site:reddit.com',
+    linkedin: 'site:linkedin.com',
+  };
+
+  const siteQuery = platforms
+    .map(p => siteFilters[p])
+    .filter(Boolean)
+    .join(' OR ');
+
+  const fullQuery = `${query} (${siteQuery})`;
+
+  const search = await braveWebSearch(fullQuery, { count: options.count || 15, ...options });
+
+  const results = (search.results || []).map(r => {
+    let platform = 'other';
+    const urlLower = r.url.toLowerCase();
+    if (urlLower.includes('x.com') || urlLower.includes('twitter.com')) platform = 'twitter';
+    else if (urlLower.includes('reddit.com')) platform = 'reddit';
+    else if (urlLower.includes('linkedin.com')) platform = 'linkedin';
+    return { ...r, platform };
+  });
+
+  const byPlatform = {};
+  for (const r of results) {
+    if (!byPlatform[r.platform]) byPlatform[r.platform] = [];
+    byPlatform[r.platform].push(r);
+  }
+
+  return { results, byPlatform, error: search.error };
+}
+
+/**
+ * Extract review ratings from search snippets
+ */
+export function extractRatingsFromResults(results) {
+  const platforms = [];
+  
+  for (const r of results) {
+    const text = `${r.title} ${r.snippet}`.toLowerCase();
+    
+    // Trustpilot pattern
+    if (/trustpilot/.test(r.url) || /trustpilot/.test(text)) {
+      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|out of 5|stars?|étoiles?)/);
+      const countMatch = text.match(/([\d\s,.]+)\s*(?:avis|reviews?|évaluations?)/);
+      if (ratingMatch || countMatch) {
+        platforms.push({
+          name: 'Trustpilot',
+          url: r.url,
+          rating: ratingMatch ? parseFloat(ratingMatch[1].replace(',', '.')) : null,
+          reviewCount: countMatch ? countMatch[1].replace(/\s/g, '').replace(',', '') : null,
+        });
+      }
+    }
+
+    // Google reviews pattern
+    if (/google/.test(text) && /avis|review/.test(text)) {
+      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|stars?|étoiles?)/);
+      const countMatch = text.match(/([\d\s,.]+)\s*(?:avis|reviews?|évaluations?)/);
+      if (ratingMatch) {
+        platforms.push({
+          name: 'Google',
+          url: r.url,
+          rating: parseFloat(ratingMatch[1].replace(',', '.')),
+          reviewCount: countMatch ? countMatch[1].replace(/\s/g, '') : null,
+        });
+      }
+    }
+
+    // Glassdoor (employer reputation)
+    if (/glassdoor/.test(r.url)) {
+      const ratingMatch = text.match(/(\d[.,]\d)\s*(?:\/\s*5|sur\s*5|stars?)/);
+      if (ratingMatch) {
+        platforms.push({
+          name: 'Glassdoor',
+          url: r.url,
+          rating: parseFloat(ratingMatch[1].replace(',', '.')),
+          reviewCount: null,
+        });
+      }
+    }
+  }
+
+  return platforms;
+}

package/src/scrapers/google-news.js

@@ -0,0 +1,111 @@
+import { fetch } from '../utils/fetcher.js';
+import { load } from '../utils/parser.js';
+import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
+
+const GOOGLE_NEWS_BASE = 'https://www.google.com/search';
+const GOOGLE_SEARCH_BASE = 'https://www.google.com/search';
+
+export async function scrapeNewsMentions(brandName, options = {}) {
+  const mentions = [];
+
+  // Search Google News
+  try {
+    const newsParams = new URLSearchParams({
+      q: brandName,
+      tbm: 'nws',
+      num: '20',
+      hl: 'en',
+    });
+    const newsUrl = `${GOOGLE_NEWS_BASE}?${newsParams}`;
+    const response = await fetch(newsUrl, { retries: 3, delay: 2000 });
+
+    if (response.status === 200) {
+      const $ = load(response.data);
+
+      $('a[href]').each((_, el) => {
+        const href = $(el).attr('href') || '';
+        if (!href.startsWith('http') || href.includes('google.com')) return;
+
+        const title = $(el).text().trim();
+        if (title.length < 10) return;
+
+        try {
+          const url = new URL(href);
+          const domain = url.hostname.replace('www.', '');
+          const snippet = $(el).parent().text().replace(title, '').trim().slice(0, 200);
+          const sentiment = analyzeSentiment(title + ' ' + snippet);
+          const category = categorizeMention(href, title, snippet);
+
+          mentions.push({
+            source: 'google_news',
+            url: href,
+            domain,
+            title: title.slice(0, 200),
+            snippet: snippet.slice(0, 300),
+            sentiment: sentiment.label,
+            sentimentScore: sentiment.score,
+            category,
+            foundAt: new Date().toISOString(),
+          });
+        } catch {}
+      });
+    }
+  } catch (err) {
+    // Graceful degradation
+  }
+
+  // Also search recent web results (last 24h)
+  try {
+    await new Promise(r => setTimeout(r, 2000));
+    const webParams = new URLSearchParams({
+      q: brandName,
+      tbs: 'qdr:d',
+      num: '20',
+      hl: 'en',
+    });
+    const webUrl = `${GOOGLE_SEARCH_BASE}?${webParams}`;
+    const response = await fetch(webUrl, { retries: 3, delay: 2000 });
+
+    if (response.status === 200) {
+      const $ = load(response.data);
+
+      $('a[href]').each((_, el) => {
+        const href = $(el).attr('href') || '';
+        if (!href.startsWith('http') || href.includes('google.com')) return;
+
+        const title = $(el).find('h3').text().trim() || $(el).text().trim().slice(0, 100);
+        if (title.length < 10) return;
+
+        // Skip duplicates
+        if (mentions.some(m => m.url === href)) return;
+
+        try {
+          const url = new URL(href);
+          const domain = url.hostname.replace('www.', '');
+          const snippet = $(el).parent().text().replace(title, '').trim().slice(0, 200);
+          const sentiment = analyzeSentiment(title + ' ' + snippet);
+          const category = categorizeMention(href, title, snippet);
+
+          mentions.push({
+            source: 'google_web_24h',
+            url: href,
+            domain,
+            title: title.slice(0, 200),
+            snippet: snippet.slice(0, 300),
+            sentiment: sentiment.label,
+            sentimentScore: sentiment.score,
+            category,
+            foundAt: new Date().toISOString(),
+          });
+        } catch {}
+      });
+    }
+  } catch {}
+
+  return {
+    brandName,
+    checkedAt: new Date().toISOString(),
+    mentions: mentions.slice(0, 40),
+    mentionCount: mentions.length,
+  };
+}

package/src/scrapers/google.js

@@ -0,0 +1,113 @@
+import { fetch } from '../utils/fetcher.js';
+import { load } from '../utils/parser.js';
+
+const GOOGLE_BASE = 'https://www.google.com/search';
+
+export async function scrapeSerp(keyword, options = {}) {
+  const { num = 20, lang = 'en' } = options;
+
+  const params = new URLSearchParams({
+    q: keyword,
+    num: String(num),
+    hl: lang,
+    gl: 'us',
+  });
+
+  const url = `${GOOGLE_BASE}?${params}`;
+
+  let response;
+  try {
+    response = await fetch(url, {
+      retries: 3,
+      delay: 2000,
+      headers: {
+        'Accept': 'text/html,application/xhtml+xml',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Referer': 'https://www.google.com/',
+      },
+    });
+  } catch (err) {
+    throw new Error(`Google SERP fetch failed: ${err.message}`);
+  }
+
+  if (response.status === 429) {
+    throw new Error('Google rate limited (429). Try again later.');
+  }
+
+  if (response.status !== 200) {
+    throw new Error(`Google returned status ${response.status}`);
+  }
+
+  const $ = load(response.data);
+  const results = [];
+
+  // Parse organic results
+  $('div.g, div[data-sokoban-container]').each((i, el) => {
+    const titleEl = $(el).find('h3').first();
+    const linkEl = $(el).find('a[href]').first();
+    const snippetEl = $(el).find('[data-sncf], .VwiC3b, .s3v9rd, span[data-ved]').first();
+
+    const title = titleEl.text().trim();
+    const href = linkEl.attr('href') || '';
+    const snippet = snippetEl.text().trim();
+
+    if (!title || !href) return;
+
+    let cleanUrl = href;
+    if (href.startsWith('/url?')) {
+      const parsed = new URLSearchParams(href.slice(5));
+      cleanUrl = parsed.get('q') || href;
+    }
+
+    try {
+      const parsedUrl = new URL(cleanUrl);
+      results.push({
+        position: results.length + 1,
+        title,
+        url: cleanUrl,
+        domain: parsedUrl.hostname.replace('www.', ''),
+        snippet,
+        isFeaturedSnippet: false,
+      });
+    } catch {}
+  });
+
+  // Try alternate selectors if main ones didn't work
+  if (results.length === 0) {
+    $('a[href]').each((i, el) => {
+      const href = $(el).attr('href') || '';
+      if (!href.startsWith('http') || href.includes('google.com')) return;
+
+      const title = $(el).find('h3').text().trim() || $(el).text().trim().slice(0, 100);
+      if (!title) return;
+
+      try {
+        const parsedUrl = new URL(href);
+        if (results.length < num) {
+          results.push({
+            position: results.length + 1,
+            title,
+            url: href,
+            domain: parsedUrl.hostname.replace('www.', ''),
+            snippet: '',
+            isFeaturedSnippet: false,
+          });
+        }
+      } catch {}
+    });
+  }
+
+  // Check for featured snippet
+  const featuredSnippet = $('[data-attrid="wa:/description"], .xpdopen, .g-blk').first();
+  if (featuredSnippet.length && results.length > 0) {
+    results[0].isFeaturedSnippet = true;
+  }
+
+  return {
+    keyword,
+    checkedAt: new Date().toISOString(),
+    results: results.slice(0, num),
+    resultCount: results.length,
+    error: results.length === 0 ? 'No results parsed (Google may have changed layout or rate-limited)' : null,
+  };
+}