intelwatch 1.0.0

package/src/scrapers/pappers.js

@@ -0,0 +1,119 @@
+import axios from 'axios';
+
+const PAPPERS_API = 'https://api.pappers.fr/v1';
+
+function getApiKey() {
+  return process.env.PAPPERS_API_KEY || null;
+}
+
+export function hasPappersKey() {
+  return !!getApiKey();
+}
+
+/**
+ * Search companies by name on Pappers
+ */
+export async function pappersSearchByName(name, options = {}) {
+  const apiKey = getApiKey();
+  if (!apiKey) return { results: [], error: 'No PAPPERS_API_KEY set' };
+
+  try {
+    const resp = await axios.get(`${PAPPERS_API}/recherche`, {
+      params: {
+        api_token: apiKey,
+        q: name,
+        par_page: options.count || 5,
+      },
+      timeout: 10000,
+    });
+    return { results: resp.data.resultats || [], error: null };
+  } catch (err) {
+    return { results: [], error: err.message };
+  }
+}
+
+/**
+ * Get company details by SIREN
+ */
+export async function pappersGetBySiren(siren) {
+  const apiKey = getApiKey();
+  if (!apiKey) return { data: null, error: 'No PAPPERS_API_KEY set' };
+
+  try {
+    const resp = await axios.get(`${PAPPERS_API}/entreprise`, {
+      params: { api_token: apiKey, siren },
+      timeout: 10000,
+    });
+    return { data: resp.data, error: null };
+  } catch (err) {
+    return { data: null, error: err.message };
+  }
+}
+
+/**
+ * Main lookup: search by name, then fetch detail by SIREN.
+ * Returns null if no API key or no result found.
+ */
+export async function pappersLookup(companyName) {
+  if (!getApiKey()) return null;
+
+  const search = await pappersSearchByName(companyName);
+  if (search.error || search.results.length === 0) return null;
+
+  const top = search.results[0];
+  const siren = top.siren;
+
+  if (!siren) return formatPappersResult(top);
+
+  const detail = await pappersGetBySiren(siren);
+  if (detail.error || !detail.data) return formatPappersResult(top);
+
+  return formatPappersDetail(detail.data);
+}
+
+function formatPappersResult(r) {
+  return {
+    siren: r.siren,
+    siret: r.siret,
+    name: r.nom_entreprise || r.denomination,
+    dateCreation: r.date_creation,
+    nafCode: r.code_naf,
+    nafLabel: r.libelle_code_naf,
+    city: r.siege?.ville,
+    postalCode: r.siege?.code_postal,
+    effectifs: null,
+    ca: null,
+    caYear: null,
+    resultat: null,
+    dirigeants: [],
+    formeJuridique: r.forme_juridique || null,
+  };
+}
+
+function formatPappersDetail(d) {
+  const dirigeants = (d.dirigeants || []).slice(0, 5).map(p => ({
+    nom: p.nom,
+    prenom: p.prenom,
+    role: p.fonction,
+    dateNomination: p.date_prise_de_poste,
+  }));
+
+  const lastFin = (d.finances || [])[0] || {};
+
+  return {
+    siren: d.siren,
+    siret: d.siege?.siret,
+    name: d.nom_entreprise || d.denomination,
+    dateCreation: d.date_creation,
+    nafCode: d.code_naf,
+    nafLabel: d.libelle_code_naf,
+    city: d.siege?.ville,
+    postalCode: d.siege?.code_postal,
+    effectifs: d.tranche_effectif || d.effectif || null,
+    ca: lastFin.chiffre_affaires || null,
+    caYear: lastFin.annee || null,
+    resultat: lastFin.resultat || null,
+    dirigeants,
+    formeJuridique: d.forme_juridique || null,
+  };
+}

package/src/scrapers/site-analyzer.js

@@ -0,0 +1,252 @@
+import { fetch } from '../utils/fetcher.js';
+import { load, extractLinks, extractMeta, extractSocialLinks, extractScripts, extractPricing, simpleHash } from '../utils/parser.js';
+import { detectTechnologies } from '../utils/tech-detect.js';
+
+/**
+ * Deep site analysis — captures maximum intelligence on first check.
+ */
+export async function analyzeSite(url) {
+  const result = {
+    url,
+    checkedAt: new Date().toISOString(),
+    status: 'ok',
+    error: null,
+    meta: {},
+    techStack: [],
+    socialLinks: {},
+    links: [],
+    pageCount: 0,
+    pricing: null,
+    jobs: null,
+    keyPages: {},        // title/desc snapshots of important pages
+    contentStats: null,  // blog/content activity
+    seoSignals: null,    // basic SEO health indicators
+    performance: null,   // response time, size
+    security: null,      // basic security signals
+  };
+
+  try {
+    const startTime = Date.now();
+    const response = await fetch(url, { retries: 3, delay: 1000 });
+    const loadTime = Date.now() - startTime;
+    const html = response.data;
+    const headers = response.headers || {};
+
+    const $ = load(html);
+
+    result.meta = extractMeta($);
+    result.techStack = detectTechnologies(html, headers, url);
+    result.socialLinks = extractSocialLinks($);
+    result.links = extractLinks($, url);
+    result.pageCount = result.links.length;
+
+    // --- Performance snapshot ---
+    result.performance = {
+      responseTimeMs: loadTime,
+      htmlSizeKB: Math.round(html.length / 1024),
+      compressed: !!(headers['content-encoding']),
+      http2: (headers[':status'] || headers['http-version'] || '').includes('2'),
+    };
+
+    // --- Security signals ---
+    result.security = {
+      https: url.startsWith('https'),
+      hsts: !!(headers['strict-transport-security']),
+      xFrameOptions: !!(headers['x-frame-options']),
+      csp: !!(headers['content-security-policy']),
+      xContentType: !!(headers['x-content-type-options']),
+    };
+
+    // --- SEO signals from homepage ---
+    result.seoSignals = {
+      hasTitle: !!result.meta.title,
+      titleLength: (result.meta.title || '').length,
+      hasDescription: !!result.meta.description,
+      descriptionLength: (result.meta.description || '').length,
+      hasCanonical: !!result.meta.canonical,
+      hasOgTags: !!(result.meta.ogTitle),
+      h1Count: $('h1').length,
+      h2Count: $('h2').length,
+      imgCount: $('img').length,
+      imgWithoutAlt: $('img:not([alt]), img[alt=""]').length,
+      wordCount: $.text().replace(/\s+/g, ' ').trim().split(/\s+/).length,
+      generator: result.meta.generator || null,
+    };
+
+    // --- Crawl key pages for deeper intel ---
+    const keyPagePaths = [
+      { path: '/', label: 'homepage' },
+      { path: '/pricing', label: 'pricing' },
+      { path: '/tarifs', label: 'pricing' },
+      { path: '/plans', label: 'pricing' },
+      { path: '/about', label: 'about' },
+      { path: '/a-propos', label: 'about' },
+      { path: '/qui-sommes-nous', label: 'about' },
+      { path: '/blog', label: 'blog' },
+      { path: '/actualites', label: 'blog' },
+      { path: '/news', label: 'blog' },
+      { path: '/contact', label: 'contact' },
+      { path: '/jobs', label: 'jobs' },
+      { path: '/careers', label: 'jobs' },
+      { path: '/recrutement', label: 'jobs' },
+      { path: '/emploi', label: 'jobs' },
+      { path: '/rejoignez-nous', label: 'jobs' },
+      { path: '/produits', label: 'products' },
+      { path: '/products', label: 'products' },
+      { path: '/services', label: 'services' },
+      { path: '/solutions', label: 'services' },
+      { path: '/clients', label: 'clients' },
+      { path: '/references', label: 'clients' },
+      { path: '/temoignages', label: 'testimonials' },
+      { path: '/testimonials', label: 'testimonials' },
+    ];
+
+    const foundPages = {};
+    const jobKeywords = [
+      'engineer', 'developer', 'développeur', 'manager', 'designer',
+      'analyst', 'analyste', 'sales', 'commercial', 'marketing',
+      'support', 'consultant', 'chef de projet', 'product', 'data',
+      'devops', 'fullstack', 'frontend', 'backend', 'stage', 'alternance',
+      'cdi', 'cdd', 'freelance', 'ingénieur', 'responsable', 'directeur'
+    ];
+
+    for (const { path, label } of keyPagePaths) {
+      const pageUrl = new URL(path, url).href;
+      // Check if this page exists in discovered links OR try it anyway for key pages
+      const isLinked = result.links.some(l => l === pageUrl || l.endsWith(path));
+      
+      if (!isLinked && !['homepage', 'pricing', 'jobs', 'blog'].includes(label)) continue;
+      if (foundPages[label] && label !== 'jobs') continue; // already found one for this category
+
+      try {
+        await new Promise(r => setTimeout(r, 800));
+        const pr = await fetch(pageUrl, { retries: 1, delay: 500 });
+        if (pr.status === 200 || pr.status === 301 || pr.status === 302) {
+          const p$ = load(pr.data);
+          const pageMeta = extractMeta(p$);
+          
+          foundPages[label] = {
+            url: pageUrl,
+            title: pageMeta.title,
+            description: pageMeta.description,
+            hash: simpleHash(pageMeta.title + pageMeta.description + p$.text().substring(0, 500)),
+          };
+
+          // --- Extract pricing info ---
+          if (label === 'pricing') {
+            result.pricing = extractPricing(p$, pr.data);
+            result.pricing.url = pageUrl;
+            result.pricing.pageTitle = pageMeta.title;
+          }
+
+          // --- Extract job listings ---
+          if (label === 'jobs') {
+            let jobCount = 0;
+            const jobTitles = [];
+            p$('h1,h2,h3,h4,li,article,.job,.position,.offer,.offre').each((_, el) => {
+              const text = p$(el).text().toLowerCase().trim();
+              if (text.length > 10 && text.length < 200 && jobKeywords.some(k => text.includes(k))) {
+                jobCount++;
+                if (jobTitles.length < 15) jobTitles.push(p$(el).text().trim().substring(0, 100));
+              }
+            });
+            result.jobs = {
+              url: pageUrl,
+              estimatedOpenings: jobCount,
+              titles: [...new Set(jobTitles)],
+              pageTitle: pageMeta.title,
+            };
+          }
+
+          // --- Extract blog/content activity ---
+          if (label === 'blog') {
+            const articles = [];
+            p$('article, .post, .entry, .blog-post, .article-item').each((_, el) => {
+              const title = p$(el).find('h2,h3,.title,.entry-title').first().text().trim();
+              const link = p$(el).find('a').first().attr('href');
+              const date = p$(el).find('time,.date,.published,.post-date').first().text().trim();
+              if (title) articles.push({ title: title.substring(0, 120), link, date });
+            });
+            
+            // Fallback: look for h2/h3 links in blog page
+            if (articles.length === 0) {
+              p$('h2 a, h3 a').each((_, el) => {
+                const title = p$(el).text().trim();
+                const link = p$(el).attr('href');
+                if (title && title.length > 10) articles.push({ title: title.substring(0, 120), link });
+              });
+            }
+
+            result.contentStats = {
+              url: pageUrl,
+              recentArticles: articles.slice(0, 10),
+              articleCount: articles.length,
+              pageTitle: pageMeta.title,
+            };
+          }
+        }
+      } catch {}
+    }
+
+    result.keyPages = foundPages;
+
+    // --- If no pricing found from key pages, check homepage ---
+    if (!result.pricing) {
+      const pageText = $.text();
+      if (/\$\d+|€\d+|£\d+/.test(pageText) && /plan|pricing|price|tarif/.test(pageText.toLowerCase())) {
+        result.pricing = extractPricing($, html);
+        result.pricing.url = url;
+      }
+    }
+
+    // --- Discover additional pages by crawling 1 level deep ---
+    // Pick up to 5 important-looking internal links to crawl
+    const importantPatterns = ['/produit', '/product', '/service', '/solution', '/offre', '/promo', '/categor'];
+    const extraPages = result.links
+      .filter(l => importantPatterns.some(p => l.toLowerCase().includes(p)))
+      .slice(0, 5);
+
+    for (const extraUrl of extraPages) {
+      try {
+        await new Promise(r => setTimeout(r, 600));
+        const er = await fetch(extraUrl, { retries: 1, delay: 300 });
+        if (er.status === 200) {
+          const e$ = load(er.data);
+          const extraLinks = extractLinks(e$, url);
+          for (const el of extraLinks) {
+            if (!result.links.includes(el)) result.links.push(el);
+          }
+        }
+      } catch {}
+    }
+    
+    result.pageCount = result.links.length;
+
+  } catch (err) {
+    result.status = 'error';
+    result.error = err.message;
+  }
+
+  return result;
+}
+
+export async function analyzeKeyPages(url, pages = ['/', '/about', '/pricing']) {
+  const results = {};
+  for (const page of pages) {
+    try {
+      const pageUrl = new URL(page, url).href;
+      await new Promise(r => setTimeout(r, 1500));
+      const response = await fetch(pageUrl);
+      if (response.status === 200) {
+        const $ = load(response.data);
+        const meta = extractMeta($);
+        results[page] = {
+          title: meta.title,
+          description: meta.description,
+          hash: simpleHash(meta.title + meta.description),
+        };
+      }
+    } catch {}
+  }
+  return results;
+}

package/src/storage.js

@@ -0,0 +1,168 @@
+import { mkdirSync, readFileSync, writeFileSync, readdirSync, existsSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { randomUUID } from 'crypto';
+
+const BASE_DIR = join(homedir(), '.intelwatch');
+const TRACKERS_FILE = join(BASE_DIR, 'trackers.json');
+const SNAPSHOTS_DIR = join(BASE_DIR, 'snapshots');
+const REPORTS_DIR = join(BASE_DIR, 'reports');
+
+export function ensureDirectories() {
+  for (const dir of [BASE_DIR, SNAPSHOTS_DIR, REPORTS_DIR]) {
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+export function loadTrackers() {
+  ensureDirectories();
+  if (!existsSync(TRACKERS_FILE)) {
+    return [];
+  }
+  try {
+    return JSON.parse(readFileSync(TRACKERS_FILE, 'utf8'));
+  } catch {
+    return [];
+  }
+}
+
+export function saveTrackers(trackers) {
+  ensureDirectories();
+  writeFileSync(TRACKERS_FILE, JSON.stringify(trackers, null, 2), 'utf8');
+}
+
+export function createTracker(type, data) {
+  const trackers = loadTrackers();
+  const id = generateId(type, data);
+
+  const existing = trackers.find(t => t.id === id);
+  if (existing) {
+    return { tracker: existing, created: false };
+  }
+
+  const tracker = {
+    id,
+    type,
+    createdAt: new Date().toISOString(),
+    lastCheckedAt: null,
+    status: 'active',
+    checkCount: 0,
+    ...data,
+  };
+
+  trackers.push(tracker);
+  saveTrackers(trackers);
+  return { tracker, created: true };
+}
+
+export function getTracker(id) {
+  const trackers = loadTrackers();
+  return trackers.find(t => t.id === id) || null;
+}
+
+export function updateTracker(id, updates) {
+  const trackers = loadTrackers();
+  const idx = trackers.findIndex(t => t.id === id);
+  if (idx === -1) throw new Error(`Tracker not found: ${id}`);
+  trackers[idx] = { ...trackers[idx], ...updates };
+  saveTrackers(trackers);
+  return trackers[idx];
+}
+
+export function removeTracker(id) {
+  const trackers = loadTrackers();
+  const idx = trackers.findIndex(t => t.id === id);
+  if (idx === -1) throw new Error(`Tracker not found: ${id}`);
+  const removed = trackers.splice(idx, 1)[0];
+  saveTrackers(trackers);
+  return removed;
+}
+
+export function saveSnapshot(trackerId, snapshot) {
+  ensureDirectories();
+  const date = new Date().toISOString().slice(0, 10);
+  const filename = `${trackerId}-${date}-${Date.now()}.json`;
+  const filepath = join(SNAPSHOTS_DIR, filename);
+  writeFileSync(filepath, JSON.stringify(snapshot, null, 2), 'utf8');
+  return filepath;
+}
+
+export function loadLatestSnapshot(trackerId) {
+  const snapshots = listSnapshots(trackerId);
+  if (snapshots.length === 0) return null;
+  const latest = snapshots[snapshots.length - 1];
+  return loadSnapshot(latest.filepath);
+}
+
+export function loadSnapshot(filepath) {
+  try {
+    return JSON.parse(readFileSync(filepath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+export function listSnapshots(trackerId, limit = null) {
+  ensureDirectories();
+  const files = readdirSync(SNAPSHOTS_DIR)
+    .filter(f => f.startsWith(trackerId + '-') && f.endsWith('.json'))
+    .sort();
+
+  const snapshots = files.map(f => {
+    const filepath = join(SNAPSHOTS_DIR, f);
+    const parts = f.replace('.json', '').split('-');
+    // Format: trackerId-YYYY-MM-DD-timestamp.json
+    // trackerId may contain dashes, so parse from the end
+    const timestamp = parseInt(parts[parts.length - 1]);
+    return { filename: f, filepath, timestamp, date: parts.slice(-4, -1).join('-') };
+  }).sort((a, b) => a.timestamp - b.timestamp);
+
+  if (limit) return snapshots.slice(-limit);
+  return snapshots;
+}
+
+export function loadSnapshotByDate(trackerId, targetDate) {
+  const snapshots = listSnapshots(trackerId);
+  // Find closest snapshot to targetDate
+  const target = new Date(targetDate).getTime();
+  let closest = null;
+  let minDiff = Infinity;
+
+  for (const s of snapshots) {
+    const diff = Math.abs(s.timestamp - target);
+    if (diff < minDiff) {
+      minDiff = diff;
+      closest = s;
+    }
+  }
+
+  if (!closest) return null;
+  return loadSnapshot(closest.filepath);
+}
+
+export function saveReport(filename, content) {
+  ensureDirectories();
+  const filepath = join(REPORTS_DIR, filename);
+  writeFileSync(filepath, content, 'utf8');
+  return filepath;
+}
+
+function generateId(type, data) {
+  const base = type === 'competitor' ? slugify(data.url)
+    : type === 'keyword' ? 'kw-' + slugify(data.keyword)
+    : type === 'person' ? 'person-' + slugify(data.personName + (data.org ? '-' + data.org : ''))
+    : 'brand-' + slugify(data.brandName);
+  return base.slice(0, 50);
+}
+
+function slugify(str) {
+  return str
+    .toLowerCase()
+    .replace(/https?:\/\//g, '')
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+export { BASE_DIR, SNAPSHOTS_DIR, REPORTS_DIR };

package/src/trackers/brand.js

@@ -0,0 +1,76 @@
+import { scrapeNewsMentions } from '../scrapers/google-news.js';
+
+export async function runBrandCheck(tracker) {
+  const { brandName } = tracker;
+
+  const mentionData = await scrapeNewsMentions(brandName);
+
+  return {
+    type: 'brand',
+    trackerId: tracker.id,
+    brandName,
+    checkedAt: new Date().toISOString(),
+    mentions: mentionData.mentions,
+    mentionCount: mentionData.mentionCount,
+    error: mentionData.error || null,
+  };
+}
+
+export function diffBrandSnapshots(prev, curr) {
+  const changes = [];
+
+  if (!prev) {
+    changes.push({
+      type: 'new',
+      field: 'tracker',
+      value: `Initial snapshot — ${curr.mentionCount} mentions found`,
+    });
+    return changes;
+  }
+
+  const prevUrls = new Set((prev.mentions || []).map(m => m.url));
+
+  for (const mention of (curr.mentions || [])) {
+    if (!prevUrls.has(mention.url)) {
+      const sentLabel = mention.sentiment === 'negative' || mention.sentiment === 'slightly_negative'
+        ? '⚠ ' : '';
+      changes.push({
+        type: 'new',
+        field: 'mention',
+        value: `${sentLabel}[${mention.category}] ${mention.title?.slice(0, 80)} — ${mention.domain}`,
+        mention,
+      });
+    }
+  }
+
+  const prevCount = prev.mentionCount || 0;
+  const currCount = curr.mentionCount || 0;
+  if (currCount !== prevCount) {
+    changes.push({
+      type: 'changed',
+      field: 'mentionCount',
+      value: `Mention count: ${prevCount} → ${currCount}`,
+    });
+  }
+
+  return changes;
+}
+
+export function getMentionSummary(snapshot) {
+  const mentions = snapshot.mentions || [];
+  const byCategory = {};
+  const bySentiment = {};
+
+  for (const m of mentions) {
+    byCategory[m.category] = (byCategory[m.category] || 0) + 1;
+    bySentiment[m.sentiment] = (bySentiment[m.sentiment] || 0) + 1;
+  }
+
+  return {
+    total: mentions.length,
+    byCategory,
+    bySentiment,
+    negativeMentions: mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative'),
+    positiveMentions: mentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive'),
+  };
+}