intelwatch 1.0.0

package/src/utils/parser.js

@@ -0,0 +1,110 @@
+import * as cheerio from 'cheerio';
+
+export function load(html) {
+  return cheerio.load(html);
+}
+
+export function extractLinks($, baseUrl) {
+  const links = new Set();
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (!href) return;
+    try {
+      const url = new URL(href, baseUrl);
+      if (url.hostname === new URL(baseUrl).hostname) {
+        links.add(url.href.split('#')[0].split('?')[0]);
+      }
+    } catch {}
+  });
+  return [...links];
+}
+
+export function extractMeta($) {
+  const meta = {};
+  meta.title = $('title').first().text().trim();
+  meta.description = $('meta[name="description"]').attr('content') || '';
+  meta.keywords = $('meta[name="keywords"]').attr('content') || '';
+  meta.ogTitle = $('meta[property="og:title"]').attr('content') || '';
+  meta.ogDescription = $('meta[property="og:description"]').attr('content') || '';
+  meta.canonical = $('link[rel="canonical"]').attr('href') || '';
+  meta.generator = $('meta[name="generator"]').attr('content') || '';
+  return meta;
+}
+
+export function extractSocialLinks($) {
+  const socialPatterns = {
+    twitter: /twitter\.com|x\.com/,
+    facebook: /facebook\.com/,
+    linkedin: /linkedin\.com/,
+    instagram: /instagram\.com/,
+    youtube: /youtube\.com/,
+    github: /github\.com/,
+    tiktok: /tiktok\.com/,
+    pinterest: /pinterest\.com/,
+  };
+
+  const socials = {};
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href') || '';
+    for (const [platform, pattern] of Object.entries(socialPatterns)) {
+      if (pattern.test(href) && !socials[platform]) {
+        socials[platform] = href;
+      }
+    }
+  });
+  return socials;
+}
+
+export function extractScripts($) {
+  const scripts = [];
+  $('script[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (src) scripts.push(src);
+  });
+  return scripts;
+}
+
+export function extractHtml($) {
+  return $.html();
+}
+
+export function textContent($, selector) {
+  return $(selector).text().trim();
+}
+
+export function extractPricing($, html) {
+  const pricePatterns = [
+    /\$\d+(?:\.\d{2})?(?:\s*\/\s*(?:mo|month|yr|year|user|seat))?/gi,
+    /€\d+(?:\.\d{2})?(?:\s*\/\s*(?:mo|month|yr|year|user|seat))?/gi,
+    /£\d+(?:\.\d{2})?(?:\s*\/\s*(?:mo|month|yr|year|user|seat))?/gi,
+    /\d+(?:\.\d{2})?\s*(?:USD|EUR|GBP)(?:\s*\/\s*(?:mo|month|yr|year))?/gi,
+  ];
+
+  const prices = new Set();
+  for (const pattern of pricePatterns) {
+    const matches = html.match(pattern) || [];
+    for (const m of matches) prices.add(m.trim());
+  }
+
+  const planKeywords = ['starter', 'basic', 'pro', 'professional', 'business', 'enterprise', 'free', 'premium', 'plus'];
+  const plans = [];
+  for (const kw of planKeywords) {
+    const regex = new RegExp(`${kw}[^\\n]*?\\$[\\d,]+`, 'gi');
+    const matches = html.match(regex) || [];
+    plans.push(...matches.slice(0, 2));
+  }
+
+  return {
+    prices: [...prices].slice(0, 20),
+    plans: [...new Set(plans)].slice(0, 10),
+    hash: simpleHash([...prices].sort().join('|')),
+  };
+}
+
+export function simpleHash(str) {
+  let h = 0;
+  for (let i = 0; i < str.length; i++) {
+    h = (Math.imul(31, h) + str.charCodeAt(i)) | 0;
+  }
+  return (h >>> 0).toString(16);
+}

package/src/utils/sentiment.js

@@ -0,0 +1,95 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const dataDir = join(__dirname, '../../data');
+
+let wordLists = null;
+
+function loadWordLists() {
+  if (wordLists) return wordLists;
+  wordLists = {
+    negativeEn: JSON.parse(readFileSync(join(dataDir, 'negative-words-en.json'), 'utf8')),
+    negativeFr: JSON.parse(readFileSync(join(dataDir, 'negative-words-fr.json'), 'utf8')),
+    positiveEn: JSON.parse(readFileSync(join(dataDir, 'positive-words-en.json'), 'utf8')),
+    positiveFr: JSON.parse(readFileSync(join(dataDir, 'positive-words-fr.json'), 'utf8')),
+  };
+  return wordLists;
+}
+
+export function analyzeSentiment(text, lang = 'auto') {
+  if (!text) return { score: 0, label: 'neutral', positiveHits: [], negativeHits: [] };
+
+  const lists = loadWordLists();
+  const lower = text.toLowerCase();
+
+  const negativeLists = lang === 'fr'
+    ? [lists.negativeFr]
+    : lang === 'en'
+    ? [lists.negativeEn]
+    : [lists.negativeEn, lists.negativeFr];
+
+  const positiveLists = lang === 'fr'
+    ? [lists.positiveFr]
+    : lang === 'en'
+    ? [lists.positiveEn]
+    : [lists.positiveEn, lists.positiveFr];
+
+  const negativeHits = [];
+  for (const list of negativeLists) {
+    for (const word of list) {
+      if (lower.includes(word.toLowerCase()) && !negativeHits.includes(word)) {
+        negativeHits.push(word);
+      }
+    }
+  }
+
+  const positiveHits = [];
+  for (const list of positiveLists) {
+    for (const word of list) {
+      if (lower.includes(word.toLowerCase()) && !positiveHits.includes(word)) {
+        positiveHits.push(word);
+      }
+    }
+  }
+
+  const score = positiveHits.length - negativeHits.length;
+  let label;
+  if (score > 1) label = 'positive';
+  else if (score < -1) label = 'negative';
+  else if (negativeHits.length > 0) label = 'slightly_negative';
+  else if (positiveHits.length > 0) label = 'slightly_positive';
+  else label = 'neutral';
+
+  return { score, label, positiveHits: positiveHits.slice(0, 5), negativeHits: negativeHits.slice(0, 5) };
+}
+
+export function sentimentEmoji(label) {
+  switch (label) {
+    case 'positive': return '😊';
+    case 'slightly_positive': return '🙂';
+    case 'neutral': return '😐';
+    case 'slightly_negative': return '😕';
+    case 'negative': return '😞';
+    default: return '❓';
+  }
+}
+
+export function categorizeMention(url, title, snippet) {
+  const text = `${url} ${title} ${snippet}`.toLowerCase();
+
+  if (/techcrunch|wired|reuters|bloomberg|bbc|forbes|businessinsider|wsj|nytimes|lemonde|lefigaro/.test(text)) {
+    return 'press';
+  }
+  if (/reddit|hacker news|news\.ycombinator|forum|community|discussion|ask\./.test(text)) {
+    return 'forum';
+  }
+  if (/twitter|x\.com|linkedin|facebook|instagram|tiktok|youtube/.test(text)) {
+    return 'social';
+  }
+  if (/trustpilot|g2\.com|capterra|getapp|review|avis|rating/.test(text)) {
+    return 'review';
+  }
+  return 'blog';
+}

package/src/utils/tech-detect.js

@@ -0,0 +1,94 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const signaturesPath = join(__dirname, '../../data/tech-signatures.json');
+
+let signatures = null;
+
+function loadSignatures() {
+  if (signatures) return signatures;
+  signatures = JSON.parse(readFileSync(signaturesPath, 'utf8')).technologies;
+  return signatures;
+}
+
+export function detectTechnologies(html, headers = {}, url = '') {
+  const techs = loadSignatures();
+  const detected = [];
+
+  const lowerHtml = html.toLowerCase();
+  const lowerHeaders = Object.fromEntries(
+    Object.entries(headers).map(([k, v]) => [k.toLowerCase(), (Array.isArray(v) ? v.join(', ') : String(v || '')).toLowerCase()])
+  );
+
+  for (const tech of techs) {
+    let found = false;
+
+    // Check headers
+    for (const hCheck of (tech.checks.headers || [])) {
+      const headerVal = lowerHeaders[hCheck.name.toLowerCase()] || '';
+      if (headerVal && new RegExp(hCheck.pattern, 'i').test(headerVal)) {
+        found = true;
+        break;
+      }
+    }
+
+    // Check meta tags (generator etc)
+    if (!found) {
+      for (const mCheck of (tech.checks.meta || [])) {
+        const pattern = new RegExp(mCheck.pattern, 'i');
+        if (pattern.test(lowerHtml)) {
+          found = true;
+          break;
+        }
+      }
+    }
+
+    // Check script sources
+    if (!found) {
+      for (const scriptPattern of (tech.checks.scripts || [])) {
+        if (lowerHtml.includes(scriptPattern.toLowerCase())) {
+          found = true;
+          break;
+        }
+      }
+    }
+
+    // Check HTML patterns
+    if (!found) {
+      for (const htmlPattern of (tech.checks.html || [])) {
+        if (lowerHtml.includes(htmlPattern.toLowerCase())) {
+          found = true;
+          break;
+        }
+      }
+    }
+
+    // Check known paths in URL
+    if (!found && url) {
+      for (const pathPattern of (tech.checks.paths || [])) {
+        if (url.includes(pathPattern)) {
+          found = true;
+          break;
+        }
+      }
+    }
+
+    if (found) {
+      detected.push({ name: tech.name, category: tech.category });
+    }
+  }
+
+  return detected;
+}
+
+export function diffTechStacks(prev, curr) {
+  const prevNames = new Set(prev.map(t => t.name));
+  const currNames = new Set(curr.map(t => t.name));
+
+  const added = curr.filter(t => !prevNames.has(t.name));
+  const removed = prev.filter(t => !currNames.has(t.name));
+
+  return { added, removed };
+}