intelwatch 1.2.0 → 1.3.2

package/src/scrapers/reddit-hn.js

@@ -0,0 +1,161 @@
+/**
+ * Reddit & Hacker News scrapers for brand/keyword mentions.
+ *
+ * Uses public JSON APIs (no auth required):
+ * - Reddit: https://www.reddit.com/search.json?q=<query>
+ * - HN (Algolia): https://hn.algolia.com/api/v1/search?query=<query>
+ */
+
+import { fetch as fetchWithRetry } from '../utils/fetcher.js';
+import { getLimits, isPro } from '../license.js';
+
+const REDDIT_SEARCH_URL = 'https://www.reddit.com/search.json';
+const HN_SEARCH_URL = 'https://hn.algolia.com/api/v1/search';
+
+// ── Reddit ───────────────────────────────────────────────────────────────────
+
+/**
+ * Search Reddit for mentions of a brand/keyword.
+ * @param {string} query
+ * @param {{ limit?: number, sort?: string, timeFilter?: string }} options
+ * @returns {Promise<Array<{ title: string, url: string, subreddit: string, score: number, numComments: number, author: string, createdAt: string, selftext: string, domain: string, source: string }>>}
+ */
+export async function searchReddit(query, options = {}) {
+  // Pro-only: Reddit scraping requires a license
+  if (!isPro()) {
+    if (process.env.DEBUG) {
+      console.error('[reddit] Skipped — Pro license required');
+    }
+    return [];
+  }
+
+  const limits = getLimits();
+  const { limit = limits.redditMaxResults, sort = 'relevance', timeFilter = 'month' } = options;
+
+  try {
+    const params = new URLSearchParams({
+      q: query,
+      limit: String(Math.min(limit, 100)),
+      sort,
+      t: timeFilter,
+      type: 'link',
+    });
+
+    const url = `${REDDIT_SEARCH_URL}?${params}`;
+    const resp = await fetchWithRetry(url, {
+      headers: {
+        'User-Agent': 'intelwatch/1.2.0 (competitive intelligence CLI)',
+        'Accept': 'application/json',
+      },
+      timeout: 15000,
+    });
+
+    if (!resp || resp.status >= 400) {
+      return [];
+    }
+
+    const data = typeof resp.data === 'string' ? JSON.parse(resp.data) : resp.data;
+    const posts = data?.data?.children || [];
+
+    return posts.slice(0, limits.redditMaxResults).map(({ data: post }) => ({
+      title: post.title || '',
+      url: `https://www.reddit.com${post.permalink}`,
+      subreddit: post.subreddit_name_prefixed || `r/${post.subreddit}`,
+      score: post.score || 0,
+      numComments: post.num_comments || 0,
+      author: post.author || '[deleted]',
+      createdAt: new Date((post.created_utc || 0) * 1000).toISOString(),
+      selftext: (post.selftext || '').slice(0, 500),
+      domain: post.domain || '',
+      source: 'reddit',
+    }));
+  } catch (err) {
+    // Silently fail — Reddit rate-limits aggressively
+    if (process.env.DEBUG) {
+      console.error(`[reddit] Search failed: ${err.message}`);
+    }
+    return [];
+  }
+}
+
+// ── Hacker News ──────────────────────────────────────────────────────────────
+
+/**
+ * Search Hacker News via Algolia API.
+ * @param {string} query
+ * @param {{ limit?: number, tags?: string }} options
+ * @returns {Promise<Array<{ title: string, url: string, hnUrl: string, points: number, numComments: number, author: string, createdAt: string, source: string }>>}
+ */
+export async function searchHackerNews(query, options = {}) {
+  // Pro-only: HackerNews scraping requires a license
+  if (!isPro()) {
+    if (process.env.DEBUG) {
+      console.error('[hn] Skipped — Pro license required');
+    }
+    return [];
+  }
+
+  const limits = getLimits();
+  const { limit = limits.hnMaxResults, tags = 'story' } = options;
+
+  try {
+    const params = new URLSearchParams({
+      query,
+      tags,
+      hitsPerPage: String(Math.min(limit, 100)),
+    });
+
+    const url = `${HN_SEARCH_URL}?${params}`;
+    const resp = await fetchWithRetry(url, {
+      headers: { 'Accept': 'application/json' },
+      timeout: 15000,
+    });
+
+    if (!resp || resp.status >= 400) {
+      return [];
+    }
+
+    const data = typeof resp.data === 'string' ? JSON.parse(resp.data) : resp.data;
+    const hits = data?.hits || [];
+
+    return hits.slice(0, limits.hnMaxResults).map(hit => ({
+      title: hit.title || '',
+      url: hit.url || `https://news.ycombinator.com/item?id=${hit.objectID}`,
+      hnUrl: `https://news.ycombinator.com/item?id=${hit.objectID}`,
+      points: hit.points || 0,
+      numComments: hit.num_comments || 0,
+      author: hit.author || '',
+      createdAt: hit.created_at || '',
+      source: 'hackernews',
+    }));
+  } catch (err) {
+    if (process.env.DEBUG) {
+      console.error(`[hn] Search failed: ${err.message}`);
+    }
+    return [];
+  }
+}
+
+// ── Combined search ──────────────────────────────────────────────────────────
+
+/**
+ * Search both Reddit and HN, return combined results sorted by recency.
+ * @param {string} query
+ * @param {{ redditLimit?: number, hnLimit?: number }} options
+ * @returns {Promise<Array<object>>}
+ */
+export async function searchCommunities(query, options = {}) {
+  const [redditResults, hnResults] = await Promise.all([
+    searchReddit(query, { limit: options.redditLimit || 15 }),
+    searchHackerNews(query, { limit: options.hnLimit || 15 }),
+  ]);
+
+  // Merge and sort by date (most recent first)
+  const all = [...redditResults, ...hnResults].sort((a, b) => {
+    const dateA = new Date(a.createdAt).getTime() || 0;
+    const dateB = new Date(b.createdAt).getTime() || 0;
+    return dateB - dateA;
+  });
+
+  return all;
+}

package/src/trackers/brand.js

@@ -1,21 +1,84 @@
 import { scrapeNewsMentions } from '../scrapers/google-news.js';
+import { searchReddit, searchHackerNews } from '../scrapers/reddit-hn.js';
+import { isPro, printProUpgrade } from '../license.js';
 
 export async function runBrandCheck(tracker) {
   const { brandName } = tracker;
 
-  const mentionData = await scrapeNewsMentions(brandName);
+  // Fetch from all sources in parallel
+  const [mentionData, redditResults, hnResults] = await Promise.all([
+    scrapeNewsMentions(brandName),
+    searchReddit(brandName, { limit: 15, timeFilter: 'month' }).catch(() => []),
+    searchHackerNews(brandName, { limit: 15 }).catch(() => []),
+  ]);
+
+  // Convert Reddit results to mention format
+  const redditMentions = redditResults.map(r => ({
+    title: r.title,
+    url: r.url,
+    domain: 'reddit.com',
+    category: r.subreddit,
+    source: 'reddit',
+    sentiment: scoreSentiment(r.title + ' ' + r.selftext),
+    score: r.score,
+    numComments: r.numComments,
+    author: r.author,
+    date: r.createdAt,
+  }));
+
+  // Convert HN results to mention format
+  const hnMentions = hnResults.map(r => ({
+    title: r.title,
+    url: r.hnUrl,
+    domain: 'news.ycombinator.com',
+    category: 'hackernews',
+    source: 'hackernews',
+    sentiment: 'neutral',
+    score: r.points,
+    numComments: r.numComments,
+    author: r.author,
+    date: r.createdAt,
+  }));
+
+  const allMentions = [...(mentionData.mentions || []), ...redditMentions, ...hnMentions];
 
   return {
     type: 'brand',
     trackerId: tracker.id,
     brandName,
     checkedAt: new Date().toISOString(),
-    mentions: mentionData.mentions,
-    mentionCount: mentionData.mentionCount,
+    mentions: allMentions,
+    mentionCount: allMentions.length,
+    sources: {
+      googleNews: (mentionData.mentions || []).length,
+      reddit: redditMentions.length,
+      hackerNews: hnMentions.length,
+    },
     error: mentionData.error || null,
+    tier: isPro() ? 'pro' : 'free',
   };
 }
 
+/**
+ * Simple sentiment scorer for Reddit/HN text.
+ */
+function scoreSentiment(text) {
+  if (!text) return 'neutral';
+  const lower = text.toLowerCase();
+  const positive = ['great', 'awesome', 'excellent', 'love', 'best', 'amazing', 'good', 'fantastic', 'recommend', 'impressed'];
+  const negative = ['bad', 'terrible', 'worst', 'hate', 'awful', 'horrible', 'scam', 'avoid', 'disappointed', 'broken', 'bug'];
+
+  let score = 0;
+  for (const word of positive) { if (lower.includes(word)) score++; }
+  for (const word of negative) { if (lower.includes(word)) score--; }
+
+  if (score >= 2) return 'positive';
+  if (score === 1) return 'slightly_positive';
+  if (score <= -2) return 'negative';
+  if (score === -1) return 'slightly_negative';
+  return 'neutral';
+}
+
 export function diffBrandSnapshots(prev, curr) {
   const changes = [];
 

package/src/trackers/competitor.js

@@ -1,7 +1,7 @@
 import { analyzeSite, analyzeKeyPages } from '../scrapers/site-analyzer.js';
 import { scrapeNewsMentions } from '../scrapers/google-news.js';
 import { searchPressMentions, extractRatingsFromResults } from '../scrapers/brave-search.js';
-import { pappersLookup, hasPappersKey } from '../scrapers/pappers.js';
+import { lookupCompany, resolveProvider } from '../providers/registry.js';
 import { diffTechStacks } from '../utils/tech-detect.js';
 import { fetch } from '../utils/fetcher.js';
 import { load } from '../utils/parser.js';
@@ -83,14 +83,11 @@ export async function runCompetitorCheck(tracker) {
     }
   } catch {}
 
-  // --- Pappers lookup for .fr domains ---
-  let pappers = null;
-  const hostname = new URL(url).hostname;
-  if (hostname.endsWith('.fr') && hasPappersKey()) {
-    try {
-      pappers = await pappersLookup(brandName);
-    } catch {}
-  }
+  // --- Company data lookup (adapts to TLD: Pappers for .fr, OpenCorporates for international) ---
+  let companyData = null;
+  try {
+    companyData = await lookupCompany(brandName, url);
+  } catch {}
 
   return {
     type: 'competitor',
@@ -113,7 +110,9 @@ export async function runCompetitorCheck(tracker) {
     contentStats: siteData.contentStats,
     press,
     reputation,
-    pappers,
+    companyData,
+    // Backward compat: keep 'pappers' key if data came from Pappers
+    pappers: companyData?.source === 'pappers' ? companyData : (companyData || null),
   };
 }
 

package/src/utils/error-handler.js

@@ -47,6 +47,16 @@ export function withErrorHandling(fn) {
  * Handle and format errors appropriately
  */
 export function handleError(error, context = '') {
+  // Guard against null/undefined/non-object errors
+  if (error == null) {
+    console.error(chalk.red(`\n❌ Unknown error${context ? ` in ${context}` : ''}`));
+    return;
+  }
+  if (typeof error === 'string') {
+    console.error(chalk.red(`\n❌ ${error}`));
+    return;
+  }
+
   if (process.env.NODE_ENV === 'development' || process.env.DEBUG_ERRORS) {
     console.error(chalk.red(`\n❌ Error${context ? ` in ${context}` : ''}:`));
     console.error(error.stack || error);