intelwatch 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,268 @@
1
+ import { analyzeSite, analyzeKeyPages } from '../scrapers/site-analyzer.js';
2
+ import { scrapeNewsMentions } from '../scrapers/google-news.js';
3
+ import { searchPressMentions, extractRatingsFromResults } from '../scrapers/brave-search.js';
4
+ import { pappersLookup, hasPappersKey } from '../scrapers/pappers.js';
5
+ import { diffTechStacks } from '../utils/tech-detect.js';
6
+ import { fetch } from '../utils/fetcher.js';
7
+ import { load } from '../utils/parser.js';
8
+ import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
9
+
10
+ export async function runCompetitorCheck(tracker) {
11
+ const { url } = tracker;
12
+
13
+ const siteData = await analyzeSite(url);
14
+ const keyPages = await analyzeKeyPages(url, ['/', '/about', '/pricing']);
15
+
16
+ // Merge keyPages from deep analysis with the separate call
17
+ const mergedKeyPages = { ...siteData.keyPages, ...keyPages };
18
+
19
+ // --- Press & reputation layer ---
20
+ const brandName = tracker.name || new URL(url).hostname.replace('www.', '').split('.')[0];
21
+
22
+ let press = { articles: [], totalCount: 0 };
23
+ let reputation = { reviews: [], avgRating: null, platforms: [] };
24
+
25
+ try {
26
+ // Try Brave Search API first (reliable, no rate limiting)
27
+ const braveData = await searchPressMentions(brandName);
28
+
29
+ if (!braveData.error && braveData.mentions.length > 0) {
30
+ // Brave API worked
31
+ const allMentions = braveData.mentions;
32
+ const pressArticles = allMentions.filter(m => m.category === 'press' || m.source === 'news');
33
+ const forumMentions = allMentions.filter(m => m.category === 'forum' || m.category === 'social');
34
+ const reviewMentions = allMentions.filter(m => m.category === 'review' || m.source === 'review');
35
+
36
+ press = {
37
+ articles: allMentions.filter(m => m.source !== 'review').slice(0, 15).map(m => ({
38
+ title: m.title,
39
+ url: m.url,
40
+ domain: m.domain,
41
+ sentiment: m.sentiment,
42
+ category: m.category,
43
+ source: m.source,
44
+ age: m.age,
45
+ })),
46
+ totalCount: allMentions.filter(m => m.source !== 'review').length,
47
+ pressCount: pressArticles.length,
48
+ forumCount: forumMentions.length,
49
+ sentimentBreakdown: {
50
+ positive: allMentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
51
+ neutral: allMentions.filter(m => m.sentiment === 'neutral').length,
52
+ negative: allMentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
53
+ },
54
+ };
55
+
56
+ // Extract ratings from review results
57
+ reputation.platforms = extractRatingsFromResults(reviewMentions);
58
+ reputation.reviews = reviewMentions.slice(0, 5).map(m => ({
59
+ title: m.title,
60
+ url: m.url,
61
+ domain: m.domain,
62
+ sentiment: m.sentiment,
63
+ }));
64
+ } else {
65
+ // Fallback to Google scraping
66
+ const newsData = await scrapeNewsMentions(brandName, { lang: 'fr' });
67
+ if (newsData.mentions.length > 0) {
68
+ press = {
69
+ articles: newsData.mentions.slice(0, 15).map(m => ({
70
+ title: m.title, url: m.url, domain: m.domain,
71
+ sentiment: m.sentiment, category: m.category, source: m.source,
72
+ })),
73
+ totalCount: newsData.mentionCount,
74
+ pressCount: newsData.mentions.filter(m => m.category === 'press').length,
75
+ forumCount: newsData.mentions.filter(m => m.category === 'forum' || m.category === 'social').length,
76
+ sentimentBreakdown: {
77
+ positive: newsData.mentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
78
+ neutral: newsData.mentions.filter(m => m.sentiment === 'neutral').length,
79
+ negative: newsData.mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
80
+ },
81
+ };
82
+ }
83
+ }
84
+ } catch {}
85
+
86
+ // --- Pappers lookup for .fr domains ---
87
+ let pappers = null;
88
+ const hostname = new URL(url).hostname;
89
+ if (hostname.endsWith('.fr') && hasPappersKey()) {
90
+ try {
91
+ pappers = await pappersLookup(brandName);
92
+ } catch {}
93
+ }
94
+
95
+ return {
96
+ type: 'competitor',
97
+ trackerId: tracker.id,
98
+ url,
99
+ checkedAt: new Date().toISOString(),
100
+ status: siteData.status,
101
+ error: siteData.error,
102
+ meta: siteData.meta,
103
+ techStack: siteData.techStack,
104
+ socialLinks: siteData.socialLinks,
105
+ links: siteData.links,
106
+ pageCount: siteData.pageCount,
107
+ pricing: siteData.pricing,
108
+ jobs: siteData.jobs,
109
+ keyPages: mergedKeyPages,
110
+ performance: siteData.performance,
111
+ security: siteData.security,
112
+ seoSignals: siteData.seoSignals,
113
+ contentStats: siteData.contentStats,
114
+ press,
115
+ reputation,
116
+ pappers,
117
+ };
118
+ }
119
+
120
+ export function diffCompetitorSnapshots(prev, curr) {
121
+ const changes = [];
122
+
123
+ if (!prev) {
124
+ changes.push({ type: 'new', field: 'tracker', value: 'Initial snapshot created' });
125
+ return changes;
126
+ }
127
+
128
+ // Page count change
129
+ const prevCount = prev.pageCount || 0;
130
+ const currCount = curr.pageCount || 0;
131
+ if (currCount !== prevCount) {
132
+ const diff = currCount - prevCount;
133
+ changes.push({
134
+ type: diff > 0 ? 'new' : 'removed',
135
+ field: 'pageCount',
136
+ value: `${prevCount} → ${currCount} (${diff > 0 ? '+' : ''}${diff} pages)`,
137
+ });
138
+ }
139
+
140
+ // New/removed pages
141
+ const prevLinks = new Set(prev.links || []);
142
+ const currLinks = new Set(curr.links || []);
143
+ const newPages = [...currLinks].filter(l => !prevLinks.has(l)).slice(0, 10);
144
+ const removedPages = [...prevLinks].filter(l => !currLinks.has(l)).slice(0, 10);
145
+
146
+ for (const page of newPages) {
147
+ changes.push({ type: 'new', field: 'page', value: page });
148
+ }
149
+ for (const page of removedPages) {
150
+ changes.push({ type: 'removed', field: 'page', value: page });
151
+ }
152
+
153
+ // Tech stack changes
154
+ const techDiff = diffTechStacks(prev.techStack || [], curr.techStack || []);
155
+ for (const tech of techDiff.added) {
156
+ changes.push({ type: 'new', field: 'tech', value: `${tech.name} (${tech.category})` });
157
+ }
158
+ for (const tech of techDiff.removed) {
159
+ changes.push({ type: 'removed', field: 'tech', value: `${tech.name} (${tech.category})` });
160
+ }
161
+
162
+ // Pricing changes
163
+ if (prev.pricing && curr.pricing) {
164
+ if (prev.pricing.hash !== curr.pricing.hash) {
165
+ changes.push({
166
+ type: 'changed',
167
+ field: 'pricing',
168
+ value: `Pricing page content changed`,
169
+ });
170
+ }
171
+ } else if (!prev.pricing && curr.pricing) {
172
+ changes.push({ type: 'new', field: 'pricing', value: 'Pricing page detected' });
173
+ }
174
+
175
+ // Job changes
176
+ const prevJobs = prev.jobs?.estimatedOpenings || 0;
177
+ const currJobs = curr.jobs?.estimatedOpenings || 0;
178
+ if (currJobs !== prevJobs && (prevJobs > 0 || currJobs > 0)) {
179
+ const diff = currJobs - prevJobs;
180
+ changes.push({
181
+ type: diff > 0 ? 'new' : 'changed',
182
+ field: 'jobs',
183
+ value: `Estimated openings: ${prevJobs} → ${currJobs} (${diff > 0 ? '+' : ''}${diff})`,
184
+ });
185
+ }
186
+
187
+ // Meta title/description changes on key pages
188
+ for (const [page, currPage] of Object.entries(curr.keyPages || {})) {
189
+ const prevPage = (prev.keyPages || {})[page];
190
+ if (!prevPage) {
191
+ if (currPage.title) {
192
+ changes.push({ type: 'new', field: `meta:${page}`, value: currPage.title });
193
+ }
194
+ continue;
195
+ }
196
+ if (prevPage.title !== currPage.title) {
197
+ changes.push({
198
+ type: 'changed',
199
+ field: `title:${page}`,
200
+ value: `"${prevPage.title}" → "${currPage.title}"`,
201
+ });
202
+ }
203
+ if (prevPage.description !== currPage.description) {
204
+ changes.push({
205
+ type: 'changed',
206
+ field: `description:${page}`,
207
+ value: currPage.description,
208
+ });
209
+ }
210
+ }
211
+
212
+ // Social links changes
213
+ const prevSocials = Object.keys(prev.socialLinks || {});
214
+ const currSocials = Object.keys(curr.socialLinks || {});
215
+ for (const platform of currSocials) {
216
+ if (!prevSocials.includes(platform)) {
217
+ changes.push({ type: 'new', field: 'social', value: `${platform}: ${curr.socialLinks[platform]}` });
218
+ }
219
+ }
220
+ for (const platform of prevSocials) {
221
+ if (!currSocials.includes(platform)) {
222
+ changes.push({ type: 'removed', field: 'social', value: `${platform} removed` });
223
+ }
224
+ }
225
+
226
+ // Press mention changes
227
+ const prevPressCount = prev.press?.totalCount || 0;
228
+ const currPressCount = curr.press?.totalCount || 0;
229
+ if (currPressCount > 0 && currPressCount !== prevPressCount) {
230
+ changes.push({
231
+ type: currPressCount > prevPressCount ? 'new' : 'changed',
232
+ field: 'press',
233
+ value: `${prevPressCount} → ${currPressCount} mentions (${curr.press.sentimentBreakdown?.negative || 0} negative)`,
234
+ });
235
+ }
236
+
237
+ // Reputation changes
238
+ const prevRating = prev.reputation?.platforms?.[0]?.rating;
239
+ const currRating = curr.reputation?.platforms?.[0]?.rating;
240
+ if (currRating && currRating !== prevRating) {
241
+ changes.push({
242
+ type: 'changed',
243
+ field: 'reputation',
244
+ value: prevRating ? `Rating: ${prevRating} → ${currRating}` : `Rating: ${currRating}/5`,
245
+ });
246
+ }
247
+
248
+ return changes;
249
+ }
250
+
251
+ export function computeThreatScore(tracker, recentChanges) {
252
+ let score = 0;
253
+
254
+ for (const change of recentChanges) {
255
+ if (change.field === 'pageCount') score += 1;
256
+ if (change.field === 'page') score += 0.3;
257
+ if (change.field === 'tech') score += 2;
258
+ if (change.field === 'pricing') score += 3;
259
+ if (change.field === 'jobs') {
260
+ const match = change.value.match(/\+(\d+)/);
261
+ if (match) score += Math.min(parseInt(match[1]), 5);
262
+ }
263
+ if (change.field.startsWith('title:')) score += 2;
264
+ if (change.field.startsWith('description:')) score += 1;
265
+ }
266
+
267
+ return Math.min(Math.round(score), 10);
268
+ }
@@ -0,0 +1,121 @@
1
+ import { scrapeSerp } from '../scrapers/google.js';
2
+ import { searchKeywordRankings } from '../scrapers/brave-search.js';
3
+
4
+ export async function runKeywordCheck(tracker) {
5
+ const { keyword } = tracker;
6
+
7
+ // Try Brave Search API first
8
+ let results = [];
9
+ let error = null;
10
+
11
+ try {
12
+ const braveResults = await searchKeywordRankings(keyword);
13
+ if (braveResults.length > 0) {
14
+ results = braveResults.map(r => ({
15
+ ...r,
16
+ isFeaturedSnippet: false,
17
+ }));
18
+ }
19
+ } catch (e) {
20
+ // Brave failed, try Google fallback
21
+ }
22
+
23
+ if (results.length === 0) {
24
+ const serpData = await scrapeSerp(keyword, { num: 20 });
25
+ results = serpData.results || [];
26
+ error = serpData.error;
27
+ }
28
+
29
+ return {
30
+ type: 'keyword',
31
+ trackerId: tracker.id,
32
+ keyword,
33
+ checkedAt: new Date().toISOString(),
34
+ results,
35
+ resultCount: results.length,
36
+ error,
37
+ featuredSnippet: results.find(r => r.isFeaturedSnippet) || null,
38
+ };
39
+ }
40
+
41
+ export function diffKeywordSnapshots(prev, curr) {
42
+ const changes = [];
43
+
44
+ if (!prev) {
45
+ changes.push({ type: 'new', field: 'tracker', value: `Initial snapshot — ${curr.resultCount} results found` });
46
+ return changes;
47
+ }
48
+
49
+ const prevByDomain = {};
50
+ for (const r of (prev.results || [])) {
51
+ prevByDomain[r.domain] = r.position;
52
+ }
53
+
54
+ const currByDomain = {};
55
+ for (const r of (curr.results || [])) {
56
+ currByDomain[r.domain] = r.position;
57
+ }
58
+
59
+ // Position changes
60
+ for (const [domain, currPos] of Object.entries(currByDomain)) {
61
+ const prevPos = prevByDomain[domain];
62
+ if (prevPos === undefined) {
63
+ changes.push({
64
+ type: 'new',
65
+ field: 'serp_entry',
66
+ value: `${domain} entered at #${currPos}`,
67
+ domain,
68
+ position: currPos,
69
+ });
70
+ } else if (prevPos !== currPos) {
71
+ const diff = prevPos - currPos;
72
+ changes.push({
73
+ type: 'changed',
74
+ field: 'serp_position',
75
+ value: `${domain}: #${prevPos} → #${currPos} (${diff > 0 ? '↑' : '↓'}${Math.abs(diff)})`,
76
+ domain,
77
+ prevPosition: prevPos,
78
+ currPosition: currPos,
79
+ delta: diff,
80
+ });
81
+ }
82
+ }
83
+
84
+ // Exits
85
+ for (const [domain, prevPos] of Object.entries(prevByDomain)) {
86
+ if (!currByDomain[domain]) {
87
+ changes.push({
88
+ type: 'removed',
89
+ field: 'serp_exit',
90
+ value: `${domain} dropped out (was #${prevPos})`,
91
+ domain,
92
+ prevPosition: prevPos,
93
+ });
94
+ }
95
+ }
96
+
97
+ // Featured snippet change
98
+ const prevFeatured = prev.featuredSnippet?.domain;
99
+ const currFeatured = curr.featuredSnippet?.domain;
100
+ if (prevFeatured !== currFeatured) {
101
+ if (currFeatured) {
102
+ changes.push({
103
+ type: 'changed',
104
+ field: 'featured_snippet',
105
+ value: `Featured snippet: ${prevFeatured || 'none'} → ${currFeatured}`,
106
+ });
107
+ }
108
+ }
109
+
110
+ return changes;
111
+ }
112
+
113
+ export function getRankingsTable(snapshot) {
114
+ return (snapshot.results || []).map(r => ({
115
+ position: r.position,
116
+ domain: r.domain,
117
+ title: r.title?.slice(0, 60) || '',
118
+ snippet: r.snippet?.slice(0, 80) || '',
119
+ isFeaturedSnippet: r.isFeaturedSnippet,
120
+ }));
121
+ }
@@ -0,0 +1,132 @@
1
+ import { braveNewsSearch, braveWebSearch, searchSocial } from '../scrapers/brave-search.js';
2
+ import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
3
+
4
+ export async function runPersonCheck(tracker) {
5
+ const { personName, org } = tracker;
6
+ const query = org ? `"${personName}" "${org}"` : `"${personName}"`;
7
+
8
+ const mentions = [];
9
+
10
+ // 1. News search
11
+ const news = await braveNewsSearch(personName, { freshness: 'pm' });
12
+ for (const r of news.results) {
13
+ // Filter by org if provided (keep results that mention org or have no org context)
14
+ if (org) {
15
+ const text = (r.title + ' ' + r.snippet).toLowerCase();
16
+ if (!text.includes(org.toLowerCase())) continue;
17
+ }
18
+ const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
19
+ mentions.push({
20
+ source: 'news',
21
+ url: r.url,
22
+ domain: r.domain || r.source,
23
+ title: r.title,
24
+ snippet: r.snippet?.substring(0, 300),
25
+ age: r.age,
26
+ sentiment: sentiment.label,
27
+ sentimentScore: sentiment.score,
28
+ category: categorizeMention(r.url, r.title, r.snippet),
29
+ });
30
+ }
31
+
32
+ // 2. Web search for recent mentions
33
+ await new Promise(r => setTimeout(r, 500));
34
+ const web = await braveWebSearch(query, { freshness: 'pw' });
35
+ for (const r of web.results) {
36
+ if (mentions.some(m => m.url === r.url)) continue;
37
+ const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
38
+ mentions.push({
39
+ source: 'web',
40
+ url: r.url,
41
+ domain: r.domain,
42
+ title: r.title,
43
+ snippet: r.snippet?.substring(0, 300),
44
+ age: r.age,
45
+ sentiment: sentiment.label,
46
+ sentimentScore: sentiment.score,
47
+ category: categorizeMention(r.url, r.title, r.snippet),
48
+ });
49
+ }
50
+
51
+ // 3. Social search (X, Reddit, LinkedIn)
52
+ await new Promise(r => setTimeout(r, 500));
53
+ const social = await searchSocial(query, ['twitter', 'reddit', 'linkedin']);
54
+ const socialMentions = {};
55
+ for (const [platform, results] of Object.entries(social.byPlatform || {})) {
56
+ socialMentions[platform] = results.slice(0, 5).map(r => ({
57
+ url: r.url,
58
+ title: r.title,
59
+ snippet: r.snippet?.substring(0, 200),
60
+ age: r.age,
61
+ }));
62
+ }
63
+
64
+ const sentimentBreakdown = {
65
+ positive: mentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
66
+ neutral: mentions.filter(m => m.sentiment === 'neutral').length,
67
+ negative: mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
68
+ };
69
+
70
+ return {
71
+ type: 'person',
72
+ trackerId: tracker.id,
73
+ personName,
74
+ org: org || null,
75
+ checkedAt: new Date().toISOString(),
76
+ mentions: mentions.slice(0, 30),
77
+ mentionCount: mentions.length,
78
+ socialMentions,
79
+ socialCount: social.results.length,
80
+ sentimentBreakdown,
81
+ error: (news.error && web.error) ? news.error : null,
82
+ };
83
+ }
84
+
85
+ export function diffPersonSnapshots(prev, curr) {
86
+ const changes = [];
87
+
88
+ if (!prev) {
89
+ changes.push({
90
+ type: 'new',
91
+ field: 'tracker',
92
+ value: `Initial snapshot — ${curr.mentionCount} mentions found`,
93
+ });
94
+ return changes;
95
+ }
96
+
97
+ const prevUrls = new Set((prev.mentions || []).map(m => m.url));
98
+
99
+ for (const mention of (curr.mentions || [])) {
100
+ if (!prevUrls.has(mention.url)) {
101
+ const sentLabel = mention.sentiment === 'negative' || mention.sentiment === 'slightly_negative' ? '⚠ ' : '';
102
+ changes.push({
103
+ type: 'new',
104
+ field: 'mention',
105
+ value: `${sentLabel}[${mention.category}] ${mention.title?.slice(0, 80)} — ${mention.domain}`,
106
+ });
107
+ }
108
+ }
109
+
110
+ const prevCount = prev.mentionCount || 0;
111
+ const currCount = curr.mentionCount || 0;
112
+ if (currCount !== prevCount) {
113
+ changes.push({
114
+ type: 'changed',
115
+ field: 'mentionCount',
116
+ value: `Mention count: ${prevCount} → ${currCount}`,
117
+ });
118
+ }
119
+
120
+ // Social count changes
121
+ const prevSocial = prev.socialCount || 0;
122
+ const currSocial = curr.socialCount || 0;
123
+ if (currSocial !== prevSocial && (prevSocial > 0 || currSocial > 0)) {
124
+ changes.push({
125
+ type: 'changed',
126
+ field: 'socialMentions',
127
+ value: `Social mentions: ${prevSocial} → ${currSocial}`,
128
+ });
129
+ }
130
+
131
+ return changes;
132
+ }
@@ -0,0 +1,102 @@
1
+ import chalk from 'chalk';
2
+ import Table from 'cli-table3';
3
+
4
+ export function createTable(headers, options = {}) {
5
+ return new Table({
6
+ head: headers.map(h => chalk.cyan.bold(h)),
7
+ style: { head: [], border: ['grey'] },
8
+ ...options,
9
+ });
10
+ }
11
+
12
+ export function formatDate(dateStr) {
13
+ if (!dateStr) return chalk.gray('never');
14
+ const d = new Date(dateStr);
15
+ const now = new Date();
16
+ const diff = Math.floor((now - d) / 1000);
17
+
18
+ if (diff < 60) return `${diff}s ago`;
19
+ if (diff < 3600) return `${Math.floor(diff / 60)}m ago`;
20
+ if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
21
+ return `${Math.floor(diff / 86400)}d ago`;
22
+ }
23
+
24
+ export function statusBadge(status) {
25
+ switch (status) {
26
+ case 'active': return chalk.green('● active');
27
+ case 'error': return chalk.red('● error');
28
+ case 'checking': return chalk.yellow('● checking');
29
+ default: return chalk.gray('○ ' + status);
30
+ }
31
+ }
32
+
33
+ export function changeBadge(type) {
34
+ switch (type) {
35
+ case 'new': return chalk.green('+ NEW');
36
+ case 'changed': return chalk.yellow('~ CHG');
37
+ case 'removed': return chalk.red('- REM');
38
+ default: return chalk.gray(' ---');
39
+ }
40
+ }
41
+
42
+ export function diffLine(type, label, value = '') {
43
+ const badge = changeBadge(type);
44
+ const truncated = value.length > 80 ? value.slice(0, 77) + '...' : value;
45
+ switch (type) {
46
+ case 'new': return `${badge} ${chalk.green(label)}: ${chalk.gray(truncated)}`;
47
+ case 'changed': return `${badge} ${chalk.yellow(label)}: ${chalk.gray(truncated)}`;
48
+ case 'removed': return `${badge} ${chalk.red(label)}: ${chalk.gray(truncated)}`;
49
+ default: return ` ${chalk.gray(label)}: ${truncated}`;
50
+ }
51
+ }
52
+
53
+ export function header(text) {
54
+ const line = '─'.repeat(Math.min(60, process.stdout.columns || 60));
55
+ console.log('\n' + chalk.cyan.bold(text));
56
+ console.log(chalk.gray(line));
57
+ }
58
+
59
+ export function section(text) {
60
+ console.log('\n' + chalk.bold(text));
61
+ }
62
+
63
+ export function info(text) {
64
+ console.log(chalk.gray(text));
65
+ }
66
+
67
+ export function success(text) {
68
+ console.log(chalk.green('✓ ') + text);
69
+ }
70
+
71
+ export function warn(text) {
72
+ console.log(chalk.yellow('⚠ ') + text);
73
+ }
74
+
75
+ export function error(text) {
76
+ console.log(chalk.red('✗ ') + text);
77
+ }
78
+
79
+ export function trackerTypeIcon(type) {
80
+ switch (type) {
81
+ case 'competitor': return '🏢';
82
+ case 'keyword': return '🔍';
83
+ case 'brand': return '📣';
84
+ case 'person': return '👤';
85
+ default: return '📌';
86
+ }
87
+ }
88
+
89
+ export function threatLevel(score) {
90
+ if (score >= 8) return chalk.red('🔴 HIGH');
91
+ if (score >= 4) return chalk.yellow('🟡 MED');
92
+ return chalk.green('🟢 LOW');
93
+ }
94
+
95
+ export function printJson(obj) {
96
+ console.log(JSON.stringify(obj, null, 2));
97
+ }
98
+
99
+ export function truncate(str, len = 60) {
100
+ if (!str) return '';
101
+ return str.length > len ? str.slice(0, len - 3) + '...' : str;
102
+ }
@@ -0,0 +1,82 @@
1
+ import axios from 'axios';
2
+
3
+ const USER_AGENTS = [
4
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
5
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
6
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
7
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
8
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
9
+ ];
10
+
11
+ function randomUserAgent() {
12
+ return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
13
+ }
14
+
15
+ function sleep(ms) {
16
+ return new Promise(resolve => setTimeout(resolve, ms));
17
+ }
18
+
19
+ export async function fetch(url, options = {}) {
20
+ const {
21
+ retries = 3,
22
+ delay = 1500,
23
+ timeout = 15000,
24
+ headers = {},
25
+ } = options;
26
+
27
+ const config = {
28
+ url,
29
+ method: options.method || 'GET',
30
+ timeout,
31
+ headers: {
32
+ 'User-Agent': randomUserAgent(),
33
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
34
+ 'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8',
35
+ 'Accept-Encoding': 'gzip, deflate, br',
36
+ 'Cache-Control': 'no-cache',
37
+ ...headers,
38
+ },
39
+ maxRedirects: 5,
40
+ validateStatus: status => status < 500,
41
+ };
42
+
43
+ let lastError;
44
+ for (let attempt = 1; attempt <= retries; attempt++) {
45
+ try {
46
+ if (attempt > 1) {
47
+ const backoff = delay * Math.pow(2, attempt - 2);
48
+ await sleep(backoff);
49
+ }
50
+
51
+ const response = await axios(config);
52
+
53
+ if (response.status === 429) {
54
+ const retryAfter = parseInt(response.headers['retry-after'] || '60', 10);
55
+ if (attempt < retries) {
56
+ await sleep(retryAfter * 1000);
57
+ continue;
58
+ }
59
+ throw new Error(`Rate limited (429) after ${retries} attempts`);
60
+ }
61
+
62
+ return response;
63
+ } catch (err) {
64
+ lastError = err;
65
+ if (attempt < retries) {
66
+ await sleep(delay);
67
+ }
68
+ }
69
+ }
70
+
71
+ throw lastError;
72
+ }
73
+
74
+ export async function fetchWithDelay(url, options = {}) {
75
+ const minDelay = options.minDelay ?? 1000;
76
+ const maxDelay = options.maxDelay ?? 2000;
77
+ const jitter = Math.floor(Math.random() * (maxDelay - minDelay)) + minDelay;
78
+ await sleep(jitter);
79
+ return fetch(url, options);
80
+ }
81
+
82
+ export { sleep };