intelwatch 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +175 -0
- package/bin/intelwatch.js +8 -0
- package/package.json +43 -0
- package/src/ai/client.js +130 -0
- package/src/commands/ai-summary.js +147 -0
- package/src/commands/check.js +267 -0
- package/src/commands/compare.js +124 -0
- package/src/commands/diff.js +118 -0
- package/src/commands/digest.js +156 -0
- package/src/commands/discover.js +301 -0
- package/src/commands/history.js +60 -0
- package/src/commands/list.js +43 -0
- package/src/commands/notify.js +121 -0
- package/src/commands/pitch.js +156 -0
- package/src/commands/report.js +82 -0
- package/src/commands/track.js +94 -0
- package/src/config.js +65 -0
- package/src/index.js +182 -0
- package/src/report/html.js +499 -0
- package/src/report/json.js +44 -0
- package/src/report/markdown.js +156 -0
- package/src/scrapers/brave-search.js +268 -0
- package/src/scrapers/google-news.js +111 -0
- package/src/scrapers/google.js +113 -0
- package/src/scrapers/pappers.js +119 -0
- package/src/scrapers/site-analyzer.js +252 -0
- package/src/storage.js +168 -0
- package/src/trackers/brand.js +76 -0
- package/src/trackers/competitor.js +268 -0
- package/src/trackers/keyword.js +121 -0
- package/src/trackers/person.js +132 -0
- package/src/utils/display.js +102 -0
- package/src/utils/fetcher.js +82 -0
- package/src/utils/parser.js +110 -0
- package/src/utils/sentiment.js +95 -0
- package/src/utils/tech-detect.js +94 -0
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { analyzeSite, analyzeKeyPages } from '../scrapers/site-analyzer.js';
|
|
2
|
+
import { scrapeNewsMentions } from '../scrapers/google-news.js';
|
|
3
|
+
import { searchPressMentions, extractRatingsFromResults } from '../scrapers/brave-search.js';
|
|
4
|
+
import { pappersLookup, hasPappersKey } from '../scrapers/pappers.js';
|
|
5
|
+
import { diffTechStacks } from '../utils/tech-detect.js';
|
|
6
|
+
import { fetch } from '../utils/fetcher.js';
|
|
7
|
+
import { load } from '../utils/parser.js';
|
|
8
|
+
import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
|
|
9
|
+
|
|
10
|
+
export async function runCompetitorCheck(tracker) {
|
|
11
|
+
const { url } = tracker;
|
|
12
|
+
|
|
13
|
+
const siteData = await analyzeSite(url);
|
|
14
|
+
const keyPages = await analyzeKeyPages(url, ['/', '/about', '/pricing']);
|
|
15
|
+
|
|
16
|
+
// Merge keyPages from deep analysis with the separate call
|
|
17
|
+
const mergedKeyPages = { ...siteData.keyPages, ...keyPages };
|
|
18
|
+
|
|
19
|
+
// --- Press & reputation layer ---
|
|
20
|
+
const brandName = tracker.name || new URL(url).hostname.replace('www.', '').split('.')[0];
|
|
21
|
+
|
|
22
|
+
let press = { articles: [], totalCount: 0 };
|
|
23
|
+
let reputation = { reviews: [], avgRating: null, platforms: [] };
|
|
24
|
+
|
|
25
|
+
try {
|
|
26
|
+
// Try Brave Search API first (reliable, no rate limiting)
|
|
27
|
+
const braveData = await searchPressMentions(brandName);
|
|
28
|
+
|
|
29
|
+
if (!braveData.error && braveData.mentions.length > 0) {
|
|
30
|
+
// Brave API worked
|
|
31
|
+
const allMentions = braveData.mentions;
|
|
32
|
+
const pressArticles = allMentions.filter(m => m.category === 'press' || m.source === 'news');
|
|
33
|
+
const forumMentions = allMentions.filter(m => m.category === 'forum' || m.category === 'social');
|
|
34
|
+
const reviewMentions = allMentions.filter(m => m.category === 'review' || m.source === 'review');
|
|
35
|
+
|
|
36
|
+
press = {
|
|
37
|
+
articles: allMentions.filter(m => m.source !== 'review').slice(0, 15).map(m => ({
|
|
38
|
+
title: m.title,
|
|
39
|
+
url: m.url,
|
|
40
|
+
domain: m.domain,
|
|
41
|
+
sentiment: m.sentiment,
|
|
42
|
+
category: m.category,
|
|
43
|
+
source: m.source,
|
|
44
|
+
age: m.age,
|
|
45
|
+
})),
|
|
46
|
+
totalCount: allMentions.filter(m => m.source !== 'review').length,
|
|
47
|
+
pressCount: pressArticles.length,
|
|
48
|
+
forumCount: forumMentions.length,
|
|
49
|
+
sentimentBreakdown: {
|
|
50
|
+
positive: allMentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
|
|
51
|
+
neutral: allMentions.filter(m => m.sentiment === 'neutral').length,
|
|
52
|
+
negative: allMentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
// Extract ratings from review results
|
|
57
|
+
reputation.platforms = extractRatingsFromResults(reviewMentions);
|
|
58
|
+
reputation.reviews = reviewMentions.slice(0, 5).map(m => ({
|
|
59
|
+
title: m.title,
|
|
60
|
+
url: m.url,
|
|
61
|
+
domain: m.domain,
|
|
62
|
+
sentiment: m.sentiment,
|
|
63
|
+
}));
|
|
64
|
+
} else {
|
|
65
|
+
// Fallback to Google scraping
|
|
66
|
+
const newsData = await scrapeNewsMentions(brandName, { lang: 'fr' });
|
|
67
|
+
if (newsData.mentions.length > 0) {
|
|
68
|
+
press = {
|
|
69
|
+
articles: newsData.mentions.slice(0, 15).map(m => ({
|
|
70
|
+
title: m.title, url: m.url, domain: m.domain,
|
|
71
|
+
sentiment: m.sentiment, category: m.category, source: m.source,
|
|
72
|
+
})),
|
|
73
|
+
totalCount: newsData.mentionCount,
|
|
74
|
+
pressCount: newsData.mentions.filter(m => m.category === 'press').length,
|
|
75
|
+
forumCount: newsData.mentions.filter(m => m.category === 'forum' || m.category === 'social').length,
|
|
76
|
+
sentimentBreakdown: {
|
|
77
|
+
positive: newsData.mentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
|
|
78
|
+
neutral: newsData.mentions.filter(m => m.sentiment === 'neutral').length,
|
|
79
|
+
negative: newsData.mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
} catch {}
|
|
85
|
+
|
|
86
|
+
// --- Pappers lookup for .fr domains ---
|
|
87
|
+
let pappers = null;
|
|
88
|
+
const hostname = new URL(url).hostname;
|
|
89
|
+
if (hostname.endsWith('.fr') && hasPappersKey()) {
|
|
90
|
+
try {
|
|
91
|
+
pappers = await pappersLookup(brandName);
|
|
92
|
+
} catch {}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
type: 'competitor',
|
|
97
|
+
trackerId: tracker.id,
|
|
98
|
+
url,
|
|
99
|
+
checkedAt: new Date().toISOString(),
|
|
100
|
+
status: siteData.status,
|
|
101
|
+
error: siteData.error,
|
|
102
|
+
meta: siteData.meta,
|
|
103
|
+
techStack: siteData.techStack,
|
|
104
|
+
socialLinks: siteData.socialLinks,
|
|
105
|
+
links: siteData.links,
|
|
106
|
+
pageCount: siteData.pageCount,
|
|
107
|
+
pricing: siteData.pricing,
|
|
108
|
+
jobs: siteData.jobs,
|
|
109
|
+
keyPages: mergedKeyPages,
|
|
110
|
+
performance: siteData.performance,
|
|
111
|
+
security: siteData.security,
|
|
112
|
+
seoSignals: siteData.seoSignals,
|
|
113
|
+
contentStats: siteData.contentStats,
|
|
114
|
+
press,
|
|
115
|
+
reputation,
|
|
116
|
+
pappers,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function diffCompetitorSnapshots(prev, curr) {
|
|
121
|
+
const changes = [];
|
|
122
|
+
|
|
123
|
+
if (!prev) {
|
|
124
|
+
changes.push({ type: 'new', field: 'tracker', value: 'Initial snapshot created' });
|
|
125
|
+
return changes;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Page count change
|
|
129
|
+
const prevCount = prev.pageCount || 0;
|
|
130
|
+
const currCount = curr.pageCount || 0;
|
|
131
|
+
if (currCount !== prevCount) {
|
|
132
|
+
const diff = currCount - prevCount;
|
|
133
|
+
changes.push({
|
|
134
|
+
type: diff > 0 ? 'new' : 'removed',
|
|
135
|
+
field: 'pageCount',
|
|
136
|
+
value: `${prevCount} → ${currCount} (${diff > 0 ? '+' : ''}${diff} pages)`,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// New/removed pages
|
|
141
|
+
const prevLinks = new Set(prev.links || []);
|
|
142
|
+
const currLinks = new Set(curr.links || []);
|
|
143
|
+
const newPages = [...currLinks].filter(l => !prevLinks.has(l)).slice(0, 10);
|
|
144
|
+
const removedPages = [...prevLinks].filter(l => !currLinks.has(l)).slice(0, 10);
|
|
145
|
+
|
|
146
|
+
for (const page of newPages) {
|
|
147
|
+
changes.push({ type: 'new', field: 'page', value: page });
|
|
148
|
+
}
|
|
149
|
+
for (const page of removedPages) {
|
|
150
|
+
changes.push({ type: 'removed', field: 'page', value: page });
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Tech stack changes
|
|
154
|
+
const techDiff = diffTechStacks(prev.techStack || [], curr.techStack || []);
|
|
155
|
+
for (const tech of techDiff.added) {
|
|
156
|
+
changes.push({ type: 'new', field: 'tech', value: `${tech.name} (${tech.category})` });
|
|
157
|
+
}
|
|
158
|
+
for (const tech of techDiff.removed) {
|
|
159
|
+
changes.push({ type: 'removed', field: 'tech', value: `${tech.name} (${tech.category})` });
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Pricing changes
|
|
163
|
+
if (prev.pricing && curr.pricing) {
|
|
164
|
+
if (prev.pricing.hash !== curr.pricing.hash) {
|
|
165
|
+
changes.push({
|
|
166
|
+
type: 'changed',
|
|
167
|
+
field: 'pricing',
|
|
168
|
+
value: `Pricing page content changed`,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
} else if (!prev.pricing && curr.pricing) {
|
|
172
|
+
changes.push({ type: 'new', field: 'pricing', value: 'Pricing page detected' });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Job changes
|
|
176
|
+
const prevJobs = prev.jobs?.estimatedOpenings || 0;
|
|
177
|
+
const currJobs = curr.jobs?.estimatedOpenings || 0;
|
|
178
|
+
if (currJobs !== prevJobs && (prevJobs > 0 || currJobs > 0)) {
|
|
179
|
+
const diff = currJobs - prevJobs;
|
|
180
|
+
changes.push({
|
|
181
|
+
type: diff > 0 ? 'new' : 'changed',
|
|
182
|
+
field: 'jobs',
|
|
183
|
+
value: `Estimated openings: ${prevJobs} → ${currJobs} (${diff > 0 ? '+' : ''}${diff})`,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Meta title/description changes on key pages
|
|
188
|
+
for (const [page, currPage] of Object.entries(curr.keyPages || {})) {
|
|
189
|
+
const prevPage = (prev.keyPages || {})[page];
|
|
190
|
+
if (!prevPage) {
|
|
191
|
+
if (currPage.title) {
|
|
192
|
+
changes.push({ type: 'new', field: `meta:${page}`, value: currPage.title });
|
|
193
|
+
}
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (prevPage.title !== currPage.title) {
|
|
197
|
+
changes.push({
|
|
198
|
+
type: 'changed',
|
|
199
|
+
field: `title:${page}`,
|
|
200
|
+
value: `"${prevPage.title}" → "${currPage.title}"`,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
if (prevPage.description !== currPage.description) {
|
|
204
|
+
changes.push({
|
|
205
|
+
type: 'changed',
|
|
206
|
+
field: `description:${page}`,
|
|
207
|
+
value: currPage.description,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Social links changes
|
|
213
|
+
const prevSocials = Object.keys(prev.socialLinks || {});
|
|
214
|
+
const currSocials = Object.keys(curr.socialLinks || {});
|
|
215
|
+
for (const platform of currSocials) {
|
|
216
|
+
if (!prevSocials.includes(platform)) {
|
|
217
|
+
changes.push({ type: 'new', field: 'social', value: `${platform}: ${curr.socialLinks[platform]}` });
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
for (const platform of prevSocials) {
|
|
221
|
+
if (!currSocials.includes(platform)) {
|
|
222
|
+
changes.push({ type: 'removed', field: 'social', value: `${platform} removed` });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Press mention changes
|
|
227
|
+
const prevPressCount = prev.press?.totalCount || 0;
|
|
228
|
+
const currPressCount = curr.press?.totalCount || 0;
|
|
229
|
+
if (currPressCount > 0 && currPressCount !== prevPressCount) {
|
|
230
|
+
changes.push({
|
|
231
|
+
type: currPressCount > prevPressCount ? 'new' : 'changed',
|
|
232
|
+
field: 'press',
|
|
233
|
+
value: `${prevPressCount} → ${currPressCount} mentions (${curr.press.sentimentBreakdown?.negative || 0} negative)`,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Reputation changes
|
|
238
|
+
const prevRating = prev.reputation?.platforms?.[0]?.rating;
|
|
239
|
+
const currRating = curr.reputation?.platforms?.[0]?.rating;
|
|
240
|
+
if (currRating && currRating !== prevRating) {
|
|
241
|
+
changes.push({
|
|
242
|
+
type: 'changed',
|
|
243
|
+
field: 'reputation',
|
|
244
|
+
value: prevRating ? `Rating: ${prevRating} → ${currRating}` : `Rating: ${currRating}/5`,
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return changes;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function computeThreatScore(tracker, recentChanges) {
|
|
252
|
+
let score = 0;
|
|
253
|
+
|
|
254
|
+
for (const change of recentChanges) {
|
|
255
|
+
if (change.field === 'pageCount') score += 1;
|
|
256
|
+
if (change.field === 'page') score += 0.3;
|
|
257
|
+
if (change.field === 'tech') score += 2;
|
|
258
|
+
if (change.field === 'pricing') score += 3;
|
|
259
|
+
if (change.field === 'jobs') {
|
|
260
|
+
const match = change.value.match(/\+(\d+)/);
|
|
261
|
+
if (match) score += Math.min(parseInt(match[1]), 5);
|
|
262
|
+
}
|
|
263
|
+
if (change.field.startsWith('title:')) score += 2;
|
|
264
|
+
if (change.field.startsWith('description:')) score += 1;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
return Math.min(Math.round(score), 10);
|
|
268
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { scrapeSerp } from '../scrapers/google.js';
|
|
2
|
+
import { searchKeywordRankings } from '../scrapers/brave-search.js';
|
|
3
|
+
|
|
4
|
+
export async function runKeywordCheck(tracker) {
|
|
5
|
+
const { keyword } = tracker;
|
|
6
|
+
|
|
7
|
+
// Try Brave Search API first
|
|
8
|
+
let results = [];
|
|
9
|
+
let error = null;
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const braveResults = await searchKeywordRankings(keyword);
|
|
13
|
+
if (braveResults.length > 0) {
|
|
14
|
+
results = braveResults.map(r => ({
|
|
15
|
+
...r,
|
|
16
|
+
isFeaturedSnippet: false,
|
|
17
|
+
}));
|
|
18
|
+
}
|
|
19
|
+
} catch (e) {
|
|
20
|
+
// Brave failed, try Google fallback
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (results.length === 0) {
|
|
24
|
+
const serpData = await scrapeSerp(keyword, { num: 20 });
|
|
25
|
+
results = serpData.results || [];
|
|
26
|
+
error = serpData.error;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
type: 'keyword',
|
|
31
|
+
trackerId: tracker.id,
|
|
32
|
+
keyword,
|
|
33
|
+
checkedAt: new Date().toISOString(),
|
|
34
|
+
results,
|
|
35
|
+
resultCount: results.length,
|
|
36
|
+
error,
|
|
37
|
+
featuredSnippet: results.find(r => r.isFeaturedSnippet) || null,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function diffKeywordSnapshots(prev, curr) {
|
|
42
|
+
const changes = [];
|
|
43
|
+
|
|
44
|
+
if (!prev) {
|
|
45
|
+
changes.push({ type: 'new', field: 'tracker', value: `Initial snapshot — ${curr.resultCount} results found` });
|
|
46
|
+
return changes;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const prevByDomain = {};
|
|
50
|
+
for (const r of (prev.results || [])) {
|
|
51
|
+
prevByDomain[r.domain] = r.position;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const currByDomain = {};
|
|
55
|
+
for (const r of (curr.results || [])) {
|
|
56
|
+
currByDomain[r.domain] = r.position;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Position changes
|
|
60
|
+
for (const [domain, currPos] of Object.entries(currByDomain)) {
|
|
61
|
+
const prevPos = prevByDomain[domain];
|
|
62
|
+
if (prevPos === undefined) {
|
|
63
|
+
changes.push({
|
|
64
|
+
type: 'new',
|
|
65
|
+
field: 'serp_entry',
|
|
66
|
+
value: `${domain} entered at #${currPos}`,
|
|
67
|
+
domain,
|
|
68
|
+
position: currPos,
|
|
69
|
+
});
|
|
70
|
+
} else if (prevPos !== currPos) {
|
|
71
|
+
const diff = prevPos - currPos;
|
|
72
|
+
changes.push({
|
|
73
|
+
type: 'changed',
|
|
74
|
+
field: 'serp_position',
|
|
75
|
+
value: `${domain}: #${prevPos} → #${currPos} (${diff > 0 ? '↑' : '↓'}${Math.abs(diff)})`,
|
|
76
|
+
domain,
|
|
77
|
+
prevPosition: prevPos,
|
|
78
|
+
currPosition: currPos,
|
|
79
|
+
delta: diff,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Exits
|
|
85
|
+
for (const [domain, prevPos] of Object.entries(prevByDomain)) {
|
|
86
|
+
if (!currByDomain[domain]) {
|
|
87
|
+
changes.push({
|
|
88
|
+
type: 'removed',
|
|
89
|
+
field: 'serp_exit',
|
|
90
|
+
value: `${domain} dropped out (was #${prevPos})`,
|
|
91
|
+
domain,
|
|
92
|
+
prevPosition: prevPos,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Featured snippet change
|
|
98
|
+
const prevFeatured = prev.featuredSnippet?.domain;
|
|
99
|
+
const currFeatured = curr.featuredSnippet?.domain;
|
|
100
|
+
if (prevFeatured !== currFeatured) {
|
|
101
|
+
if (currFeatured) {
|
|
102
|
+
changes.push({
|
|
103
|
+
type: 'changed',
|
|
104
|
+
field: 'featured_snippet',
|
|
105
|
+
value: `Featured snippet: ${prevFeatured || 'none'} → ${currFeatured}`,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return changes;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function getRankingsTable(snapshot) {
|
|
114
|
+
return (snapshot.results || []).map(r => ({
|
|
115
|
+
position: r.position,
|
|
116
|
+
domain: r.domain,
|
|
117
|
+
title: r.title?.slice(0, 60) || '',
|
|
118
|
+
snippet: r.snippet?.slice(0, 80) || '',
|
|
119
|
+
isFeaturedSnippet: r.isFeaturedSnippet,
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { braveNewsSearch, braveWebSearch, searchSocial } from '../scrapers/brave-search.js';
|
|
2
|
+
import { analyzeSentiment, categorizeMention } from '../utils/sentiment.js';
|
|
3
|
+
|
|
4
|
+
export async function runPersonCheck(tracker) {
|
|
5
|
+
const { personName, org } = tracker;
|
|
6
|
+
const query = org ? `"${personName}" "${org}"` : `"${personName}"`;
|
|
7
|
+
|
|
8
|
+
const mentions = [];
|
|
9
|
+
|
|
10
|
+
// 1. News search
|
|
11
|
+
const news = await braveNewsSearch(personName, { freshness: 'pm' });
|
|
12
|
+
for (const r of news.results) {
|
|
13
|
+
// Filter by org if provided (keep results that mention org or have no org context)
|
|
14
|
+
if (org) {
|
|
15
|
+
const text = (r.title + ' ' + r.snippet).toLowerCase();
|
|
16
|
+
if (!text.includes(org.toLowerCase())) continue;
|
|
17
|
+
}
|
|
18
|
+
const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
|
|
19
|
+
mentions.push({
|
|
20
|
+
source: 'news',
|
|
21
|
+
url: r.url,
|
|
22
|
+
domain: r.domain || r.source,
|
|
23
|
+
title: r.title,
|
|
24
|
+
snippet: r.snippet?.substring(0, 300),
|
|
25
|
+
age: r.age,
|
|
26
|
+
sentiment: sentiment.label,
|
|
27
|
+
sentimentScore: sentiment.score,
|
|
28
|
+
category: categorizeMention(r.url, r.title, r.snippet),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// 2. Web search for recent mentions
|
|
33
|
+
await new Promise(r => setTimeout(r, 500));
|
|
34
|
+
const web = await braveWebSearch(query, { freshness: 'pw' });
|
|
35
|
+
for (const r of web.results) {
|
|
36
|
+
if (mentions.some(m => m.url === r.url)) continue;
|
|
37
|
+
const sentiment = analyzeSentiment(r.title + ' ' + r.snippet);
|
|
38
|
+
mentions.push({
|
|
39
|
+
source: 'web',
|
|
40
|
+
url: r.url,
|
|
41
|
+
domain: r.domain,
|
|
42
|
+
title: r.title,
|
|
43
|
+
snippet: r.snippet?.substring(0, 300),
|
|
44
|
+
age: r.age,
|
|
45
|
+
sentiment: sentiment.label,
|
|
46
|
+
sentimentScore: sentiment.score,
|
|
47
|
+
category: categorizeMention(r.url, r.title, r.snippet),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// 3. Social search (X, Reddit, LinkedIn)
|
|
52
|
+
await new Promise(r => setTimeout(r, 500));
|
|
53
|
+
const social = await searchSocial(query, ['twitter', 'reddit', 'linkedin']);
|
|
54
|
+
const socialMentions = {};
|
|
55
|
+
for (const [platform, results] of Object.entries(social.byPlatform || {})) {
|
|
56
|
+
socialMentions[platform] = results.slice(0, 5).map(r => ({
|
|
57
|
+
url: r.url,
|
|
58
|
+
title: r.title,
|
|
59
|
+
snippet: r.snippet?.substring(0, 200),
|
|
60
|
+
age: r.age,
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const sentimentBreakdown = {
|
|
65
|
+
positive: mentions.filter(m => m.sentiment === 'positive' || m.sentiment === 'slightly_positive').length,
|
|
66
|
+
neutral: mentions.filter(m => m.sentiment === 'neutral').length,
|
|
67
|
+
negative: mentions.filter(m => m.sentiment === 'negative' || m.sentiment === 'slightly_negative').length,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
return {
|
|
71
|
+
type: 'person',
|
|
72
|
+
trackerId: tracker.id,
|
|
73
|
+
personName,
|
|
74
|
+
org: org || null,
|
|
75
|
+
checkedAt: new Date().toISOString(),
|
|
76
|
+
mentions: mentions.slice(0, 30),
|
|
77
|
+
mentionCount: mentions.length,
|
|
78
|
+
socialMentions,
|
|
79
|
+
socialCount: social.results.length,
|
|
80
|
+
sentimentBreakdown,
|
|
81
|
+
error: (news.error && web.error) ? news.error : null,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function diffPersonSnapshots(prev, curr) {
|
|
86
|
+
const changes = [];
|
|
87
|
+
|
|
88
|
+
if (!prev) {
|
|
89
|
+
changes.push({
|
|
90
|
+
type: 'new',
|
|
91
|
+
field: 'tracker',
|
|
92
|
+
value: `Initial snapshot — ${curr.mentionCount} mentions found`,
|
|
93
|
+
});
|
|
94
|
+
return changes;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const prevUrls = new Set((prev.mentions || []).map(m => m.url));
|
|
98
|
+
|
|
99
|
+
for (const mention of (curr.mentions || [])) {
|
|
100
|
+
if (!prevUrls.has(mention.url)) {
|
|
101
|
+
const sentLabel = mention.sentiment === 'negative' || mention.sentiment === 'slightly_negative' ? '⚠ ' : '';
|
|
102
|
+
changes.push({
|
|
103
|
+
type: 'new',
|
|
104
|
+
field: 'mention',
|
|
105
|
+
value: `${sentLabel}[${mention.category}] ${mention.title?.slice(0, 80)} — ${mention.domain}`,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const prevCount = prev.mentionCount || 0;
|
|
111
|
+
const currCount = curr.mentionCount || 0;
|
|
112
|
+
if (currCount !== prevCount) {
|
|
113
|
+
changes.push({
|
|
114
|
+
type: 'changed',
|
|
115
|
+
field: 'mentionCount',
|
|
116
|
+
value: `Mention count: ${prevCount} → ${currCount}`,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Social count changes
|
|
121
|
+
const prevSocial = prev.socialCount || 0;
|
|
122
|
+
const currSocial = curr.socialCount || 0;
|
|
123
|
+
if (currSocial !== prevSocial && (prevSocial > 0 || currSocial > 0)) {
|
|
124
|
+
changes.push({
|
|
125
|
+
type: 'changed',
|
|
126
|
+
field: 'socialMentions',
|
|
127
|
+
value: `Social mentions: ${prevSocial} → ${currSocial}`,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return changes;
|
|
132
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import Table from 'cli-table3';
|
|
3
|
+
|
|
4
|
+
export function createTable(headers, options = {}) {
|
|
5
|
+
return new Table({
|
|
6
|
+
head: headers.map(h => chalk.cyan.bold(h)),
|
|
7
|
+
style: { head: [], border: ['grey'] },
|
|
8
|
+
...options,
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function formatDate(dateStr) {
|
|
13
|
+
if (!dateStr) return chalk.gray('never');
|
|
14
|
+
const d = new Date(dateStr);
|
|
15
|
+
const now = new Date();
|
|
16
|
+
const diff = Math.floor((now - d) / 1000);
|
|
17
|
+
|
|
18
|
+
if (diff < 60) return `${diff}s ago`;
|
|
19
|
+
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`;
|
|
20
|
+
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`;
|
|
21
|
+
return `${Math.floor(diff / 86400)}d ago`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function statusBadge(status) {
|
|
25
|
+
switch (status) {
|
|
26
|
+
case 'active': return chalk.green('● active');
|
|
27
|
+
case 'error': return chalk.red('● error');
|
|
28
|
+
case 'checking': return chalk.yellow('● checking');
|
|
29
|
+
default: return chalk.gray('○ ' + status);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function changeBadge(type) {
|
|
34
|
+
switch (type) {
|
|
35
|
+
case 'new': return chalk.green('+ NEW');
|
|
36
|
+
case 'changed': return chalk.yellow('~ CHG');
|
|
37
|
+
case 'removed': return chalk.red('- REM');
|
|
38
|
+
default: return chalk.gray(' ---');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function diffLine(type, label, value = '') {
|
|
43
|
+
const badge = changeBadge(type);
|
|
44
|
+
const truncated = value.length > 80 ? value.slice(0, 77) + '...' : value;
|
|
45
|
+
switch (type) {
|
|
46
|
+
case 'new': return `${badge} ${chalk.green(label)}: ${chalk.gray(truncated)}`;
|
|
47
|
+
case 'changed': return `${badge} ${chalk.yellow(label)}: ${chalk.gray(truncated)}`;
|
|
48
|
+
case 'removed': return `${badge} ${chalk.red(label)}: ${chalk.gray(truncated)}`;
|
|
49
|
+
default: return ` ${chalk.gray(label)}: ${truncated}`;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function header(text) {
|
|
54
|
+
const line = '─'.repeat(Math.min(60, process.stdout.columns || 60));
|
|
55
|
+
console.log('\n' + chalk.cyan.bold(text));
|
|
56
|
+
console.log(chalk.gray(line));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function section(text) {
|
|
60
|
+
console.log('\n' + chalk.bold(text));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function info(text) {
|
|
64
|
+
console.log(chalk.gray(text));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function success(text) {
|
|
68
|
+
console.log(chalk.green('✓ ') + text);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function warn(text) {
|
|
72
|
+
console.log(chalk.yellow('⚠ ') + text);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function error(text) {
|
|
76
|
+
console.log(chalk.red('✗ ') + text);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function trackerTypeIcon(type) {
|
|
80
|
+
switch (type) {
|
|
81
|
+
case 'competitor': return '🏢';
|
|
82
|
+
case 'keyword': return '🔍';
|
|
83
|
+
case 'brand': return '📣';
|
|
84
|
+
case 'person': return '👤';
|
|
85
|
+
default: return '📌';
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function threatLevel(score) {
|
|
90
|
+
if (score >= 8) return chalk.red('🔴 HIGH');
|
|
91
|
+
if (score >= 4) return chalk.yellow('🟡 MED');
|
|
92
|
+
return chalk.green('🟢 LOW');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function printJson(obj) {
|
|
96
|
+
console.log(JSON.stringify(obj, null, 2));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function truncate(str, len = 60) {
|
|
100
|
+
if (!str) return '';
|
|
101
|
+
return str.length > len ? str.slice(0, len - 3) + '...' : str;
|
|
102
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
|
|
3
|
+
const USER_AGENTS = [
|
|
4
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
5
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
6
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
7
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
8
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
9
|
+
];
|
|
10
|
+
|
|
11
|
+
function randomUserAgent() {
|
|
12
|
+
return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function fetch(url, options = {}) {
|
|
20
|
+
const {
|
|
21
|
+
retries = 3,
|
|
22
|
+
delay = 1500,
|
|
23
|
+
timeout = 15000,
|
|
24
|
+
headers = {},
|
|
25
|
+
} = options;
|
|
26
|
+
|
|
27
|
+
const config = {
|
|
28
|
+
url,
|
|
29
|
+
method: options.method || 'GET',
|
|
30
|
+
timeout,
|
|
31
|
+
headers: {
|
|
32
|
+
'User-Agent': randomUserAgent(),
|
|
33
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
34
|
+
'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8',
|
|
35
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
36
|
+
'Cache-Control': 'no-cache',
|
|
37
|
+
...headers,
|
|
38
|
+
},
|
|
39
|
+
maxRedirects: 5,
|
|
40
|
+
validateStatus: status => status < 500,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
let lastError;
|
|
44
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
45
|
+
try {
|
|
46
|
+
if (attempt > 1) {
|
|
47
|
+
const backoff = delay * Math.pow(2, attempt - 2);
|
|
48
|
+
await sleep(backoff);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const response = await axios(config);
|
|
52
|
+
|
|
53
|
+
if (response.status === 429) {
|
|
54
|
+
const retryAfter = parseInt(response.headers['retry-after'] || '60', 10);
|
|
55
|
+
if (attempt < retries) {
|
|
56
|
+
await sleep(retryAfter * 1000);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
throw new Error(`Rate limited (429) after ${retries} attempts`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return response;
|
|
63
|
+
} catch (err) {
|
|
64
|
+
lastError = err;
|
|
65
|
+
if (attempt < retries) {
|
|
66
|
+
await sleep(delay);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
throw lastError;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export async function fetchWithDelay(url, options = {}) {
|
|
75
|
+
const minDelay = options.minDelay ?? 1000;
|
|
76
|
+
const maxDelay = options.maxDelay ?? 2000;
|
|
77
|
+
const jitter = Math.floor(Math.random() * (maxDelay - minDelay)) + minDelay;
|
|
78
|
+
await sleep(jitter);
|
|
79
|
+
return fetch(url, options);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export { sleep };
|