crawlforge-mcp-server 3.0.17 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CLAUDE.md +2 -0
  2. package/README.md +1 -0
  3. package/package.json +6 -2
  4. package/server.js +192 -1277
  5. package/src/constants/config.js +2 -1
  6. package/src/core/ActionExecutor.js +2 -43
  7. package/src/core/AuthManager.js +230 -32
  8. package/src/core/BrowserContextPool.js +187 -0
  9. package/src/core/JobManager.js +7 -5
  10. package/src/core/LocalizationManager.js +14 -125
  11. package/src/core/ResearchOrchestrator.js +86 -5
  12. package/src/core/StealthBrowserManager.js +26 -18
  13. package/src/core/cache/CacheManager.js +4 -1
  14. package/src/core/crawlers/BFSCrawler.js +19 -5
  15. package/src/core/endpointGuard.js +37 -0
  16. package/src/observability/metrics.js +137 -0
  17. package/src/observability/tracing.js +74 -0
  18. package/src/server/auth/oauth.js +388 -0
  19. package/src/server/registerTool.js +41 -0
  20. package/src/server/schemas/common.js +29 -0
  21. package/src/server/transports/http.js +22 -0
  22. package/src/server/transports/stdio.js +16 -0
  23. package/src/server/transports/streamableHttp.js +226 -0
  24. package/src/server/withAuth.js +121 -0
  25. package/src/tools/advanced/BatchScrapeTool.js +12 -1086
  26. package/src/tools/advanced/ScrapeWithActionsTool.js +105 -19
  27. package/src/tools/advanced/batchScrape/index.js +328 -0
  28. package/src/tools/advanced/batchScrape/queue.js +91 -0
  29. package/src/tools/advanced/batchScrape/reporter.js +26 -0
  30. package/src/tools/advanced/batchScrape/schema.js +37 -0
  31. package/src/tools/advanced/batchScrape/worker.js +179 -0
  32. package/src/tools/advanced/scrapeWithActions/recorder.js +188 -0
  33. package/src/tools/basic/_fetch.js +35 -0
  34. package/src/tools/basic/extractLinks.js +74 -0
  35. package/src/tools/basic/extractMetadata.js +74 -0
  36. package/src/tools/basic/extractText.js +46 -0
  37. package/src/tools/basic/fetchUrl.js +44 -0
  38. package/src/tools/basic/scrapeStructured.js +58 -0
  39. package/src/tools/crawl/_sessionContext.js +234 -0
  40. package/src/tools/crawl/crawlDeep.js +55 -5
  41. package/src/tools/crawl/mapSite.js +23 -2
  42. package/src/tools/extract/_fetchAndParse.js +57 -0
  43. package/src/tools/extract/extractStructured.js +3 -19
  44. package/src/tools/extract/extractWithLlm.js +295 -0
  45. package/src/tools/research/deepResearch.js +33 -8
  46. package/src/tools/search/providers/searxng.js +126 -0
  47. package/src/tools/search/ranking/ResultDeduplicator.js +18 -11
  48. package/src/tools/search/ranking/ResultRanker.js +17 -10
  49. package/src/tools/search/ranking/SearchResultCache.js +52 -0
  50. package/src/tools/search/searchWeb.js +112 -6
  51. package/src/tools/tracking/trackChanges/differ.js +98 -0
  52. package/src/tools/tracking/trackChanges/index.js +432 -0
  53. package/src/tools/tracking/trackChanges/monitor.js +93 -0
  54. package/src/tools/tracking/trackChanges/notifier.js +105 -0
  55. package/src/tools/tracking/trackChanges/schema.js +127 -0
  56. package/src/tools/tracking/trackChanges.js +12 -1374
@@ -0,0 +1,432 @@
1
+ /**
2
+ * TrackChanges — entry-point (index.js).
3
+ *
4
+ * Preserves the same exports as the original single-file trackChanges.js:
5
+ * export class TrackChangesTool
6
+ * export default TrackChangesTool
7
+ * export const trackChangesTool (singleton)
8
+ *
9
+ * Handler logic ≤ 150 LOC here; heavy work delegated to:
10
+ * schema.js — Zod input schema
11
+ * differ.js — fetch, merge-history, stat helpers
12
+ * monitor.js — polling monitor lifecycle
13
+ * notifier.js — webhook / email / Slack notifications
14
+ */
15
+
16
+ import { EventEmitter } from 'events';
17
+ import ChangeTracker from '../../../core/ChangeTracker.js';
18
+ import SnapshotManager from '../../../core/SnapshotManager.js';
19
+ import CacheManager from '../../../core/cache/CacheManager.js';
20
+ import { TrackChangesSchema } from './schema.js';
21
+ import { fetchContent, mergeHistoryData, matchesSignificanceFilter, calculateAverageInterval, calculateSignificanceDistribution } from './differ.js';
22
+ import { performMonitoringCheck, stopMonitor } from './monitor.js';
23
+ import { sendNotifications } from './notifier.js';
24
+
25
+ export class TrackChangesTool extends EventEmitter {
26
+ constructor(options = {}) {
27
+ super();
28
+
29
+ this.options = {
30
+ cacheEnabled: true,
31
+ cacheTTL: 3600000,
32
+ snapshotStorageDir: './snapshots',
33
+ enableRealTimeMonitoring: true,
34
+ maxConcurrentMonitors: 50,
35
+ defaultPollingInterval: 300000,
36
+ ...options
37
+ };
38
+
39
+ this.changeTracker = new ChangeTracker({
40
+ enableRealTimeTracking: this.options.enableRealTimeMonitoring,
41
+ enableSemanticAnalysis: false,
42
+ contentSimilarityThreshold: 0.8
43
+ });
44
+
45
+ this.snapshotManager = new SnapshotManager({
46
+ storageDir: this.options.snapshotStorageDir,
47
+ enableCompression: true,
48
+ enableDeltaStorage: true,
49
+ cacheEnabled: this.options.cacheEnabled
50
+ });
51
+
52
+ this.cache = this.options.cacheEnabled
53
+ ? new CacheManager({ ttl: this.options.cacheTTL })
54
+ : null;
55
+
56
+ this.activeMonitors = new Map();
57
+ this.monitorStats = new Map();
58
+
59
+ this.initialize();
60
+ }
61
+
62
+ async initialize() {
63
+ try {
64
+ await this.snapshotManager.initialize();
65
+ this._setupEventHandlers();
66
+ this.emit('initialized');
67
+ } catch (error) {
68
+ this.emit('error', { operation: 'initialize', error: error.message });
69
+ throw error;
70
+ }
71
+ }
72
+
73
+ _setupEventHandlers() {
74
+ this.changeTracker.on('changeDetected', async (changeRecord) => {
75
+ if (changeRecord.significance !== 'none') {
76
+ try {
77
+ await this.snapshotManager.storeSnapshot(
78
+ changeRecord.url,
79
+ changeRecord.details.current || '',
80
+ { changes: changeRecord.details, significance: changeRecord.significance, changeType: changeRecord.changeType }
81
+ );
82
+ } catch (error) {
83
+ this.emit('error', { operation: 'storeChangeSnapshot', url: changeRecord.url, error: error.message });
84
+ }
85
+ }
86
+ });
87
+
88
+ this.changeTracker.on('baselineCreated', (baseline) => this.emit('baselineCreated', baseline));
89
+ this.snapshotManager.on('snapshotStored', (snapshot) => this.emit('snapshotStored', snapshot));
90
+ this.snapshotManager.on('error', (error) => this.emit('error', error));
91
+ }
92
+
93
+ async execute(params) {
94
+ try {
95
+ const validated = TrackChangesSchema.parse(params);
96
+ const { operation } = validated;
97
+
98
+ switch (operation) {
99
+ case 'create_baseline': return await this.createBaseline(validated);
100
+ case 'compare': return await this.compareWithBaseline(validated);
101
+ case 'monitor': return await this.setupMonitoring(validated);
102
+ case 'get_history': return await this.getChangeHistory(validated);
103
+ case 'get_stats': return await this.getStatistics(validated);
104
+ case 'create_scheduled_monitor':return await this.createScheduledMonitor(validated);
105
+ case 'stop_scheduled_monitor': return await this.stopScheduledMonitor(validated);
106
+ case 'get_dashboard': return await this.getMonitoringDashboard(validated);
107
+ case 'export_history': return await this.exportHistoricalData(validated);
108
+ case 'create_alert_rule': return await this.createAlertRule(validated);
109
+ case 'generate_trend_report': return await this.generateTrendReport(validated);
110
+ case 'get_monitoring_templates':return await this.getMonitoringTemplates(validated);
111
+ default: throw new Error(`Unknown operation: ${operation}`);
112
+ }
113
+ } catch (error) {
114
+ return { success: false, error: error.message, timestamp: Date.now() };
115
+ }
116
+ }
117
+
118
+ async createBaseline(params) {
119
+ const { url, content, html, trackingOptions, storageOptions = {} } = params;
120
+ const enableSnapshots = storageOptions.enableSnapshots !== false;
121
+
122
+ let sourceContent = content || html;
123
+ let fetchMeta = {};
124
+ if (!sourceContent) {
125
+ const r = await fetchContent(url);
126
+ sourceContent = r.content;
127
+ fetchMeta = r.metadata;
128
+ }
129
+ if (!sourceContent || typeof sourceContent !== 'string') throw new Error('Invalid content');
130
+
131
+ const baseline = await this.changeTracker.createBaseline(url, sourceContent, trackingOptions);
132
+ let snapshotInfo = null;
133
+ if (enableSnapshots) {
134
+ snapshotInfo = await this.snapshotManager.storeSnapshot(url, sourceContent, { ...fetchMeta, baseline: true, trackingOptions });
135
+ }
136
+
137
+ return {
138
+ success: true, operation: 'create_baseline', url,
139
+ baseline: {
140
+ version: baseline.version,
141
+ contentHash: baseline.analysis?.hashes?.page,
142
+ sections: Object.keys(baseline.analysis?.hashes?.sections || {}).length,
143
+ elements: Object.keys(baseline.analysis?.hashes?.elements || {}).length,
144
+ createdAt: baseline.timestamp,
145
+ options: trackingOptions
146
+ },
147
+ snapshot: snapshotInfo, timestamp: Date.now()
148
+ };
149
+ }
150
+
151
+ async compareWithBaseline(params) {
152
+ const { url, content, html, trackingOptions, storageOptions = {}, notificationOptions } = params;
153
+ const enableSnapshots = storageOptions.enableSnapshots !== false;
154
+
155
+ let currentContent = content || html;
156
+ let fetchMeta = {};
157
+ if (!currentContent) {
158
+ const r = await fetchContent(url);
159
+ currentContent = r.content;
160
+ fetchMeta = r.metadata;
161
+ }
162
+ if (!currentContent || typeof currentContent !== 'string') throw new Error('Invalid content');
163
+
164
+ const comparisonResult = await this.changeTracker.compareWithBaseline(url, currentContent, trackingOptions);
165
+
166
+ let snapshotInfo = null;
167
+ if (comparisonResult.hasChanges && enableSnapshots) {
168
+ snapshotInfo = await this.snapshotManager.storeSnapshot(url, currentContent, {
169
+ ...fetchMeta, changes: comparisonResult.summary, significance: comparisonResult.significance
170
+ });
171
+ }
172
+
173
+ if (comparisonResult.hasChanges && notificationOptions) {
174
+ await sendNotifications(url, comparisonResult, notificationOptions, this);
175
+ }
176
+
177
+ return {
178
+ success: true, operation: 'compare', url,
179
+ hasChanges: comparisonResult.hasChanges,
180
+ significance: comparisonResult.significance,
181
+ changeType: comparisonResult.changeType,
182
+ summary: comparisonResult.summary,
183
+ details: comparisonResult.details,
184
+ metrics: comparisonResult.metrics,
185
+ recommendations: comparisonResult.recommendations,
186
+ snapshot: snapshotInfo, timestamp: Date.now()
187
+ };
188
+ }
189
+
190
+ async setupMonitoring(params) {
191
+ const { url, monitoringOptions, trackingOptions, storageOptions, notificationOptions } = params;
192
+
193
+ if (this.activeMonitors.has(url)) {
194
+ clearInterval(this.activeMonitors.get(url).timer);
195
+ }
196
+
197
+ const deps = { changeTracker: this.changeTracker, snapshotManager: this.snapshotManager, emitter: this };
198
+
199
+ const monitorConfig = {
200
+ url,
201
+ options: { ...monitoringOptions, trackingOptions, storageOptions, notificationOptions },
202
+ stats: { started: Date.now(), checks: 0, changesDetected: 0, errors: 0, lastCheck: null, lastChange: null, averageResponseTime: 0 }
203
+ };
204
+
205
+ monitorConfig.timer = setInterval(
206
+ () => performMonitoringCheck(url, monitorConfig, deps),
207
+ monitoringOptions.interval
208
+ );
209
+
210
+ this.activeMonitors.set(url, monitorConfig);
211
+ this.monitorStats.set(url, monitorConfig.stats);
212
+
213
+ await performMonitoringCheck(url, monitorConfig, deps);
214
+
215
+ return {
216
+ success: true, operation: 'monitor', url,
217
+ monitoring: { enabled: true, interval: monitoringOptions.interval, notificationThreshold: monitoringOptions.notificationThreshold, startedAt: monitorConfig.stats.started },
218
+ timestamp: Date.now()
219
+ };
220
+ }
221
+
222
+ async getChangeHistory(params) {
223
+ const { url, queryOptions } = params;
224
+
225
+ const changeHistory = this.changeTracker.getChangeHistory(url, queryOptions.limit);
226
+ const snapshotHistory = await this.snapshotManager.getChangeHistory(url, queryOptions);
227
+ let combined = mergeHistoryData(changeHistory, snapshotHistory.history);
228
+
229
+ if (queryOptions.significanceFilter && queryOptions.significanceFilter !== 'all') {
230
+ combined = combined.filter(e => matchesSignificanceFilter(e, queryOptions.significanceFilter));
231
+ }
232
+
233
+ const start = queryOptions.offset || 0;
234
+ const end = start + (queryOptions.limit || 50);
235
+
236
+ return {
237
+ success: true, operation: 'get_history', url,
238
+ history: combined.slice(start, end),
239
+ pagination: { total: combined.length, limit: queryOptions.limit, offset: queryOptions.offset, hasMore: end < combined.length },
240
+ timespan: {
241
+ earliest: combined.length > 0 ? combined[combined.length - 1].timestamp : null,
242
+ latest: combined.length > 0 ? combined[0].timestamp : null,
243
+ totalEntries: combined.length
244
+ },
245
+ timestamp: Date.now()
246
+ };
247
+ }
248
+
249
+ async getStatistics(params) {
250
+ const { url } = params;
251
+ const monitoringStats = url ? this.monitorStats.get(url) : this._getAggregatedMonitoringStats();
252
+ let urlStats = null;
253
+ if (url) {
254
+ try {
255
+ const changeHistory = this.changeTracker.getChangeHistory(url, 100);
256
+ const snapshotHistory = await this.snapshotManager.querySnapshots({ url, limit: 100, includeContent: false });
257
+ urlStats = {
258
+ totalChanges: changeHistory.length,
259
+ totalSnapshots: snapshotHistory.snapshots.length,
260
+ lastChange: changeHistory.length > 0 ? changeHistory[0].timestamp : null,
261
+ averageChangeInterval: calculateAverageInterval(changeHistory),
262
+ significanceDistribution: calculateSignificanceDistribution(changeHistory),
263
+ isBeingMonitored: this.activeMonitors.has(url)
264
+ };
265
+ } catch (error) {
266
+ urlStats = { error: error.message };
267
+ }
268
+ }
269
+
270
+ return {
271
+ success: true, operation: 'get_stats', url: url || 'global',
272
+ stats: {
273
+ changeTracking: this.changeTracker.getStats(),
274
+ snapshotStorage: this.snapshotManager.getStats(),
275
+ monitoring: monitoringStats,
276
+ urlSpecific: urlStats,
277
+ system: { activeMonitors: this.activeMonitors.size, cacheEnabled: !!this.cache, cacheStats: this.cache ? this.cache.getStats() : null }
278
+ },
279
+ timestamp: Date.now()
280
+ };
281
+ }
282
+
283
+ async createScheduledMonitor(params) {
284
+ const { url, scheduledMonitorOptions, trackingOptions, notificationOptions } = params;
285
+ const schedule = scheduledMonitorOptions?.schedule || '0 */1 * * *';
286
+ const templateId = scheduledMonitorOptions?.templateId;
287
+ let monitorOptions = { ...trackingOptions };
288
+ if (templateId && this.changeTracker.monitoringTemplates.has(templateId)) {
289
+ monitorOptions = { ...this.changeTracker.monitoringTemplates.get(templateId).options, ...monitorOptions };
290
+ }
291
+ const result = await this.changeTracker.createScheduledMonitor(url, schedule, {
292
+ ...monitorOptions,
293
+ alertRules: { threshold: 'moderate', methods: ['webhook'], throttle: 600000, ...notificationOptions }
294
+ });
295
+ return { success: true, operation: 'create_scheduled_monitor', url, monitor: result, template: templateId ? this.changeTracker.monitoringTemplates.get(templateId)?.name : null, timestamp: Date.now() };
296
+ }
297
+
298
+ async stopScheduledMonitor(params) {
299
+ const { url } = params;
300
+ let stoppedMonitors = 0;
301
+ for (const [id, monitor] of this.changeTracker.scheduledMonitors.entries()) {
302
+ if (monitor.url === url) {
303
+ monitor.cronJob?.destroy();
304
+ monitor.status = 'stopped';
305
+ this.changeTracker.scheduledMonitors.delete(id);
306
+ stoppedMonitors++;
307
+ }
308
+ }
309
+ return { success: true, operation: 'stop_scheduled_monitor', url, stoppedMonitors, timestamp: Date.now() };
310
+ }
311
+
312
+ async getMonitoringDashboard(params) {
313
+ const { dashboardOptions } = params;
314
+ const dashboard = this.changeTracker.getMonitoringDashboard();
315
+ if (!dashboardOptions?.includeRecentAlerts) delete dashboard.recentAlerts;
316
+ if (!dashboardOptions?.includeTrends) delete dashboard.trends;
317
+ if (!dashboardOptions?.includeMonitorStatus) {
318
+ dashboard.monitors = dashboard.monitors.map(m => ({ id: m.id, url: m.url, status: m.status }));
319
+ }
320
+ return { success: true, operation: 'get_dashboard', dashboard, timestamp: Date.now() };
321
+ }
322
+
323
+ async exportHistoricalData(params) {
324
+ const { url, exportOptions } = params;
325
+ const exportData = await this.changeTracker.exportHistoricalData({ ...exportOptions, url });
326
+ return { success: true, operation: 'export_history', url: url || 'global', export: exportData, timestamp: Date.now() };
327
+ }
328
+
329
+ async createAlertRule(params) {
330
+ const { alertRuleOptions } = params;
331
+ const ruleId = alertRuleOptions?.ruleId || `custom_rule_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
332
+ const rule = {
333
+ condition: this._parseCondition(alertRuleOptions?.condition || 'significance === "major"'),
334
+ actions: alertRuleOptions?.actions || ['webhook'],
335
+ throttle: alertRuleOptions?.throttle || 600000,
336
+ priority: alertRuleOptions?.priority || 'medium'
337
+ };
338
+ this.changeTracker.alertRules.set(ruleId, rule);
339
+ return { success: true, operation: 'create_alert_rule', ruleId, rule, timestamp: Date.now() };
340
+ }
341
+
342
+ async generateTrendReport(params) {
343
+ const report = await this.changeTracker.generateTrendAnalysisReport(params.url);
344
+ return { success: true, operation: 'generate_trend_report', report, timestamp: Date.now() };
345
+ }
346
+
347
+ async getMonitoringTemplates() {
348
+ const templates = {};
349
+ for (const [id, template] of this.changeTracker.monitoringTemplates.entries()) {
350
+ templates[id] = { name: template.name, frequency: template.frequency, options: template.options, alertRules: template.alertRules };
351
+ }
352
+ return { success: true, operation: 'get_monitoring_templates', templates, count: Object.keys(templates).length, timestamp: Date.now() };
353
+ }
354
+
355
+ // ── Public API ────────────────────────────────────────────────────────────────
356
+
357
+ stopMonitoring(url) {
358
+ if (!this.activeMonitors.has(url)) return false;
359
+ const monitorConfig = this.activeMonitors.get(url);
360
+ stopMonitor(url, monitorConfig, this);
361
+ this.activeMonitors.delete(url);
362
+ return true;
363
+ }
364
+
365
+ stopAllMonitoring() {
366
+ const urls = Array.from(this.activeMonitors.keys());
367
+ urls.forEach(url => this.stopMonitoring(url));
368
+ this.emit('allMonitoringStopped', { count: urls.length });
369
+ return urls.length;
370
+ }
371
+
372
+ getActiveMonitors() {
373
+ return Array.from(this.activeMonitors.keys()).map(url => ({
374
+ url,
375
+ config: this.activeMonitors.get(url).options,
376
+ stats: this.monitorStats.get(url)
377
+ }));
378
+ }
379
+
380
+ async shutdown() {
381
+ this.stopAllMonitoring();
382
+ await this.snapshotManager.shutdown();
383
+ await this.changeTracker.cleanup();
384
+ this.emit('shutdown');
385
+ }
386
+
387
+ // ── Private helpers ────────────────────────────────────────────────────────────
388
+
389
+ _getAggregatedMonitoringStats() {
390
+ const stats = { totalMonitors: this.activeMonitors.size, totalChecks: 0, totalChanges: 0, totalErrors: 0, averageResponseTime: 0, oldestMonitor: null, newestMonitor: null };
391
+ const all = Array.from(this.monitorStats.values());
392
+ if (all.length === 0) return stats;
393
+ stats.totalChecks = all.reduce((s, v) => s + v.checks, 0);
394
+ stats.totalChanges = all.reduce((s, v) => s + v.changesDetected, 0);
395
+ stats.totalErrors = all.reduce((s, v) => s + v.errors, 0);
396
+ stats.averageResponseTime = all.reduce((s, v) => s + v.averageResponseTime, 0) / all.length;
397
+ stats.oldestMonitor = Math.min(...all.map(v => v.started));
398
+ stats.newestMonitor = Math.max(...all.map(v => v.started));
399
+ return stats;
400
+ }
401
+
402
+ _parseCondition(conditionString) {
403
+ return (changeResult) => {
404
+ try {
405
+ if (conditionString.includes('significance')) {
406
+ const match = conditionString.match(/significance\s*===\s*["'](\w+)["']/);
407
+ if (match) return changeResult.significance === match[1];
408
+ }
409
+ return false;
410
+ } catch {
411
+ return false;
412
+ }
413
+ };
414
+ }
415
+ }
416
+
417
+ export default TrackChangesTool;
418
+
419
+ // Singleton instance — kept for backward-compat with any code that imports it directly
420
+ export const trackChangesTool = new TrackChangesTool();
421
+ trackChangesTool.name = 'track_changes';
422
+ trackChangesTool.validateParameters = (params) => TrackChangesSchema.parse(params);
423
+ trackChangesTool.description = 'Track and analyze content changes with baseline capture, comparison, and monitoring capabilities';
424
+ trackChangesTool.inputSchema = {
425
+ type: 'object',
426
+ properties: {
427
+ url: { type: 'string', description: 'URL to track for changes' },
428
+ operation: { type: 'string', description: 'Operation to perform: create_baseline, compare, monitor, get_history, get_stats' },
429
+ content: { type: 'string', description: 'Content to analyze or compare' }
430
+ },
431
+ required: ['url']
432
+ };
@@ -0,0 +1,93 @@
1
+ /**
2
+ * TrackChanges — monitor module.
3
+ * Handles the polling monitor lifecycle: start, check, stop.
4
+ * Used by the TrackChangesTool class (index.js).
5
+ */
6
+
7
+ import { fetchContent, meetsNotificationThreshold } from './differ.js';
8
+ import { sendNotifications } from './notifier.js';
9
+
10
+ /**
11
+ * Perform a single monitoring check for a URL.
12
+ * Mutates monitorConfig.stats in place.
13
+ *
14
+ * @param {string} url
15
+ * @param {Object} monitorConfig
16
+ * @param {Object} deps — { changeTracker, snapshotManager, emitter }
17
+ */
18
+ export async function performMonitoringCheck(url, monitorConfig, { changeTracker, snapshotManager, emitter }) {
19
+ const startTime = Date.now();
20
+
21
+ try {
22
+ monitorConfig.stats.checks++;
23
+
24
+ const fetchResult = await fetchContent(url);
25
+
26
+ const comparisonResult = await changeTracker.compareWithBaseline(
27
+ url,
28
+ fetchResult.content,
29
+ monitorConfig.options.trackingOptions
30
+ );
31
+
32
+ const responseTime = Date.now() - startTime;
33
+ monitorConfig.stats.averageResponseTime =
34
+ (monitorConfig.stats.averageResponseTime * (monitorConfig.stats.checks - 1) + responseTime) /
35
+ monitorConfig.stats.checks;
36
+
37
+ monitorConfig.stats.lastCheck = Date.now();
38
+
39
+ if (comparisonResult.hasChanges) {
40
+ monitorConfig.stats.changesDetected++;
41
+ monitorConfig.stats.lastChange = Date.now();
42
+
43
+ if (meetsNotificationThreshold(
44
+ comparisonResult.significance,
45
+ monitorConfig.options.notificationThreshold
46
+ )) {
47
+ if (monitorConfig.options.storageOptions?.enableSnapshots) {
48
+ await snapshotManager.storeSnapshot(url, fetchResult.content, {
49
+ ...fetchResult.metadata,
50
+ changes: comparisonResult.summary,
51
+ significance: comparisonResult.significance,
52
+ monitoring: true
53
+ });
54
+ }
55
+
56
+ if (monitorConfig.options.notificationOptions) {
57
+ await sendNotifications(url, comparisonResult, monitorConfig.options.notificationOptions, emitter);
58
+ }
59
+ }
60
+ }
61
+
62
+ emitter?.emit('monitoringCheck', {
63
+ url,
64
+ hasChanges: comparisonResult.hasChanges,
65
+ significance: comparisonResult.significance,
66
+ responseTime,
67
+ timestamp: Date.now()
68
+ });
69
+ } catch (error) {
70
+ monitorConfig.stats.errors++;
71
+
72
+ emitter?.emit('monitoringError', { url, error: error.message, timestamp: Date.now() });
73
+
74
+ if (monitorConfig.stats.errors > monitorConfig.options.maxRetries) {
75
+ stopMonitor(url, monitorConfig, emitter);
76
+ emitter?.emit('monitoringDisabled', {
77
+ url,
78
+ reason: 'Too many errors',
79
+ totalErrors: monitorConfig.stats.errors
80
+ });
81
+ }
82
+ }
83
+ }
84
+
85
+ /**
86
+ * Stop a single active monitor (clears its interval).
87
+ */
88
+ export function stopMonitor(url, monitorConfig, emitter) {
89
+ if (monitorConfig?.timer) {
90
+ clearInterval(monitorConfig.timer);
91
+ }
92
+ emitter?.emit('monitoringStopped', { url });
93
+ }
@@ -0,0 +1,105 @@
1
+ /**
2
+ * TrackChanges — notifier module.
3
+ * Handles webhook, email and Slack change notifications.
4
+ * Used by monitor.js and the main TrackChangesTool class.
5
+ */
6
+
7
+ /**
8
+ * Send all enabled notifications for a detected change.
9
+ * @param {string} url
10
+ * @param {Object} changeResult
11
+ * @param {Object} notificationOptions
12
+ * @param {EventEmitter} emitter — tool instance for event emission
13
+ */
14
+ export async function sendNotifications(url, changeResult, notificationOptions, emitter) {
15
+ const notifications = [];
16
+
17
+ if (notificationOptions.webhook?.enabled) {
18
+ notifications.push(sendWebhookNotification(url, changeResult, notificationOptions.webhook, emitter));
19
+ }
20
+ if (notificationOptions.email?.enabled) {
21
+ notifications.push(sendEmailNotification(url, changeResult, notificationOptions.email, emitter));
22
+ }
23
+ if (notificationOptions.slack?.enabled) {
24
+ notifications.push(sendSlackNotification(url, changeResult, notificationOptions.slack, emitter));
25
+ }
26
+
27
+ await Promise.allSettled(notifications);
28
+ }
29
+
30
+ export async function sendWebhookNotification(url, changeResult, webhookConfig, emitter) {
31
+ try {
32
+ const payload = {
33
+ event: 'change_detected',
34
+ url,
35
+ timestamp: Date.now(),
36
+ significance: changeResult.significance,
37
+ changeType: changeResult.changeType,
38
+ summary: changeResult.summary,
39
+ details: webhookConfig.includeContent ? changeResult.details : undefined
40
+ };
41
+
42
+ const response = await fetch(webhookConfig.url, {
43
+ method: webhookConfig.method || 'POST',
44
+ headers: {
45
+ 'Content-Type': 'application/json',
46
+ 'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0',
47
+ ...webhookConfig.headers
48
+ },
49
+ body: JSON.stringify(payload)
50
+ });
51
+
52
+ if (!response.ok) {
53
+ throw new Error(`Webhook failed: ${response.status} ${response.statusText}`);
54
+ }
55
+
56
+ emitter?.emit('notificationSent', { type: 'webhook', url, success: true });
57
+ } catch (error) {
58
+ emitter?.emit('notificationError', { type: 'webhook', url, error: error.message });
59
+ }
60
+ }
61
+
62
+ export async function sendEmailNotification(url, changeResult, emailConfig, emitter) {
63
+ // Email integration placeholder — requires external service
64
+ emitter?.emit('notificationSent', {
65
+ type: 'email',
66
+ url,
67
+ success: true,
68
+ note: 'Email notifications require external service integration'
69
+ });
70
+ }
71
+
72
+ export async function sendSlackNotification(url, changeResult, slackConfig, emitter) {
73
+ try {
74
+ const colors = { none: '#36a64f', minor: '#ffeb3b', moderate: '#ff9800', major: '#f44336', critical: '#9c27b0' };
75
+ const payload = {
76
+ text: '🔄 Content Change Detected',
77
+ attachments: [{
78
+ color: colors[changeResult.significance] || '#36a64f',
79
+ fields: [
80
+ { title: 'URL', value: url, short: false },
81
+ { title: 'Significance', value: changeResult.significance.toUpperCase(), short: true },
82
+ { title: 'Change Type', value: changeResult.changeType.replace('_', ' '), short: true },
83
+ { title: 'Summary', value: changeResult.summary.changeDescription, short: false }
84
+ ],
85
+ timestamp: Math.floor(Date.now() / 1000)
86
+ }],
87
+ channel: slackConfig.channel,
88
+ username: slackConfig.username || 'Change Tracker'
89
+ };
90
+
91
+ const response = await fetch(slackConfig.webhookUrl, {
92
+ method: 'POST',
93
+ headers: { 'Content-Type': 'application/json' },
94
+ body: JSON.stringify(payload)
95
+ });
96
+
97
+ if (!response.ok) {
98
+ throw new Error(`Slack notification failed: ${response.status}`);
99
+ }
100
+
101
+ emitter?.emit('notificationSent', { type: 'slack', url, success: true });
102
+ } catch (error) {
103
+ emitter?.emit('notificationError', { type: 'slack', url, error: error.message });
104
+ }
105
+ }