crawlforge-mcp-server 3.0.18 → 3.3.1

package/src/tools/tracking/trackChanges/monitor.js

@@ -0,0 +1,93 @@
+/**
+ * TrackChanges — monitor module.
+ * Handles the polling monitor lifecycle: start, check, stop.
+ * Used by the TrackChangesTool class (index.js).
+ */
+
+import { fetchContent, meetsNotificationThreshold } from './differ.js';
+import { sendNotifications } from './notifier.js';
+
+/**
+ * Perform a single monitoring check for a URL.
+ * Mutates monitorConfig.stats in place.
+ *
+ * @param {string} url
+ * @param {Object} monitorConfig
+ * @param {Object} deps — { changeTracker, snapshotManager, emitter }
+ */
+export async function performMonitoringCheck(url, monitorConfig, { changeTracker, snapshotManager, emitter }) {
+  const startTime = Date.now();
+
+  try {
+    monitorConfig.stats.checks++;
+
+    const fetchResult = await fetchContent(url);
+
+    const comparisonResult = await changeTracker.compareWithBaseline(
+      url,
+      fetchResult.content,
+      monitorConfig.options.trackingOptions
+    );
+
+    const responseTime = Date.now() - startTime;
+    monitorConfig.stats.averageResponseTime =
+      (monitorConfig.stats.averageResponseTime * (monitorConfig.stats.checks - 1) + responseTime) /
+      monitorConfig.stats.checks;
+
+    monitorConfig.stats.lastCheck = Date.now();
+
+    if (comparisonResult.hasChanges) {
+      monitorConfig.stats.changesDetected++;
+      monitorConfig.stats.lastChange = Date.now();
+
+      if (meetsNotificationThreshold(
+        comparisonResult.significance,
+        monitorConfig.options.notificationThreshold
+      )) {
+        if (monitorConfig.options.storageOptions?.enableSnapshots) {
+          await snapshotManager.storeSnapshot(url, fetchResult.content, {
+            ...fetchResult.metadata,
+            changes: comparisonResult.summary,
+            significance: comparisonResult.significance,
+            monitoring: true
+          });
+        }
+
+        if (monitorConfig.options.notificationOptions) {
+          await sendNotifications(url, comparisonResult, monitorConfig.options.notificationOptions, emitter);
+        }
+      }
+    }
+
+    emitter?.emit('monitoringCheck', {
+      url,
+      hasChanges: comparisonResult.hasChanges,
+      significance: comparisonResult.significance,
+      responseTime,
+      timestamp: Date.now()
+    });
+  } catch (error) {
+    monitorConfig.stats.errors++;
+
+    emitter?.emit('monitoringError', { url, error: error.message, timestamp: Date.now() });
+
+    if (monitorConfig.stats.errors > monitorConfig.options.maxRetries) {
+      stopMonitor(url, monitorConfig, emitter);
+      emitter?.emit('monitoringDisabled', {
+        url,
+        reason: 'Too many errors',
+        totalErrors: monitorConfig.stats.errors
+      });
+    }
+  }
+}
+
+/**
+ * Stop a single active monitor (clears its interval).
+ */
+export function stopMonitor(url, monitorConfig, emitter) {
+  if (monitorConfig?.timer) {
+    clearInterval(monitorConfig.timer);
+  }
+  emitter?.emit('monitoringStopped', { url });
+}

package/src/tools/tracking/trackChanges/notifier.js

@@ -0,0 +1,105 @@
+/**
+ * TrackChanges — notifier module.
+ * Handles webhook, email and Slack change notifications.
+ * Used by monitor.js and the main TrackChangesTool class.
+ */
+
+/**
+ * Send all enabled notifications for a detected change.
+ * @param {string} url
+ * @param {Object} changeResult
+ * @param {Object} notificationOptions
+ * @param {EventEmitter} emitter — tool instance for event emission
+ */
+export async function sendNotifications(url, changeResult, notificationOptions, emitter) {
+  const notifications = [];
+
+  if (notificationOptions.webhook?.enabled) {
+    notifications.push(sendWebhookNotification(url, changeResult, notificationOptions.webhook, emitter));
+  }
+  if (notificationOptions.email?.enabled) {
+    notifications.push(sendEmailNotification(url, changeResult, notificationOptions.email, emitter));
+  }
+  if (notificationOptions.slack?.enabled) {
+    notifications.push(sendSlackNotification(url, changeResult, notificationOptions.slack, emitter));
+  }
+
+  await Promise.allSettled(notifications);
+}
+
+export async function sendWebhookNotification(url, changeResult, webhookConfig, emitter) {
+  try {
+    const payload = {
+      event: 'change_detected',
+      url,
+      timestamp: Date.now(),
+      significance: changeResult.significance,
+      changeType: changeResult.changeType,
+      summary: changeResult.summary,
+      details: webhookConfig.includeContent ? changeResult.details : undefined
+    };
+
+    const response = await fetch(webhookConfig.url, {
+      method: webhookConfig.method || 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0',
+        ...webhookConfig.headers
+      },
+      body: JSON.stringify(payload)
+    });
+
+    if (!response.ok) {
+      throw new Error(`Webhook failed: ${response.status} ${response.statusText}`);
+    }
+
+    emitter?.emit('notificationSent', { type: 'webhook', url, success: true });
+  } catch (error) {
+    emitter?.emit('notificationError', { type: 'webhook', url, error: error.message });
+  }
+}
+
+export async function sendEmailNotification(url, changeResult, emailConfig, emitter) {
+  // Email integration placeholder — requires external service
+  emitter?.emit('notificationSent', {
+    type: 'email',
+    url,
+    success: true,
+    note: 'Email notifications require external service integration'
+  });
+}
+
+export async function sendSlackNotification(url, changeResult, slackConfig, emitter) {
+  try {
+    const colors = { none: '#36a64f', minor: '#ffeb3b', moderate: '#ff9800', major: '#f44336', critical: '#9c27b0' };
+    const payload = {
+      text: '🔄 Content Change Detected',
+      attachments: [{
+        color: colors[changeResult.significance] || '#36a64f',
+        fields: [
+          { title: 'URL', value: url, short: false },
+          { title: 'Significance', value: changeResult.significance.toUpperCase(), short: true },
+          { title: 'Change Type', value: changeResult.changeType.replace('_', ' '), short: true },
+          { title: 'Summary', value: changeResult.summary.changeDescription, short: false }
+        ],
+        timestamp: Math.floor(Date.now() / 1000)
+      }],
+      channel: slackConfig.channel,
+      username: slackConfig.username || 'Change Tracker'
+    };
+
+    const response = await fetch(slackConfig.webhookUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload)
+    });
+
+    if (!response.ok) {
+      throw new Error(`Slack notification failed: ${response.status}`);
+    }
+
+    emitter?.emit('notificationSent', { type: 'slack', url, success: true });
+  } catch (error) {
+    emitter?.emit('notificationError', { type: 'slack', url, error: error.message });
+  }
+}

package/src/tools/tracking/trackChanges/schema.js

@@ -0,0 +1,127 @@
+/**
+ * TrackChanges — schema module.
+ * Centralises the Zod input schema so monitor.js, differ.js, notifier.js
+ * and the entry-point index.js can all import from one place.
+ */
+
+import { z } from 'zod';
+
+export const TrackChangesSchema = z.object({
+  url: z.string().url(),
+  operation: z.enum([
+    'create_baseline',
+    'compare',
+    'monitor',
+    'get_history',
+    'get_stats',
+    'create_scheduled_monitor',
+    'stop_scheduled_monitor',
+    'get_dashboard',
+    'export_history',
+    'create_alert_rule',
+    'generate_trend_report',
+    'get_monitoring_templates'
+  ]).default('compare'),
+
+  content: z.string().optional(),
+  html: z.string().optional(),
+
+  trackingOptions: z.object({
+    granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
+    trackText: z.boolean().default(true),
+    trackStructure: z.boolean().default(true),
+    trackAttributes: z.boolean().default(false),
+    trackImages: z.boolean().default(false),
+    trackLinks: z.boolean().default(true),
+    ignoreWhitespace: z.boolean().default(true),
+    ignoreCase: z.boolean().default(false),
+    customSelectors: z.array(z.string()).optional(),
+    excludeSelectors: z.array(z.string()).optional().default([
+      'script', 'style', 'noscript', '.advertisement', '.ad', '#comments'
+    ]),
+    significanceThresholds: z.object({
+      minor: z.number().min(0).max(1).default(0.1),
+      moderate: z.number().min(0).max(1).default(0.3),
+      major: z.number().min(0).max(1).default(0.7)
+    }).optional()
+  }).optional().default({}),
+
+  monitoringOptions: z.object({
+    enabled: z.boolean().default(false),
+    interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
+    maxRetries: z.number().min(0).max(5).default(3),
+    retryDelay: z.number().min(1000).max(60000).default(5000),
+    notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).default('moderate'),
+    enableWebhook: z.boolean().default(false),
+    webhookUrl: z.string().url().optional(),
+    webhookSecret: z.string().optional()
+  }).optional(),
+
+  storageOptions: z.object({
+    enableSnapshots: z.boolean().default(true),
+    retainHistory: z.boolean().default(true),
+    maxHistoryEntries: z.number().min(1).max(1000).default(100),
+    compressionEnabled: z.boolean().default(true),
+    deltaStorageEnabled: z.boolean().default(true)
+  }).optional().default({}),
+
+  queryOptions: z.object({
+    limit: z.number().min(1).max(500).default(50),
+    offset: z.number().min(0).default(0),
+    startTime: z.number().optional(),
+    endTime: z.number().optional(),
+    includeContent: z.boolean().default(false),
+    significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
+  }).optional(),
+
+  notificationOptions: z.object({
+    email: z.object({
+      enabled: z.boolean().default(false),
+      recipients: z.array(z.string().email()).optional(),
+      subject: z.string().optional(),
+      includeDetails: z.boolean().default(true)
+    }).optional(),
+    webhook: z.object({
+      enabled: z.boolean().default(false),
+      url: z.string().url().optional(),
+      method: z.enum(['POST', 'PUT']).default('POST'),
+      headers: z.record(z.string()).optional(),
+      signingSecret: z.string().optional(),
+      includeContent: z.boolean().default(false)
+    }).optional(),
+    slack: z.object({
+      enabled: z.boolean().default(false),
+      webhookUrl: z.string().url().optional(),
+      channel: z.string().optional(),
+      username: z.string().optional()
+    }).optional()
+  }).optional(),
+
+  scheduledMonitorOptions: z.object({
+    schedule: z.string().optional(),
+    templateId: z.string().optional(),
+    enabled: z.boolean().default(true)
+  }).optional(),
+
+  alertRuleOptions: z.object({
+    ruleId: z.string().optional(),
+    condition: z.string().optional(),
+    actions: z.array(z.enum(['webhook', 'email', 'slack'])).optional(),
+    throttle: z.number().min(0).optional(),
+    priority: z.enum(['low', 'medium', 'high']).optional()
+  }).optional(),
+
+  exportOptions: z.object({
+    format: z.enum(['json', 'csv']).default('json'),
+    startTime: z.number().optional(),
+    endTime: z.number().optional(),
+    includeContent: z.boolean().default(false),
+    includeSnapshots: z.boolean().default(false)
+  }).optional(),
+
+  dashboardOptions: z.object({
+    includeRecentAlerts: z.boolean().default(true),
+    includeTrends: z.boolean().default(true),
+    includeMonitorStatus: z.boolean().default(true)
+  }).optional()
+});