crawlforge-mcp-server 5.2.9 → 5.3.0

package/src/tools/scrape/_brandingExtractor.js

@@ -258,7 +258,43 @@ function extractColors(cssText, $, cssVariables) {
     .slice(0, 24);
 }
 
-function extractFonts(cssText, $) {
+// Splits a CSS value on commas that are not inside parentheses, so a
+// `var(--x, sans-serif)` reference survives as one entry instead of being cut in half.
+function splitTopLevel(value) {
+  const parts = [];
+  let depth = 0;
+  let cur = '';
+  for (const ch of value) {
+    if (ch === '(') depth++;
+    else if (ch === ')') depth = Math.max(0, depth - 1);
+    if (ch === ',' && depth === 0) {
+      parts.push(cur);
+      cur = '';
+      continue;
+    }
+    cur += ch;
+  }
+  parts.push(cur);
+  return parts.map((p) => p.trim()).filter(Boolean);
+}
+
+// Resolves a `var(--x)` / `var(--x, fallback)` entry against the collected variables,
+// returning the family names it stands for. An unresolvable reference yields nothing
+// rather than leaking `var(` text into the font list.
+function resolveVarRef(entry, cssVariables, seen) {
+  const m = entry.match(/^var\(\s*(--[\w-]+)\s*(?:,([\s\S]*))?\)$/);
+  if (!m) return [entry];
+  const name = m[1];
+  const fallback = m[2];
+  if (!seen.has(name)) {
+    seen.add(name);
+    const value = cssVariables[name];
+    if (value) return splitTopLevel(value).flatMap((v) => resolveVarRef(v, cssVariables, seen));
+  }
+  return fallback ? splitTopLevel(fallback).flatMap((v) => resolveVarRef(v, cssVariables, seen)) : [];
+}
+
+function extractFonts(cssText, $, cssVariables) {
   const families = new Map();
   const generics = new Set();
   const fontFaces = [];
@@ -266,7 +302,10 @@ function extractFonts(cssText, $) {
   const ffRe = /font-family\s*:\s*([^;}{]+)/gi;
   let m;
   while ((m = ffRe.exec(cssText)) !== null) {
-    const list = m[1].split(',').map((f) => f.trim().replace(/^['"]|['"]$/g, '')).filter(Boolean);
+    const list = splitTopLevel(m[1])
+      .flatMap((f) => resolveVarRef(f, cssVariables, new Set()))
+      .map((f) => f.trim().replace(/^['"]|['"]$/g, ''))
+      .filter((f) => f && !f.includes('var('));
     for (const f of list) {
       const lf = f.toLowerCase();
       if (GENERIC_FAMILIES.has(lf)) { generics.add(lf); continue; }
@@ -386,7 +425,7 @@ export async function extractBranding($, pageUrl, opts = {}) {
 
   const cssVariables = safe(() => extractCssVariables(sources.cssText), {}, 'css-variables');
   const colors = safe(() => extractColors(sources.cssText, $, cssVariables), [], 'colors');
-  const fontInfo = safe(() => extractFonts(sources.cssText, $), { fonts: [], genericFallbacks: [], webfontProviders: [], fontFaces: [] }, 'fonts');
+  const fontInfo = safe(() => extractFonts(sources.cssText, $, cssVariables), { fonts: [], genericFallbacks: [], webfontProviders: [], fontFaces: [] }, 'fonts');
   const logo = safe(() => extractLogo($, pageUrl), { favicons: [], ogImage: null, candidates: [], inlineHeaderSvg: null }, 'logo');
   const tokens = safe(() => extractTokens(sources.cssText, cssVariables), { radii: [], shadows: [], spacingVariables: {} }, 'tokens');
 

package/src/tools/scrape/_mainContent.js

@@ -0,0 +1,105 @@
+/**
+ * _mainContent.js — Readability main-content extraction, plus recovery of the
+ * data tables Readability drops.
+ *
+ * Readability keeps one article candidate and discards everything outside it.
+ * On a table-led page that silently loses the payload: Wikipedia's *List of
+ * S&P 500 companies* came back with zero table rows at scrape's default
+ * onlyMainContent:true, and 505 pipe-table lines with it off. No Readability
+ * option recovers them (charThreshold:100 and nbTopCandidates:20 were both
+ * probed) — the tables are simply not in the candidate — so they are
+ * re-attached afterwards.
+ *
+ * Shared by unifiedScrape (wants HTML) and extractStructured (wants text).
+ */
+
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+
+// How much of a table's text is compared against the article to decide whether
+// Readability already kept it. Long enough to be unique to that table.
+const SIGNATURE_LENGTH = 120;
+
+function normalizeWhitespace(text) {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+/**
+ * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
+ * 4 columns means the table carries data rather than layout, counting
+ * rowspan/colspan the way Readability does.
+ * @param {HTMLTableElement} table
+ * @returns {boolean}
+ */
+function isDataTable(table) {
+  let rows = 0;
+  let columns = 0;
+  for (const row of Array.from(table.rows)) {
+    rows += parseInt(row.getAttribute('rowspan') || '1', 10) || 1;
+    let columnsInRow = 0;
+    for (const cell of Array.from(row.cells)) {
+      columnsInRow += parseInt(cell.getAttribute('colspan') || '1', 10) || 1;
+    }
+    columns = Math.max(columns, columnsInRow);
+  }
+  return rows >= 10 || columns > 4;
+}
+
+/**
+ * Extract a page's main content, re-attaching data tables Readability dropped.
+ *
+ * `title` is Readability's article title, which it strips out of `html` — a
+ * caller feeding this to an LLM has to put it back or the headline is simply
+ * not in the text (the IANA page's main content never says "Example Domains").
+ *
+ * @param {string} html - the page's HTML
+ * @param {string} [url] - document URL, used as the base for relative links
+ * @returns {{ html: string|null, title: string, tablesRecovered: number }} `html`
+ *   is null when Readability found no article; callers decide what to fall back to.
+ */
+export function extractMainContent(html, url) {
+  let article;
+  try {
+    const dom = new JSDOM(html, { url });
+    article = new Readability(dom.window.document).parse();
+  } catch {
+    return { html: null, title: '', tablesRecovered: 0 };
+  }
+  if (!article || !article.content) return { html: null, title: '', tablesRecovered: 0 };
+
+  const title = article.title || '';
+
+  // Nothing to recover, and no reason to pay for a second parse.
+  if (!/<table[\s>]/i.test(html)) {
+    return { html: article.content, title, tablesRecovered: 0 };
+  }
+
+  // Readability mutates the document it is handed — the dropped tables are
+  // already gone from `dom` by the time parse() returns — so recovery has to
+  // parse the original HTML again.
+  let recovered;
+  try {
+    const fresh = new JSDOM(html, { url });
+    const kept = normalizeWhitespace(article.textContent || '');
+    recovered = Array.from(fresh.window.document.querySelectorAll('table'))
+      // A nested table travels with its parent; re-attaching it separately
+      // would duplicate it.
+      .filter((table) => !table.parentElement?.closest('table'))
+      .filter(isDataTable)
+      .filter((table) => {
+        const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
+        return signature.length > 0 && !kept.includes(signature);
+      })
+      .map((table) => table.outerHTML);
+  } catch {
+    return { html: article.content, title, tablesRecovered: 0 };
+  }
+
+  return {
+    html: recovered.length > 0 ? `${article.content}\n${recovered.join('\n')}` : article.content,
+    title,
+    tablesRecovered: recovered.length
+  };
+}
+
+export default extractMainContent;

package/src/tools/scrape/unifiedScrape.js

@@ -11,12 +11,11 @@
  */
 
 import { z } from 'zod';
-import { JSDOM } from 'jsdom';
-import { Readability } from '@mozilla/readability';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
+import { extractMainContent } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
-import { extractBlockText, readabilityToMarkdown } from '../basic/extractText.js';
+import { extractBlockText } from '../basic/extractText.js';
 
 // ── Schema ────────────────────────────────────────────────────────────────────
 
@@ -42,6 +41,10 @@ export const UnifiedScrapeSchema = z.object({
   resolveHiddenContent: z.enum(['linked', 'inline', 'off']).optional().default('linked'),
   // Pass-through to fetchAndParse
   timeoutMs: z.number().min(1000).max(60000).optional().default(15000),
+  // Compliance overrides, per request: identify as yourself for a target you
+  // have your own agreement with, and take responsibility for ignoring robots.
+  user_agent: z.string().optional(),
+  respect_robots: z.boolean().optional(),
   // Optional, additive: only consulted when 'branding' / 'screenshot' is requested.
   brandingOptions: z.object({
     fetchLinkedCss: z.boolean().optional().default(true),
@@ -201,10 +204,13 @@ export class UnifiedScrapeTool {
     const { url, formats, onlyMainContent, timeoutMs, brandingOptions, screenshotOptions, resolveHiddenContent } = validated;
 
     // Single fetch
-    let html, $, finalUrl;
+    let html, $, finalUrl, fetchWarnings;
     try {
-      ({ html, $, finalUrl } = await fetchAndParse(url, {
+      ({ html, $, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
         timeoutMs,
+        userAgent: validated.user_agent,
+        respectRobots: validated.respect_robots,
+        tool: 'scrape',
         stripTags: [] // we handle boilerplate ourselves
       }));
     } catch (err) {
@@ -223,19 +229,20 @@ export class UnifiedScrapeTool {
     let mainHtml = null;
     function getMainHtml() {
       if (mainHtml !== null) return mainHtml;
-      try {
-        const dom = new JSDOM(html, { url: finalUrl });
-        const reader = new Readability(dom.window.document);
-        const article = reader.parse();
-        mainHtml = article ? article.content : html;
-      } catch {
-        mainHtml = html;
+      const main = extractMainContent(html, finalUrl);
+      mainHtml = main.html ?? html;
+      if (main.tablesRecovered > 0) {
+        warnings.push(
+          `mainContent: re-attached ${main.tablesRecovered} data table(s) that main-content extraction had dropped`
+        );
       }
       return mainHtml;
     }
 
     const content = {};
-    const warnings = [];
+    // The gate's warnings (e.g. a respect_robots override) travel with the
+    // per-format ones, so the caller sees the decision in the response.
+    const warnings = [...fetchWarnings];
 
     // Kept for the rawHtml format, which must survive the strip below.
     const pristineHtml = html;
@@ -325,7 +332,7 @@ export class UnifiedScrapeTool {
         case 'markdown':
           try {
             content.markdown = onlyMainContent
-              ? readabilityToMarkdown(html, finalUrl)
+              ? htmlToMarkdown(getMainHtml())
               : htmlToMarkdown($.html('body') || html);
           } catch (err) {
             content.markdown = '';

package/src/tools/search/adapters/redditOfficialApi.js

@@ -22,13 +22,11 @@
  */
 
 import { normalizePost, normalizeTreeNodes, stripIdPrefix, stripNamePrefix } from '../redditNormalize.js';
+import { identityHeaders, resolveUserAgent } from '../../../utils/fetchIdentity.js';
 
 const TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
 const API_BASE = 'https://oauth.reddit.com';
 
-/** Reddit requires a descriptive, unique User-Agent; generic ones are throttled. */
-const DEFAULT_USER_AGENT = 'CrawlForge-MCP/5.2.1 (+https://www.crawlforge.dev)';
-
 /** Our sort is asc/desc by post date; Reddit listings/search only go newest-first. */
 const REDDIT_SORT = 'new';
 
@@ -39,7 +37,10 @@ export class RedditOfficialApiAdapter {
     }
     this.clientId = clientId;
     this.clientSecret = clientSecret;
-    this.userAgent = options.userAgent || process.env.REDDIT_USER_AGENT || DEFAULT_USER_AGENT;
+    // Reddit requires a descriptive, unique User-Agent; generic ones are
+    // throttled. The canonical identity qualifies, role-suffixed so Reddit's
+    // side can tell this traffic apart from a page crawl.
+    this.userAgent = resolveUserAgent(options.userAgent || process.env.REDDIT_USER_AGENT, 'reddit');
     this.tokenUrl = options.tokenUrl || TOKEN_URL;
     this.apiBaseUrl = options.apiBaseUrl || API_BASE;
     this.timeoutMs = options.timeoutMs ?? 30000;
@@ -60,7 +61,7 @@ export class RedditOfficialApiAdapter {
         headers: {
           Authorization: this.authHeader,
           'Content-Type': 'application/x-www-form-urlencoded',
-          'User-Agent': this.userAgent,
+          ...identityHeaders({ userAgent: this.userAgent }),
         },
         body: 'grant_type=client_credentials',
         signal: AbortSignal.timeout(this.timeoutMs),
@@ -93,7 +94,7 @@ export class RedditOfficialApiAdapter {
       let response;
       try {
         response = await fetch(url, {
-          headers: { Authorization: `Bearer ${token}`, 'User-Agent': this.userAgent },
+          headers: { Authorization: `Bearer ${token}`, ...identityHeaders({ userAgent: this.userAgent }) },
           signal: AbortSignal.timeout(this.timeoutMs),
         });
       } catch (error) {

package/src/tools/search/redditSearch.js

@@ -39,6 +39,7 @@ import {
 } from './redditNormalize.js';
 import { RedditOfficialApiAdapter } from './adapters/redditOfficialApi.js';
 import { SearchProviderFactory } from './adapters/searchProviderFactory.js';
+import { identityHeaders } from '../../utils/fetchIdentity.js';
 
 const ARCTIC_SHIFT_BASE = 'https://arctic-shift.photon-reddit.com';
 const PULLPUSH_BASE = 'https://api.pullpush.io';
@@ -46,9 +47,11 @@ const PULLPUSH_BASE = 'https://api.pullpush.io';
 /**
  * Identify ourselves. Verified live: Arctic Shift throttles UA-less clients
  * into a shared bucket (422 "Timeout. Maybe slow down a bit" while curl got
- * 200 for the same URL); with a descriptive UA it answers instantly.
+ * 200 for the same URL); with a descriptive UA it answers instantly. The
+ * canonical identity is descriptive, so the archives get the same one
+ * everything else sends.
  */
-const USER_AGENT = 'CrawlForge-MCP/5.2.1 (+https://www.crawlforge.dev)';
+const ARCHIVE_IDENTITY = identityHeaders();
 
 const RedditSearchSchema = z.object({
   query: z.string().min(1).optional(),
@@ -389,7 +392,7 @@ export class RedditSearchTool {
     let response;
     try {
       response = await fetch(url, {
-        headers: { Accept: 'application/json', 'User-Agent': USER_AGENT },
+        headers: { Accept: 'application/json', ...ARCHIVE_IDENTITY },
         signal: AbortSignal.timeout(this.timeoutMs),
       });
     } catch (error) {

package/src/tools/search/searchWeb.js

@@ -64,6 +64,14 @@ const SearchWebSchema = z.object({
   }).optional()
 });
 
+// Deduplication runs after the backend search, so asking the provider for
+// exactly `limit` items returns short whenever that page contains duplicates.
+// Over-fetch a small margin once and trim back to `limit` after dedup. Google
+// returns at most 10 items per request and bills per request, so the margin
+// never costs an extra backend search.
+const DEDUPE_OVERFETCH = 4;
+const GOOGLE_MAX_RESULTS_PER_REQUEST = 10;
+
 export class SearchWebTool {
   constructor(options = {}) {
     const {
@@ -213,7 +221,10 @@ export class SearchWebTool {
           // Perform search with localized parameters
           const searchParams = {
             query: searchQuery,
-            num: localizedParams.limit,
+            num: Math.min(
+              GOOGLE_MAX_RESULTS_PER_REQUEST,
+              localizedParams.limit + DEDUPE_OVERFETCH
+            ),
             start: localizedParams.offset + 1, // Google uses 1-based indexing
             lr: localizedParams.lr || `lang_${localizedParams.lang}`,
             safe: localizedParams.safe_search ? 'active' : 'off',
@@ -275,7 +286,13 @@ export class SearchWebTool {
           deduplicationRate: ((originalCount - processedResults.length) / originalCount * 100).toFixed(1) + '%'
         };
       }
-      
+
+      // Drop the over-fetched margin. Runs unconditionally because the extra
+      // items are requested whether or not deduplication is enabled.
+      if (processedResults.length > localizedParams.limit) {
+        processedResults = processedResults.slice(0, localizedParams.limit);
+      }
+
       // Apply ranking if enabled
       let rankingInfo = null;
       if (validated.enable_ranking && processedResults.length > 1) {
@@ -386,7 +403,8 @@ export class SearchWebTool {
 
     const adapterResult = await searchViaSearxng({
       query: validated.query,
-      limit: validated.limit,
+      // SearXNG returns a whole page per request, so the dedup margin is free.
+      limit: validated.limit + DEDUPE_OVERFETCH,
       page,
       safeSearch: validated.safe_search,
       language: validated.lang
@@ -414,6 +432,11 @@ export class SearchWebTool {
       };
     }
 
+    // Drop the over-fetched margin (see execute()).
+    if (processedResults.length > validated.limit) {
+      processedResults = processedResults.slice(0, validated.limit);
+    }
+
     let rankingInfo = null;
     if (validated.enable_ranking && processedResults.length > 1) {
       const rankingOptions = validated.ranking_weights

package/src/tools/templates/ScrapeTemplateTool.js

@@ -8,6 +8,8 @@
 
 import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 
 export class ScrapeTemplateTool {
   constructor() {
@@ -16,10 +18,11 @@ export class ScrapeTemplateTool {
 
   /**
    * Execute the scrape_template tool.
-   * @param {{ template: string, url: string, timeout?: number }} params
+   * @param {{ template: string, url: string, timeout?: number,
+   *          user_agent?: string, respect_robots?: boolean }} params
    * @returns {Promise<object>}
    */
-  async execute({ template, url, timeout = 15000 }) {
+  async execute({ template, url, timeout = 15000, user_agent, respect_robots }) {
     // list mode — return available templates without scraping
     if (template === 'list' || !url) {
       return {
@@ -40,6 +43,13 @@ export class ScrapeTemplateTool {
     // so the SSRF guard below still applies.
     const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
 
+    // Robots gate + per-host politeness before any request to the target.
+    const gate = await preflightFetch(fetchUrl, {
+      respectRobots: respect_robots,
+      userAgent: user_agent,
+      tool: 'scrape_template'
+    });
+
     // Fetch the page
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
@@ -47,13 +57,14 @@ export class ScrapeTemplateTool {
     try {
       const response = await safeFetch(fetchUrl, {
         signal: controller.signal,
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; CrawlForge-TemplateScraper/4.0)'
-        }
+        headers: { ...gate.headers }
       });
       clearTimeout(timeoutId);
 
       if (!response.ok) {
+        if (response.status === 429 || response.status === 503) {
+          noteRetryAfter(fetchUrl, response.headers.get('retry-after'));
+        }
         throw new Error(`HTTP ${response.status}: ${response.statusText}`);
       }
       html = await response.text();
@@ -67,7 +78,7 @@ export class ScrapeTemplateTool {
 
     // Run the template extractor
     const result = await this.registry.run(template, html, url, fetchUrl);
-    return result;
+    return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
 }
 

package/src/tools/tracking/trackChanges/differ.js

@@ -4,6 +4,8 @@
  */
 
 import { safeFetch } from '../../../utils/ssrfGuard.js';
+import { preflightFetch } from '../../../utils/robotsGate.js';
+import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 
 /**
  * Default Jaccard similarity threshold below which a change is considered
@@ -39,14 +41,31 @@ export function calculateSimilarity(text1, text2) {
 
 /**
  * Fetch the HTML/text content of a URL with change-tracking headers.
+ *
+ * Sends the same identity as every other fetching tool: a baseline captured
+ * under one UA and compared under another reports the difference between two
+ * server-side renderings as a content change.
+ *
  * @param {string} url
- * @returns {Promise<{ content: string, metadata: Object }>}
+ * @param {object} [options]
+ * @param {string}  [options.userAgent]     per-request identity override
+ * @param {boolean} [options.respectRobots] per-request robots override
+ * @param {string}  [options.tool]          tool name, for the audit row
+ * @param {string}  [options.apiKey]        hashed into the audit row
+ * @returns {Promise<{ content: string, metadata: Object, warnings: string[] }>}
  */
-export async function fetchContent(url) {
+export async function fetchContent(url, options = {}) {
+  const gate = await preflightFetch(url, {
+    respectRobots: options.respectRobots,
+    userAgent: options.userAgent,
+    tool: options.tool || 'track_changes',
+    apiKey: options.apiKey
+  });
+
   try {
     const response = await safeFetch(url, {
       headers: {
-        'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0',
+        ...gate.headers,
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'en-US,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
@@ -56,6 +75,9 @@ export async function fetchContent(url) {
     });
 
     if (!response.ok) {
+      if (response.status === 429 || response.status === 503) {
+        noteRetryAfter(url, response.headers.get('retry-after'));
+      }
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }
 
@@ -63,6 +85,7 @@ export async function fetchContent(url) {
 
     return {
       content,
+      warnings: gate.warnings,
       metadata: {
         statusCode: response.status,
         contentType: response.headers.get('content-type'),

package/src/tools/tracking/trackChanges/index.js

@@ -185,15 +185,17 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async createBaseline(params) {
-    const { url, content, html, trackingOptions, storageOptions = {} } = params;
+    const { url, content, html, trackingOptions, storageOptions = {}, respect_robots, user_agent } = params;
     const enableSnapshots = storageOptions.enableSnapshots !== false;
 
     let sourceContent = content || html;
     let fetchMeta = {};
+    let warnings = [];
     if (!sourceContent) {
-      const r = await fetchContent(url);
+      const r = await fetchContent(url, { respectRobots: respect_robots, userAgent: user_agent });
       sourceContent = r.content;
       fetchMeta = r.metadata;
+      warnings = r.warnings;
     }
     if (!sourceContent || typeof sourceContent !== 'string') throw new Error('Invalid content');
 
@@ -217,6 +219,7 @@ export class TrackChangesTool extends EventEmitter {
         createdAt: baseline.createdAt,
         options: trackingOptions
       },
+      ...(warnings.length ? { warnings } : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }
@@ -246,16 +249,18 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async compareWithBaseline(params) {
-    const { url, content, html, trackingOptions, storageOptions = {}, notificationOptions } = params;
+    const { url, content, html, trackingOptions, storageOptions = {}, notificationOptions, respect_robots, user_agent } = params;
     const enableSnapshots = storageOptions.enableSnapshots !== false;
     await this.rehydrateBaseline(url, trackingOptions);
 
     let currentContent = content || html;
     let fetchMeta = {};
+    let fetchWarnings = [];
     if (!currentContent) {
-      const r = await fetchContent(url);
+      const r = await fetchContent(url, { respectRobots: respect_robots, userAgent: user_agent });
       currentContent = r.content;
       fetchMeta = r.metadata;
+      fetchWarnings = r.warnings;
     }
     if (!currentContent || typeof currentContent !== 'string') throw new Error('Invalid content');
 
@@ -281,7 +286,9 @@ export class TrackChangesTool extends EventEmitter {
       details: comparisonResult.details,
       metrics: comparisonResult.metrics,
       recommendations: comparisonResult.recommendations,
-      ...(comparisonResult.warnings ? { warnings: comparisonResult.warnings } : {}),
+      ...(comparisonResult.warnings || fetchWarnings.length
+        ? { warnings: [...(comparisonResult.warnings || []), ...fetchWarnings] }
+        : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }

package/src/tools/tracking/trackChanges/notifier.js

@@ -4,6 +4,8 @@
  * Used by monitor.js and the main TrackChangesTool class.
  */
 
+import { identityHeaders } from '../../../utils/fetchIdentity.js';
+
 /**
  * Send all enabled notifications for a detected change.
  * @param {string} url
@@ -43,7 +45,7 @@ export async function sendWebhookNotification(url, changeResult, webhookConfig,
       method: webhookConfig.method || 'POST',
       headers: {
         'Content-Type': 'application/json',
-        'User-Agent': 'MCP-WebScraper-ChangeTracker/3.0',
+        ...identityHeaders({ role: 'webhook' }),
         ...webhookConfig.headers
       },
       body: JSON.stringify(payload)

package/src/tools/tracking/trackChanges/schema.js

@@ -27,6 +27,9 @@ export const TrackChangesSchema = z.object({
   content: z.string().optional(),
   html: z.string().optional(),
 
+  respect_robots: z.boolean().optional(),
+  user_agent: z.string().optional(),
+
   trackingOptions: z.object({
     granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
     trackText: z.boolean().default(true),