crawlforge-mcp-server 5.4.1 → 5.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.4.1",
+  "version": "5.4.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.4.0",
+    "crawlforge-extractors": "^1.4.1",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.3.1",
+  version: "5.4.3",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/ChangeTracker.js

@@ -103,6 +103,79 @@ function extractMonetaryValues(text) {
   return values;
 }
 
+/**
+ * Built-in monitoring presets, served by track_changes
+ * {operation:"get_monitoring_templates"} and applied by
+ * create_scheduled_monitor via scheduledMonitorOptions.templateId. `options`
+ * are trackingOptions (the baseline analysis options); `frequency` is the
+ * polling interval in ms; `goal` is the plain-English alert goal the
+ * scheduler's LLM judge uses when one is configured.
+ */
+export const MONITORING_TEMPLATES = {
+  'price-watch': {
+    name: 'Price watch',
+    description: 'Product, plan or rate pages — hourly, and any numeric change counts.',
+    frequency: 3600000,
+    notificationThreshold: 'minor',
+    goal: 'A price, discount, fee or plan cost changed',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, significanceThresholds: { minor: 0.02, moderate: 0.1, major: 0.3 } },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'availability': {
+    name: 'Stock / availability',
+    description: 'In-stock, sold-out and waitlist states — every 15 minutes, any change counts.',
+    frequency: 900000,
+    notificationThreshold: 'minor',
+    goal: 'An item went in or out of stock, or a waitlist or pre-order opened',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, significanceThresholds: { minor: 0.01, moderate: 0.1, major: 0.3 } },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'news-feed': {
+    name: 'News / announcements',
+    description: 'Blogs, changelogs and press pages — every 30 minutes, section-level diffs so a new post registers as one change.',
+    frequency: 1800000,
+    notificationThreshold: 'moderate',
+    goal: 'A new article, release note or announcement was published',
+    options: { granularity: 'section', trackText: true, trackStructure: true, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  },
+  'documentation': {
+    name: 'Documentation',
+    description: 'API references and docs pages — daily, structure tracked so a new parameter or removed endpoint shows up.',
+    frequency: 86400000,
+    notificationThreshold: 'moderate',
+    goal: 'A documented API, parameter, default or behaviour changed',
+    options: { granularity: 'section', trackText: true, trackStructure: true, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  },
+  'regulatory': {
+    name: 'Regulation / policy',
+    description: 'Terms, policies, tariffs and rule pages — daily, and every wording change counts.',
+    frequency: 86400000,
+    notificationThreshold: 'minor',
+    goal: 'A rule, deadline, fee, threshold or policy wording changed',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, ignoreCase: false },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'job-board': {
+    name: 'Job listings',
+    description: 'Careers pages — every 6 hours, links tracked so an added or removed posting registers.',
+    frequency: 21600000,
+    notificationThreshold: 'moderate',
+    goal: 'A job opening was added or removed',
+    options: { granularity: 'section', trackText: true, trackStructure: false, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  }
+};
+
+const SIGNIFICANCE_RANK = { none: 0, minor: 1, moderate: 2, major: 3, critical: 4 };
+
+function csvCell(value) {
+  if (value === null || value === undefined) return '';
+  const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
+  return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+
 export class ChangeTracker extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -147,6 +220,16 @@ export class ChangeTracker extends EventEmitter {
       processingTime: 0
     };
     
+    // Alert rules (create_alert_rule) are evaluated on every compare; the
+    // alerts they fire are kept here for get_dashboard. Presets back
+    // get_monitoring_templates and scheduledMonitorOptions.templateId.
+    this.alertRules = new Map();
+    this.recentAlerts = [];
+    this.maxRecentAlerts = 100;
+    this.monitoringTemplates = new Map(
+      Object.entries(MONITORING_TEMPLATES).map(([id, t]) => [id, { id, ...t }])
+    );
+
     // Semantic analysis tools (if enabled)
     this.semanticAnalyzer = null;
     
@@ -311,6 +394,7 @@ export class ChangeTracker extends EventEmitter {
       }
       
       this.emit('changeDetected', changeRecord);
+      const alerts = this.evaluateAlertRules(changeRecord);
       
       const ignoredOptions = this.findIgnoredCompareOptions(options, baseline.options);
 
@@ -322,6 +406,7 @@ export class ChangeTracker extends EventEmitter {
         details: changeAnalysis,
         metrics: changeRecord.metrics,
         recommendations: this.generateChangeRecommendations(changeRecord),
+        ...(alerts.length ? { alerts } : {}),
         ...(ignoredOptions.length ? {
           warnings: [
             `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
@@ -1654,6 +1739,252 @@ export class ChangeTracker extends EventEmitter {
     };
   }
 
+  // ── Alert rules, dashboard, export, trends, presets ─────────────────────────
+  // Backing for the track_changes operations create_alert_rule, get_dashboard,
+  // export_history, generate_trend_report and get_monitoring_templates. The
+  // tool has called these since v3.1.0; until 5.4.3 none of them existed.
+
+  /**
+   * Turn an alert-rule condition string into a predicate over a change record.
+   * Supported: `significance === "major"`, `significance >= "moderate"` and
+   * the other comparison operators against the significance ladder
+   * none < minor < moderate < major < critical. Anything else never matches.
+   */
+  static parseAlertCondition(conditionString = '') {
+    const match = String(conditionString).match(/significance\s*(===|==|!==|!=|>=|<=|>|<)\s*["'](\w+)["']/);
+    if (!match) return () => false;
+    const [, op, level] = match;
+    const target = SIGNIFICANCE_RANK[level];
+    if (target === undefined) return () => false;
+    return (record) => {
+      const rank = SIGNIFICANCE_RANK[record?.significance];
+      if (rank === undefined) return false;
+      switch (op) {
+        case '===': case '==': return rank === target;
+        case '!==': case '!=': return rank !== target;
+        case '>=': return rank >= target;
+        case '<=': return rank <= target;
+        case '>': return rank > target;
+        default: return rank < target;
+      }
+    };
+  }
+
+  /**
+   * Register an alert rule. `condition` is a string (see parseAlertCondition);
+   * `url` scopes the rule to one tracked page, or every page when omitted.
+   */
+  addAlertRule(ruleId, { condition, url = null, actions = ['webhook'], throttle = 600000, priority = 'medium' } = {}) {
+    const rule = {
+      id: ruleId,
+      url,
+      conditionText: condition || 'significance === "major"',
+      condition: ChangeTracker.parseAlertCondition(condition || 'significance === "major"'),
+      actions,
+      throttle,
+      priority,
+      createdAt: Date.now(),
+      fireCount: 0,
+      suppressedCount: 0,
+      lastFiredAt: null
+    };
+    this.alertRules.set(ruleId, rule);
+    return rule;
+  }
+
+  /** Evaluate every rule against a change record; returns the alerts fired. */
+  evaluateAlertRules(changeRecord) {
+    const fired = [];
+    const now = changeRecord.timestamp || Date.now();
+    for (const rule of this.alertRules.values()) {
+      if (rule.url && rule.url !== changeRecord.url) continue;
+      let matches = false;
+      try { matches = rule.condition(changeRecord); } catch { matches = false; }
+      if (!matches) continue;
+      if (rule.lastFiredAt && rule.throttle && now - rule.lastFiredAt < rule.throttle) {
+        rule.suppressedCount++;
+        continue;
+      }
+      rule.lastFiredAt = now;
+      rule.fireCount++;
+      const alert = {
+        id: `alert_${now}_${Math.random().toString(36).slice(2, 8)}`,
+        ruleId: rule.id,
+        url: changeRecord.url,
+        timestamp: now,
+        significance: changeRecord.significance,
+        changeType: changeRecord.changeType,
+        similarity: changeRecord.details?.similarity ?? null,
+        priority: rule.priority,
+        actions: rule.actions
+      };
+      this.recentAlerts.unshift(alert);
+      if (this.recentAlerts.length > this.maxRecentAlerts) this.recentAlerts.length = this.maxRecentAlerts;
+      fired.push(alert);
+      this.emit('alert', alert);
+    }
+    return fired;
+  }
+
+  /** Per-URL change trend over the in-memory history. */
+  summarizeTrend(url) {
+    const history = this.changeHistory.get(url) || [];
+    const changes = history.filter(r => r.significance !== 'none');
+    const significanceDistribution = { none: 0, minor: 0, moderate: 0, major: 0, critical: 0 };
+    for (const r of history) {
+      significanceDistribution[r.significance] = (significanceDistribution[r.significance] || 0) + 1;
+    }
+    const similarities = history.map(r => r.details?.similarity).filter(n => typeof n === 'number');
+    const avg = (arr) => (arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : null);
+    const half = Math.floor(similarities.length / 2);
+    const early = avg(similarities.slice(0, half));
+    const late = avg(similarities.slice(half));
+    let direction = 'insufficient_data';
+    if (similarities.length >= 4 && early !== null && late !== null) {
+      direction = late < early - 0.05 ? 'diverging' : late > early + 0.05 ? 'stabilizing' : 'steady';
+    }
+    const span = history.length >= 2 ? history[history.length - 1].timestamp - history[0].timestamp : 0;
+    const changeTypes = {};
+    for (const r of changes) changeTypes[r.changeType] = (changeTypes[r.changeType] || 0) + 1;
+    return {
+      url,
+      compares: history.length,
+      changes: changes.length,
+      changeRate: history.length ? changes.length / history.length : 0,
+      changesPerDay: span > 0 ? changes.length / (span / 86400000) : null,
+      averageSimilarity: avg(similarities),
+      direction,
+      significanceDistribution,
+      changeTypes,
+      firstCompareAt: history[0]?.timestamp ?? null,
+      lastCompareAt: history[history.length - 1]?.timestamp ?? null,
+      lastChangeAt: changes[changes.length - 1]?.timestamp ?? null
+    };
+  }
+
+  async generateTrendAnalysisReport(url) {
+    if (url && !this.changeHistory.has(url) && !this.snapshots.has(url)) {
+      throw new Error(`No tracking history for ${url} — run create_baseline and compare first`);
+    }
+    const urls = url ? [url] : Array.from(this.changeHistory.keys());
+    const trends = urls.map(u => this.summarizeTrend(u));
+    const recommendations = [];
+    for (const t of trends) {
+      if (t.compares < 2) {
+        recommendations.push(`${t.url}: ${t.compares} compare${t.compares === 1 ? '' : 's'} recorded — run compare a few more times before reading a trend.`);
+        continue;
+      }
+      if (t.direction === 'diverging') {
+        recommendations.push(`${t.url}: each compare drifts further from the baseline — recreate the baseline if the current page is the new normal.`);
+      }
+      if (t.changes === 0) {
+        recommendations.push(`${t.url}: no changes across ${t.compares} compares — the polling interval can be lengthened.`);
+      }
+      if (t.changesPerDay !== null && t.changesPerDay > 24) {
+        recommendations.push(`${t.url}: changes more than hourly — scope the baseline with customSelectors to skip volatile regions.`);
+      }
+    }
+    return { generatedAt: Date.now(), scope: url || 'global', urlCount: urls.length, trends, recommendations };
+  }
+
+  getMonitoringDashboard() {
+    const urls = Array.from(new Set([...this.snapshots.keys(), ...this.changeHistory.keys()]));
+    const tracked = urls.map(url => {
+      const history = this.changeHistory.get(url) || [];
+      const last = history[history.length - 1] || null;
+      const snaps = this.snapshots.get(url) || [];
+      const baseline = snaps[snaps.length - 1] || null;
+      return {
+        url,
+        baselineVersion: baseline?.version ?? null,
+        baselineCreatedAt: baseline?.timestamp ?? null,
+        compares: history.length,
+        changes: history.filter(r => r.significance !== 'none').length,
+        lastCompareAt: last?.timestamp ?? null,
+        lastSignificance: last?.significance ?? null,
+        lastChangeType: last?.changeType ?? null
+      };
+    });
+    return {
+      generatedAt: Date.now(),
+      summary: {
+        trackedUrls: urls.length,
+        totalCompares: tracked.reduce((n, t) => n + t.compares, 0),
+        totalChanges: tracked.reduce((n, t) => n + t.changes, 0),
+        alertRules: this.alertRules.size,
+        alertsFired: this.recentAlerts.length,
+        activeMonitors: 0,
+        ...this.getStats()
+      },
+      tracked,
+      alertRules: Array.from(this.alertRules.values()).map(r => ({
+        id: r.id, url: r.url, condition: r.conditionText, actions: r.actions, throttle: r.throttle,
+        priority: r.priority, fireCount: r.fireCount, suppressedCount: r.suppressedCount, lastFiredAt: r.lastFiredAt
+      })),
+      recentAlerts: this.recentAlerts.slice(0, 20),
+      trends: urls.map(u => this.summarizeTrend(u)),
+      // Polling and scheduled monitors live in the tool, which fills this in.
+      monitors: []
+    };
+  }
+
+  async exportHistoricalData(options = {}) {
+    const { url, format = 'json', startTime, endTime, includeContent = false, includeSnapshots = false } = options;
+    const urls = url ? [url] : Array.from(this.changeHistory.keys());
+    const records = [];
+    for (const u of urls) {
+      for (const r of this.changeHistory.get(u) || []) {
+        if (startTime && r.timestamp < startTime) continue;
+        if (endTime && r.timestamp > endTime) continue;
+        const row = {
+          url: r.url,
+          timestamp: r.timestamp,
+          iso: new Date(r.timestamp).toISOString(),
+          baselineVersion: r.baselineVersion ?? null,
+          changeType: r.changeType,
+          significance: r.significance,
+          similarity: r.details?.similarity ?? null,
+          structuralSimilarity: r.details?.structuralSimilarity ?? null,
+          addedElements: r.metrics?.addedElements ?? 0,
+          removedElements: r.metrics?.removedElements ?? 0,
+          modifiedElements: r.metrics?.modifiedElements ?? 0,
+          processingTime: r.processingTime ?? null
+        };
+        if (includeContent) row.details = r.details;
+        records.push(row);
+      }
+    }
+    records.sort((a, b) => a.timestamp - b.timestamp);
+
+    const result = {
+      format,
+      exportedAt: Date.now(),
+      scope: url || 'global',
+      urlCount: urls.length,
+      recordCount: records.length,
+      ...(startTime || endTime ? { range: { startTime: startTime ?? null, endTime: endTime ?? null } } : {})
+    };
+    if (includeSnapshots) {
+      result.snapshots = urls.flatMap(u => (this.snapshots.get(u) || []).map(snap => ({
+        url: u,
+        version: snap.version ?? null,
+        createdAt: snap.timestamp ?? null,
+        contentHash: snap.analysis?.hashes?.content ?? null,
+        options: snap.options ?? null
+      })));
+    }
+    if (format === 'csv') {
+      const columns = ['url', 'timestamp', 'iso', 'baselineVersion', 'changeType', 'significance', 'similarity',
+        'structuralSimilarity', 'addedElements', 'removedElements', 'modifiedElements', 'processingTime',
+        ...(includeContent ? ['details'] : [])];
+      result.columns = columns;
+      result.csv = [columns.join(','), ...records.map(r => columns.map(c => csvCell(r[c])).join(','))].join('\n');
+    } else {
+      result.records = records;
+    }
+    return result;
+  }
+
   async initializeSemanticAnalyzer() {
     // Placeholder for semantic analysis initialization
   }

package/src/core/crawlers/BFSCrawler.js

@@ -7,7 +7,7 @@ import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
-import { robotsPreflight } from '../../utils/robotsGate.js';
+import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
 
@@ -185,6 +185,18 @@ export class BFSCrawler {
     });
     if (!gate.allowed) {
       logger.debug(`Robots.txt blocks: ${normalizedUrl}`);
+      // A refused seed used to end the crawl with 0 pages and 0 errors, so the
+      // caller could not tell a robots refusal from an empty site. Children
+      // stay silent: skipping disallowed links is the normal case.
+      if (isSeed) {
+        this.errors.push({
+          url: normalizedUrl,
+          depth,
+          error: new RobotsDisallowedError(normalizedUrl).message,
+          code: 'ROBOTS_DISALLOWED',
+          timestamp: new Date().toISOString()
+        });
+      }
       return;
     }
 

package/src/schemas/toolOutputSchemas.js

@@ -188,6 +188,7 @@ const redditSearchShape = {
   link_id: z.string().optional().describe('Present in thread mode'),
   count: z.number().optional(),
   results: z.array(z.union([redditPostShape, redditCommentShape])).optional().describe('posts/comments modes'),
+  discovered: z.number().optional().describe('web_discovery: how many post ids the site-restricted web search surfaced before archive hydration'),
   post: redditPostShape.nullable().optional().describe('thread mode: the post itself'),
   comments: z.array(z.unknown()).optional().describe('thread mode: nested comment tree ({...comment, replies:[...]}); collapsed branches appear as {more_count, more_ids}'),
   comment_count: z.number().optional(),

package/src/skills/agent-skills/crawlforge-structured-extraction/references/templates.md

@@ -15,7 +15,7 @@ to enumerate at runtime. Cost: 1 credit per call.
 | `reddit-thread` | Reddit thread | title, subreddit, author, score, top comments |
 | `hacker-news-front-page` | HN front page | ranked stories: title, URL, points, author, comment count |
 | `producthunt-launch` | Product Hunt launch | name, tagline, upvotes, maker, description |
-| `stackoverflow-question` | Stack Overflow question | title, tags, votes, question body, accepted answer |
+| `stackoverflow-question` | Stack Overflow question (read from the keyless Stack Exchange API, not the page) | title, body, votes, views, tags, author, asked, answered, accepted_answer_id, answer_count, top 5 answers (accepted first: votes, author, posted, body) |
 | `npm-package` | npm package page | name, version, description, weekly downloads, license, repo link |
 
 ## Usage

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -675,19 +675,36 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // Prefer the post-action live page HTML captured during action execution.
       // This ensures the final content reflects clicks/typing/navigation rather
       // than re-fetching the original (pre-action) URL.
-      if (chainResult?.finalHtml) {
-        return await this.extractContentTool.execute({
+      const extractResult = chainResult?.finalHtml
+        ? await this.extractContentTool.execute({
           url: chainResult.finalUrl || params.url,
           html: chainResult.finalHtml,
           options
-        });
+        })
+        : await this.extractContentTool.execute({ url: params.url, options });
+
+      // extractContent never honoured customSelectors, so extractionOptions.
+      // selectors only ever surfaced inside captureIntermediateStates. Run them
+      // over the post-action DOM here, the one place every caller reads.
+      if (chainResult?.finalHtml && (params.extractionOptions?.selectors || params.formats?.includes('text'))) {
+        const $ = load(chainResult.finalHtml);
+        extractResult.content = extractResult.content || {};
+        if (params.extractionOptions?.selectors) {
+          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+        }
+        // Readability picks a "main" block and drops form controls; on a small
+        // post-action page (a demo form, a results panel) that leaves the
+        // footer alone, or a page minus the very buttons the actions produced.
+        // A short readable result means it found nothing substantial, so hand
+        // back the whole (equally short) body text instead.
+        const readable = extractResult.content.text || '';
+        const bodyText = $('body').text().replace(/\s+/g, ' ').trim();
+        if (readable.length < 300 && bodyText.length > readable.length) {
+          extractResult.content.text = bodyText;
+          extractResult.content.textSource = 'body';
+        }
       }
 
-      const extractResult = await this.extractContentTool.execute({
-        url: params.url,
-        options
-      });
-
       return extractResult;
     } catch (error) {
       this.log('warn', `Final content extraction failed: ${error.message}`);
@@ -706,6 +723,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
     if (formats.includes('json')) {
       content.json = {
         finalContent: finalContent.content || {},
+        ...(finalContent.content?.extracted ? { extracted: finalContent.content.extracted } : {}),
         metadata: finalContent.metadata || {},
         actionSummary: {
           totalActions: additionalData.actionResults.length,

package/src/tools/advanced/batchScrape/index.js

@@ -175,7 +175,7 @@ export class BatchScrapeTool extends EventEmitter {
       };
 
       if (this.enableResultCaching) {
-        this._cacheBatchResult(batchId, processedResults);
+        this._cacheBatchResult(batchId, processedResults, 'sync');
       }
 
       this.stats.completedBatches++;
@@ -237,7 +237,7 @@ export class BatchScrapeTool extends EventEmitter {
       if (Date.now() - cached.timestamp < cached.ttl) {
         const offset = (page - 1) * pageSize;
         return {
-          batchId, success: true,
+          batchId, success: true, status: 'completed', ...(cached.mode ? { mode: cached.mode } : {}),
           results: paginateResults(cached.results, offset, pageSize),
           pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
           cached: true, timestamp: cached.timestamp
@@ -267,8 +267,11 @@ export class BatchScrapeTool extends EventEmitter {
       if (job.status === 'completed' && job.result) {
         const results = job.result.results || [];
         const offset = (page - 1) * pageSize;
+        // Same status/mode fields the pending and running branches carry, so
+        // a poller can key on `status` for the whole job lifecycle.
         return {
-          batchId, success: true, jobId: job.id,
+          batchId, success: true, jobId: job.id, status: 'completed', mode: 'async',
+          completedAt: job.completedAt,
           results: paginateResults(results, offset, pageSize),
           pagination: { page, pageSize, totalResults: results.length, totalPages: Math.ceil(results.length / pageSize) },
           cached: false, timestamp: job.completedAt
@@ -366,8 +369,8 @@ export class BatchScrapeTool extends EventEmitter {
    * Cache a batch's results, capped to maxCachedBatches. Map preserves
    * insertion order, so the oldest entry is evicted first (LRU by write time).
    */
-  _cacheBatchResult(batchId, results) {
-    this.batchResults.set(batchId, { results, timestamp: Date.now(), ttl: this.resultCacheTtl });
+  _cacheBatchResult(batchId, results, mode) {
+    this.batchResults.set(batchId, { results, mode, timestamp: Date.now(), ttl: this.resultCacheTtl });
     while (this.batchResults.size > this.maxCachedBatches) {
       const oldestKey = this.batchResults.keys().next().value;
       this.batchResults.delete(oldestKey);
@@ -430,7 +433,7 @@ export class BatchScrapeTool extends EventEmitter {
         };
 
         if (this.enableResultCaching) {
-          this._cacheBatchResult(batchId, processedResults);
+          this._cacheBatchResult(batchId, processedResults, 'async');
         }
 
         this.stats.totalUrls += results.length;

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -186,6 +186,9 @@ export class GenerateLLMsTxtTool {
     if (Array.isArray(outputOptions.customGuidelines) && outputOptions.customGuidelines.length > 0) {
       details.push(...outputOptions.customGuidelines);
     }
+    if (Array.isArray(outputOptions.customRestrictions) && outputOptions.customRestrictions.length > 0) {
+      details.push(`Do not access: ${outputOptions.customRestrictions.join(', ')}.`);
+    }
     if (details.length > 0) {
       lines.push(details.join(' '));
       lines.push('');
@@ -591,6 +594,13 @@ export class GenerateLLMsTxtTool {
     lines.push('- Form submissions and interactive content');
     lines.push('- Administrative and configuration areas');
     lines.push('');
+    if (Array.isArray(outputOptions.customRestrictions) && outputOptions.customRestrictions.length > 0) {
+      lines.push('### Additional Restrictions');
+      for (const restriction of outputOptions.customRestrictions) {
+        lines.push(`- ${restriction}`);
+      }
+      lines.push('');
+    }
 
     // Section 4: APIs and Data Sources
     lines.push('## 4. API and Data Sources');
@@ -669,6 +679,13 @@ export class GenerateLLMsTxtTool {
     lines.push('4. **Cache responsibly:** Cache content appropriately to reduce server load');
     lines.push('5. **Handle errors gracefully:** Implement proper error handling and retries');
     lines.push('');
+    if (Array.isArray(outputOptions.customGuidelines) && outputOptions.customGuidelines.length > 0) {
+      lines.push('### Site-Specific Guidelines');
+      for (const guideline of outputOptions.customGuidelines) {
+        lines.push(`- ${guideline}`);
+      }
+      lines.push('');
+    }
 
     if (analysis.structure && analysis.structure.robotsTxt) {
       lines.push('### Robots.txt Status');

package/src/tools/tracking/trackChanges/index.js

@@ -289,6 +289,7 @@ export class TrackChangesTool extends EventEmitter {
       ...(comparisonResult.warnings || fetchWarnings.length
         ? { warnings: [...(comparisonResult.warnings || []), ...fetchWarnings] }
         : {}),
+      ...(comparisonResult.alerts ? { alerts: comparisonResult.alerts } : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }
@@ -390,16 +391,36 @@ export class TrackChangesTool extends EventEmitter {
     const { url, scheduledMonitorOptions, monitoringOptions, trackingOptions, notificationOptions } = params;
     if (!url) throw new Error('create_scheduled_monitor requires a url');
     const opts = scheduledMonitorOptions || {};
+
+    // A preset fills in whatever the caller left unspecified; explicit values win.
+    let preset = null;
+    if (opts.templateId) {
+      preset = this.changeTracker.monitoringTemplates.get(opts.templateId);
+      if (!preset) {
+        throw new Error(
+          `Unknown monitoring template "${opts.templateId}". ` +
+          `Available: ${Array.from(this.changeTracker.monitoringTemplates.keys()).join(', ')} ` +
+          '(operation:"get_monitoring_templates" describes each).'
+        );
+      }
+    }
+    // Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
+    // schema fills monitoringOptions.interval/notificationThreshold with
+    // defaults, so they cannot sit above a preset without always winning.
     const monitor = await this.scheduler.createMonitor({
       url,
-      interval: opts.interval ?? monitoringOptions?.interval,
+      interval: opts.interval ?? preset?.frequency ?? monitoringOptions?.interval,
       schedule: opts.schedule,
-      goal: opts.goal,
-      notificationThreshold: opts.notificationThreshold || monitoringOptions?.notificationThreshold || 'moderate',
-      trackingOptions,
+      goal: opts.goal ?? preset?.goal,
+      notificationThreshold: opts.notificationThreshold || preset?.notificationThreshold || monitoringOptions?.notificationThreshold || 'moderate',
+      trackingOptions: preset ? { ...preset.options, ...(trackingOptions || {}) } : trackingOptions,
       notificationOptions
     });
-    return { success: true, operation: 'create_scheduled_monitor', url, monitor, firingGuarantee: monitor.firingGuarantee, timestamp: Date.now() };
+    return {
+      success: true, operation: 'create_scheduled_monitor', url, monitor,
+      ...(preset ? { templateId: preset.id } : {}),
+      firingGuarantee: monitor.firingGuarantee, timestamp: Date.now()
+    };
   }
 
   async stopScheduledMonitor(params) {
@@ -424,8 +445,24 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async getMonitoringDashboard(params) {
-    const { dashboardOptions } = params;
+    // The schema defaults every flag to true, but only inside a dashboardOptions
+    // object the caller supplied; an omitted object must mean the same thing.
+    const dashboardOptions = { includeRecentAlerts: true, includeTrends: true, includeMonitorStatus: true, ...(params.dashboardOptions || {}) };
     const dashboard = this.changeTracker.getMonitoringDashboard();
+    // The tracker only knows tracked pages; the monitors that poll them live here.
+    const polling = Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
+      id: `poll:${url}`, url, kind: 'polling', status: 'active',
+      interval: m.options?.interval ?? null, stats: m.stats
+    }));
+    if (!this.monitorStore._loaded) await this.monitorStore.load();
+    const scheduled = this.scheduler.list().map(m => ({
+      id: m.id, url: m.url, kind: 'scheduled',
+      status: m.enabled === false ? 'disabled' : m.scheduled ? 'scheduled' : 'idle',
+      interval: m.interval, goal: m.goal, nextDueAt: m.nextDueAt,
+      lastCheckAt: m.lastCheckAt, lastChangeAt: m.lastChangeAt, stats: m.stats
+    }));
+    dashboard.monitors = [...polling, ...scheduled];
+    dashboard.summary.activeMonitors = dashboard.monitors.length;
     if (!dashboardOptions?.includeRecentAlerts) delete dashboard.recentAlerts;
     if (!dashboardOptions?.includeTrends) delete dashboard.trends;
     if (!dashboardOptions?.includeMonitorStatus) {
@@ -441,16 +478,22 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async createAlertRule(params) {
-    const { alertRuleOptions } = params;
+    const { url, alertRuleOptions } = params;
     const ruleId = alertRuleOptions?.ruleId || `custom_rule_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-    const rule = {
-      condition: this._parseCondition(alertRuleOptions?.condition || 'significance === "major"'),
+    const rule = this.changeTracker.addAlertRule(ruleId, {
+      condition: alertRuleOptions?.condition,
+      url: url || null,
       actions: alertRuleOptions?.actions || ['webhook'],
-      throttle: alertRuleOptions?.throttle || 600000,
+      throttle: alertRuleOptions?.throttle ?? 600000,
       priority: alertRuleOptions?.priority || 'medium'
+    });
+    // Rules fire from compare: every later compare that matches returns an
+    // `alerts` array, and get_dashboard lists what fired.
+    return {
+      success: true, operation: 'create_alert_rule', ruleId,
+      rule: { id: rule.id, url: rule.url, condition: rule.conditionText, actions: rule.actions, throttle: rule.throttle, priority: rule.priority },
+      timestamp: Date.now()
     };
-    this.changeTracker.alertRules.set(ruleId, rule);
-    return { success: true, operation: 'create_alert_rule', ruleId, rule, timestamp: Date.now() };
   }
 
   async generateTrendReport(params) {
@@ -461,7 +504,11 @@ export class TrackChangesTool extends EventEmitter {
   async getMonitoringTemplates() {
     const templates = {};
     for (const [id, template] of this.changeTracker.monitoringTemplates.entries()) {
-      templates[id] = { name: template.name, frequency: template.frequency, options: template.options, alertRules: template.alertRules };
+      templates[id] = {
+        name: template.name, description: template.description, frequency: template.frequency,
+        notificationThreshold: template.notificationThreshold, goal: template.goal,
+        options: template.options, alertRules: template.alertRules
+      };
     }
     return { success: true, operation: 'get_monitoring_templates', templates, count: Object.keys(templates).length, timestamp: Date.now() };
   }
@@ -533,20 +580,6 @@ export class TrackChangesTool extends EventEmitter {
     stats.newestMonitor = Math.max(...all.map(v => v.started));
     return stats;
   }
-
-  _parseCondition(conditionString) {
-    return (changeResult) => {
-      try {
-        if (conditionString.includes('significance')) {
-          const match = conditionString.match(/significance\s*===\s*["'](\w+)["']/);
-          if (match) return changeResult.significance === match[1];
-        }
-        return false;
-      } catch {
-        return false;
-      }
-    };
-  }
 }
 
 export default TrackChangesTool;

package/src/utils/provenance.js

@@ -47,7 +47,7 @@ const CURRENCY_SYMBOLS = /\p{Sc}/gu;
 const NUMERIC_STRING = /^[+-]?\d+(?:[.,]\d{3})*(?:[.,]\d+)?$/;
 
 /** Chained markup between digits (`<span>1</span><span>299</span>`) is welded. */
-const MARKUP_BETWEEN_DIGITS = /(\d)(?:\s*<[^>]{0,120}>\s*)+([\d.,])/g;
+const MARKUP_BETWEEN_DIGITS = /(\d)(?:(?:\s*<[^>]{0,120}>)+\s*)([\d.,])/g;
 const MAX_WELD_PASSES = 3;
 
 /**