crawlforge-mcp-server 5.4.2 → 5.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.4.2",
+  "version": "5.5.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.4.0",
+    "crawlforge-extractors": "^1.5.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.4.2",
+  version: "5.5.0",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -492,7 +492,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search has no backend — scope it or search posts. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -504,7 +504,7 @@ registerToolIfEnabled("reddit_search", {
     before: z.string().optional().describe("Only content posted before this date — same formats as after"),
     limit: z.number().min(1).max(100).optional().describe("Max results (default 25; thread mode: max comments returned)"),
     sort: z.enum(["asc", "desc"]).optional().describe("Sort by post date (default desc = newest first)"),
-    source: z.enum(["auto", "arctic_shift", "pullpush", "reddit_api"]).optional().describe("Backend: auto routes + falls back (default). reddit_api uses the official Reddit Data API — only when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set; serves posts/thread, not comment search")
+    source: z.enum(["auto", "arctic_shift", "pullpush", "reddit_api", "web_discovery"]).optional().describe("Backend: auto routes + falls back (default). web_discovery serves only unscoped keyword searches (web search finds the posts, the archive supplies the rows). reddit_api uses the official Reddit Data API — only when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set; serves posts/thread, not comment search")
   },
   outputSchema: OUTPUT_SCHEMAS.reddit_search
 }, withAuth("reddit_search", async ({ query, subreddit, author, mode, link_id, after, before, limit, sort, source }) => {
@@ -977,7 +977,7 @@ if (toolFilter.isEnabled("deep_research")) {
         maxVariations: z.number().min(1).max(20).optional().default(8)
       }).optional().describe("Query expansion settings for broader search coverage"),
       llmConfig: z.object({
-        provider: z.enum(['auto', 'openai', 'anthropic']).optional().default('auto'),
+        provider: z.enum(['auto', 'openai', 'anthropic', 'ollama']).optional().default('auto'),
         openai: z.object({
           apiKey: z.string().optional(),
           model: z.string().optional().default('gpt-3.5-turbo'),
@@ -987,9 +987,13 @@ if (toolFilter.isEnabled("deep_research")) {
           apiKey: z.string().optional(),
           model: z.string().optional().default('claude-3-haiku-20240307')
         }).optional(),
+        ollama: z.object({
+          model: z.string().optional(),
+          embeddingModel: z.string().optional()
+        }).optional(),
         enableSemanticAnalysis: z.boolean().optional().default(true),
         enableIntelligentSynthesis: z.boolean().optional().default(true)
-      }).optional().describe("LLM provider configuration for AI-powered analysis"),
+      }).optional().describe("LLM provider configuration for AI-powered analysis. provider 'auto' (default) uses a configured cloud key if there is one, else the local Ollama (http://localhost:11434, no key); 'ollama' forces the local model; 'openai'/'anthropic' need the matching API key"),
       concurrency: z.number().min(1).max(20).optional().default(5).describe("Number of concurrent research requests"),
       cacheResults: z.boolean().optional().default(true).describe("Cache research results for reuse"),
       webhook: z.object({
@@ -1534,10 +1538,30 @@ registerToolIfEnabled("localization", {
         if (!params.countryCode) throw new Error('countryCode is required for configure_country operation');
         result = await localizationManager.configureCountry(params.countryCode, params);
         break;
-      case 'localize_search':
+      case 'localize_search': {
         if (!params.searchParams) throw new Error('searchParams is required for localize_search operation');
-        result = await localizationManager.localizeSearchQuery(params.searchParams, params.countryCode);
+        const localizedParams = await localizationManager.localizeSearchQuery(params.searchParams, params.countryCode);
+        if (!params.searchParams.query) {
+          result = {
+            ...localizedParams,
+            note: 'No searchParams.query was given, so no search ran — these are the parameters a localized search would use.'
+          };
+          break;
+        }
+        // Run the search the caller asked for under the localized country and
+        // language, so the operation returns results rather than a config.
+        // Priced as a search_web call (AuthManager.getToolCost).
+        const countryCode = params.countryCode || localizationManager.getCurrentSettings().countryCode;
+        const search = await searchWebTool.execute({
+          query: params.searchParams.query,
+          limit: params.searchParams.limit,
+          offset: params.searchParams.offset,
+          lang: localizedParams.lang,
+          localization: { countryCode, language: params.language || localizedParams.lang }
+        });
+        result = { localizedParams, search };
         break;
+      }
       case 'localize_browser':
         if (!params.browserOptions) throw new Error('browserOptions is required for localize_browser operation');
         result = await localizationManager.localizeBrowserContext(params.browserOptions, params.countryCode);
@@ -1578,7 +1602,7 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package, linkedin-profile). List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
+  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package; reddit-thread reads the post from the Arctic Shift archive and reddit_search reads the comment tree). linkedin-profile and tweet are retired — those sites' robots.txt disallow every keyless path — and naming one returns the reason. List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     template: z.string().describe("Template ID (e.g. github-repo), \"auto\" to detect one from the url, or \"list\" to enumerate available templates"),

package/src/core/AuthManager.js

@@ -612,6 +612,12 @@ class AuthManager {
       if (bookkeepingOps.has(params?.operation)) return 1;
     }
 
+    // localize_search with a query runs a real web search through the same
+    // adapter search_web uses, so it is priced as one rather than undercutting it.
+    if (tool === 'localization' && params?.operation === 'localize_search' && params?.searchParams?.query) {
+      return costs.search_web;
+    }
+
     return costs[tool] ?? 1;
   }
 

package/src/core/ChangeTracker.js

@@ -103,6 +103,79 @@ function extractMonetaryValues(text) {
   return values;
 }
 
+/**
+ * Built-in monitoring presets, served by track_changes
+ * {operation:"get_monitoring_templates"} and applied by
+ * create_scheduled_monitor via scheduledMonitorOptions.templateId. `options`
+ * are trackingOptions (the baseline analysis options); `frequency` is the
+ * polling interval in ms; `goal` is the plain-English alert goal the
+ * scheduler's LLM judge uses when one is configured.
+ */
+export const MONITORING_TEMPLATES = {
+  'price-watch': {
+    name: 'Price watch',
+    description: 'Product, plan or rate pages — hourly, and any numeric change counts.',
+    frequency: 3600000,
+    notificationThreshold: 'minor',
+    goal: 'A price, discount, fee or plan cost changed',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, significanceThresholds: { minor: 0.02, moderate: 0.1, major: 0.3 } },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'availability': {
+    name: 'Stock / availability',
+    description: 'In-stock, sold-out and waitlist states — every 15 minutes, any change counts.',
+    frequency: 900000,
+    notificationThreshold: 'minor',
+    goal: 'An item went in or out of stock, or a waitlist or pre-order opened',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, significanceThresholds: { minor: 0.01, moderate: 0.1, major: 0.3 } },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'news-feed': {
+    name: 'News / announcements',
+    description: 'Blogs, changelogs and press pages — every 30 minutes, section-level diffs so a new post registers as one change.',
+    frequency: 1800000,
+    notificationThreshold: 'moderate',
+    goal: 'A new article, release note or announcement was published',
+    options: { granularity: 'section', trackText: true, trackStructure: true, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  },
+  'documentation': {
+    name: 'Documentation',
+    description: 'API references and docs pages — daily, structure tracked so a new parameter or removed endpoint shows up.',
+    frequency: 86400000,
+    notificationThreshold: 'moderate',
+    goal: 'A documented API, parameter, default or behaviour changed',
+    options: { granularity: 'section', trackText: true, trackStructure: true, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  },
+  'regulatory': {
+    name: 'Regulation / policy',
+    description: 'Terms, policies, tariffs and rule pages — daily, and every wording change counts.',
+    frequency: 86400000,
+    notificationThreshold: 'minor',
+    goal: 'A rule, deadline, fee, threshold or policy wording changed',
+    options: { granularity: 'text', trackText: true, trackStructure: false, trackLinks: false, ignoreWhitespace: true, ignoreCase: false },
+    alertRules: [{ condition: 'significance >= "minor"', actions: ['webhook'], priority: 'high' }]
+  },
+  'job-board': {
+    name: 'Job listings',
+    description: 'Careers pages — every 6 hours, links tracked so an added or removed posting registers.',
+    frequency: 21600000,
+    notificationThreshold: 'moderate',
+    goal: 'A job opening was added or removed',
+    options: { granularity: 'section', trackText: true, trackStructure: false, trackLinks: true, ignoreWhitespace: true },
+    alertRules: [{ condition: 'significance >= "moderate"', actions: ['webhook'], priority: 'medium' }]
+  }
+};
+
+const SIGNIFICANCE_RANK = { none: 0, minor: 1, moderate: 2, major: 3, critical: 4 };
+
+function csvCell(value) {
+  if (value === null || value === undefined) return '';
+  const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
+  return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+
 export class ChangeTracker extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -147,6 +220,16 @@ export class ChangeTracker extends EventEmitter {
       processingTime: 0
     };
     
+    // Alert rules (create_alert_rule) are evaluated on every compare; the
+    // alerts they fire are kept here for get_dashboard. Presets back
+    // get_monitoring_templates and scheduledMonitorOptions.templateId.
+    this.alertRules = new Map();
+    this.recentAlerts = [];
+    this.maxRecentAlerts = 100;
+    this.monitoringTemplates = new Map(
+      Object.entries(MONITORING_TEMPLATES).map(([id, t]) => [id, { id, ...t }])
+    );
+
     // Semantic analysis tools (if enabled)
     this.semanticAnalyzer = null;
     
@@ -311,6 +394,7 @@ export class ChangeTracker extends EventEmitter {
       }
       
       this.emit('changeDetected', changeRecord);
+      const alerts = this.evaluateAlertRules(changeRecord);
       
       const ignoredOptions = this.findIgnoredCompareOptions(options, baseline.options);
 
@@ -322,6 +406,7 @@ export class ChangeTracker extends EventEmitter {
         details: changeAnalysis,
         metrics: changeRecord.metrics,
         recommendations: this.generateChangeRecommendations(changeRecord),
+        ...(alerts.length ? { alerts } : {}),
         ...(ignoredOptions.length ? {
           warnings: [
             `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
@@ -1654,6 +1739,252 @@ export class ChangeTracker extends EventEmitter {
     };
   }
 
+  // ── Alert rules, dashboard, export, trends, presets ─────────────────────────
+  // Backing for the track_changes operations create_alert_rule, get_dashboard,
+  // export_history, generate_trend_report and get_monitoring_templates. The
+  // tool has called these since v3.1.0; until 5.4.3 none of them existed.
+
+  /**
+   * Turn an alert-rule condition string into a predicate over a change record.
+   * Supported: `significance === "major"`, `significance >= "moderate"` and
+   * the other comparison operators against the significance ladder
+   * none < minor < moderate < major < critical. Anything else never matches.
+   */
+  static parseAlertCondition(conditionString = '') {
+    const match = String(conditionString).match(/significance\s*(===|==|!==|!=|>=|<=|>|<)\s*["'](\w+)["']/);
+    if (!match) return () => false;
+    const [, op, level] = match;
+    const target = SIGNIFICANCE_RANK[level];
+    if (target === undefined) return () => false;
+    return (record) => {
+      const rank = SIGNIFICANCE_RANK[record?.significance];
+      if (rank === undefined) return false;
+      switch (op) {
+        case '===': case '==': return rank === target;
+        case '!==': case '!=': return rank !== target;
+        case '>=': return rank >= target;
+        case '<=': return rank <= target;
+        case '>': return rank > target;
+        default: return rank < target;
+      }
+    };
+  }
+
+  /**
+   * Register an alert rule. `condition` is a string (see parseAlertCondition);
+   * `url` scopes the rule to one tracked page, or every page when omitted.
+   */
+  addAlertRule(ruleId, { condition, url = null, actions = ['webhook'], throttle = 600000, priority = 'medium' } = {}) {
+    const rule = {
+      id: ruleId,
+      url,
+      conditionText: condition || 'significance === "major"',
+      condition: ChangeTracker.parseAlertCondition(condition || 'significance === "major"'),
+      actions,
+      throttle,
+      priority,
+      createdAt: Date.now(),
+      fireCount: 0,
+      suppressedCount: 0,
+      lastFiredAt: null
+    };
+    this.alertRules.set(ruleId, rule);
+    return rule;
+  }
+
+  /** Evaluate every rule against a change record; returns the alerts fired. */
+  evaluateAlertRules(changeRecord) {
+    const fired = [];
+    const now = changeRecord.timestamp || Date.now();
+    for (const rule of this.alertRules.values()) {
+      if (rule.url && rule.url !== changeRecord.url) continue;
+      let matches = false;
+      try { matches = rule.condition(changeRecord); } catch { matches = false; }
+      if (!matches) continue;
+      if (rule.lastFiredAt && rule.throttle && now - rule.lastFiredAt < rule.throttle) {
+        rule.suppressedCount++;
+        continue;
+      }
+      rule.lastFiredAt = now;
+      rule.fireCount++;
+      const alert = {
+        id: `alert_${now}_${Math.random().toString(36).slice(2, 8)}`,
+        ruleId: rule.id,
+        url: changeRecord.url,
+        timestamp: now,
+        significance: changeRecord.significance,
+        changeType: changeRecord.changeType,
+        similarity: changeRecord.details?.similarity ?? null,
+        priority: rule.priority,
+        actions: rule.actions
+      };
+      this.recentAlerts.unshift(alert);
+      if (this.recentAlerts.length > this.maxRecentAlerts) this.recentAlerts.length = this.maxRecentAlerts;
+      fired.push(alert);
+      this.emit('alert', alert);
+    }
+    return fired;
+  }
+
+  /** Per-URL change trend over the in-memory history. */
+  summarizeTrend(url) {
+    const history = this.changeHistory.get(url) || [];
+    const changes = history.filter(r => r.significance !== 'none');
+    const significanceDistribution = { none: 0, minor: 0, moderate: 0, major: 0, critical: 0 };
+    for (const r of history) {
+      significanceDistribution[r.significance] = (significanceDistribution[r.significance] || 0) + 1;
+    }
+    const similarities = history.map(r => r.details?.similarity).filter(n => typeof n === 'number');
+    const avg = (arr) => (arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : null);
+    const half = Math.floor(similarities.length / 2);
+    const early = avg(similarities.slice(0, half));
+    const late = avg(similarities.slice(half));
+    let direction = 'insufficient_data';
+    if (similarities.length >= 4 && early !== null && late !== null) {
+      direction = late < early - 0.05 ? 'diverging' : late > early + 0.05 ? 'stabilizing' : 'steady';
+    }
+    const span = history.length >= 2 ? history[history.length - 1].timestamp - history[0].timestamp : 0;
+    const changeTypes = {};
+    for (const r of changes) changeTypes[r.changeType] = (changeTypes[r.changeType] || 0) + 1;
+    return {
+      url,
+      compares: history.length,
+      changes: changes.length,
+      changeRate: history.length ? changes.length / history.length : 0,
+      changesPerDay: span > 0 ? changes.length / (span / 86400000) : null,
+      averageSimilarity: avg(similarities),
+      direction,
+      significanceDistribution,
+      changeTypes,
+      firstCompareAt: history[0]?.timestamp ?? null,
+      lastCompareAt: history[history.length - 1]?.timestamp ?? null,
+      lastChangeAt: changes[changes.length - 1]?.timestamp ?? null
+    };
+  }
+
+  async generateTrendAnalysisReport(url) {
+    if (url && !this.changeHistory.has(url) && !this.snapshots.has(url)) {
+      throw new Error(`No tracking history for ${url} — run create_baseline and compare first`);
+    }
+    const urls = url ? [url] : Array.from(this.changeHistory.keys());
+    const trends = urls.map(u => this.summarizeTrend(u));
+    const recommendations = [];
+    for (const t of trends) {
+      if (t.compares < 2) {
+        recommendations.push(`${t.url}: ${t.compares} compare${t.compares === 1 ? '' : 's'} recorded — run compare a few more times before reading a trend.`);
+        continue;
+      }
+      if (t.direction === 'diverging') {
+        recommendations.push(`${t.url}: each compare drifts further from the baseline — recreate the baseline if the current page is the new normal.`);
+      }
+      if (t.changes === 0) {
+        recommendations.push(`${t.url}: no changes across ${t.compares} compares — the polling interval can be lengthened.`);
+      }
+      if (t.changesPerDay !== null && t.changesPerDay > 24) {
+        recommendations.push(`${t.url}: changes more than hourly — scope the baseline with customSelectors to skip volatile regions.`);
+      }
+    }
+    return { generatedAt: Date.now(), scope: url || 'global', urlCount: urls.length, trends, recommendations };
+  }
+
+  getMonitoringDashboard() {
+    const urls = Array.from(new Set([...this.snapshots.keys(), ...this.changeHistory.keys()]));
+    const tracked = urls.map(url => {
+      const history = this.changeHistory.get(url) || [];
+      const last = history[history.length - 1] || null;
+      const snaps = this.snapshots.get(url) || [];
+      const baseline = snaps[snaps.length - 1] || null;
+      return {
+        url,
+        baselineVersion: baseline?.version ?? null,
+        baselineCreatedAt: baseline?.timestamp ?? null,
+        compares: history.length,
+        changes: history.filter(r => r.significance !== 'none').length,
+        lastCompareAt: last?.timestamp ?? null,
+        lastSignificance: last?.significance ?? null,
+        lastChangeType: last?.changeType ?? null
+      };
+    });
+    return {
+      generatedAt: Date.now(),
+      summary: {
+        trackedUrls: urls.length,
+        totalCompares: tracked.reduce((n, t) => n + t.compares, 0),
+        totalChanges: tracked.reduce((n, t) => n + t.changes, 0),
+        alertRules: this.alertRules.size,
+        alertsFired: this.recentAlerts.length,
+        activeMonitors: 0,
+        ...this.getStats()
+      },
+      tracked,
+      alertRules: Array.from(this.alertRules.values()).map(r => ({
+        id: r.id, url: r.url, condition: r.conditionText, actions: r.actions, throttle: r.throttle,
+        priority: r.priority, fireCount: r.fireCount, suppressedCount: r.suppressedCount, lastFiredAt: r.lastFiredAt
+      })),
+      recentAlerts: this.recentAlerts.slice(0, 20),
+      trends: urls.map(u => this.summarizeTrend(u)),
+      // Polling and scheduled monitors live in the tool, which fills this in.
+      monitors: []
+    };
+  }
+
+  async exportHistoricalData(options = {}) {
+    const { url, format = 'json', startTime, endTime, includeContent = false, includeSnapshots = false } = options;
+    const urls = url ? [url] : Array.from(this.changeHistory.keys());
+    const records = [];
+    for (const u of urls) {
+      for (const r of this.changeHistory.get(u) || []) {
+        if (startTime && r.timestamp < startTime) continue;
+        if (endTime && r.timestamp > endTime) continue;
+        const row = {
+          url: r.url,
+          timestamp: r.timestamp,
+          iso: new Date(r.timestamp).toISOString(),
+          baselineVersion: r.baselineVersion ?? null,
+          changeType: r.changeType,
+          significance: r.significance,
+          similarity: r.details?.similarity ?? null,
+          structuralSimilarity: r.details?.structuralSimilarity ?? null,
+          addedElements: r.metrics?.addedElements ?? 0,
+          removedElements: r.metrics?.removedElements ?? 0,
+          modifiedElements: r.metrics?.modifiedElements ?? 0,
+          processingTime: r.processingTime ?? null
+        };
+        if (includeContent) row.details = r.details;
+        records.push(row);
+      }
+    }
+    records.sort((a, b) => a.timestamp - b.timestamp);
+
+    const result = {
+      format,
+      exportedAt: Date.now(),
+      scope: url || 'global',
+      urlCount: urls.length,
+      recordCount: records.length,
+      ...(startTime || endTime ? { range: { startTime: startTime ?? null, endTime: endTime ?? null } } : {})
+    };
+    if (includeSnapshots) {
+      result.snapshots = urls.flatMap(u => (this.snapshots.get(u) || []).map(snap => ({
+        url: u,
+        version: snap.version ?? null,
+        createdAt: snap.timestamp ?? null,
+        contentHash: snap.analysis?.hashes?.content ?? null,
+        options: snap.options ?? null
+      })));
+    }
+    if (format === 'csv') {
+      const columns = ['url', 'timestamp', 'iso', 'baselineVersion', 'changeType', 'significance', 'similarity',
+        'structuralSimilarity', 'addedElements', 'removedElements', 'modifiedElements', 'processingTime',
+        ...(includeContent ? ['details'] : [])];
+      result.columns = columns;
+      result.csv = [columns.join(','), ...records.map(r => columns.map(c => csvCell(r[c])).join(','))].join('\n');
+    } else {
+      result.records = records;
+    }
+    return result;
+  }
+
   async initializeSemanticAnalyzer() {
     // Placeholder for semantic analysis initialization
   }

package/src/core/crawlers/BFSCrawler.js

@@ -7,7 +7,7 @@ import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
-import { robotsPreflight } from '../../utils/robotsGate.js';
+import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
 
@@ -185,6 +185,18 @@ export class BFSCrawler {
     });
     if (!gate.allowed) {
       logger.debug(`Robots.txt blocks: ${normalizedUrl}`);
+      // A refused seed used to end the crawl with 0 pages and 0 errors, so the
+      // caller could not tell a robots refusal from an empty site. Children
+      // stay silent: skipping disallowed links is the normal case.
+      if (isSeed) {
+        this.errors.push({
+          url: normalizedUrl,
+          depth,
+          error: new RobotsDisallowedError(normalizedUrl).message,
+          code: 'ROBOTS_DISALLOWED',
+          timestamp: new Date().toISOString()
+        });
+      }
       return;
     }
 

package/src/schemas/toolOutputSchemas.js

@@ -189,6 +189,8 @@ const redditSearchShape = {
   count: z.number().optional(),
   results: z.array(z.union([redditPostShape, redditCommentShape])).optional().describe('posts/comments modes'),
   discovered: z.number().optional().describe('web_discovery: how many post ids the site-restricted web search surfaced before archive hydration'),
+  posts_searched: z.number().optional().describe('web_discovery comments mode: how many discovered posts had their comments searched before limit was reached'),
+  window_applied: z.string().optional().describe('arctic_shift comments mode: the after-window ("7d"/"3d"/"1d") the search was narrowed to after the full-history search timed out; absent when the caller set after or no narrowing was needed'),
   post: redditPostShape.nullable().optional().describe('thread mode: the post itself'),
   comments: z.array(z.unknown()).optional().describe('thread mode: nested comment tree ({...comment, replies:[...]}); collapsed branches appear as {more_count, more_ids}'),
   comment_count: z.number().optional(),

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -53,9 +53,11 @@ source and a ready-to-paste path.
 ```
 
 Pass `template:"list"` to enumerate templates. Supports `shopify-product`,
-`amazon-product`, `linkedin-profile`, `github-repo`, `youtube-video`, `tweet`,
-`reddit-thread`, `hacker-news-front-page`, `producthunt-launch`,
-`stackoverflow-question`, `npm-package`. Prefer `shopify-product` over scraping
+`amazon-product`, `github-repo`, `youtube-video`, `reddit-thread` (the post,
+from the Arctic Shift archive; `reddit_search` reads the comment tree),
+`hacker-news-front-page`, `producthunt-launch`, `stackoverflow-question`,
+`npm-package`. `linkedin-profile` and `tweet` are retired — those sites'
+robots.txt disallow every keyless path. Prefer `shopify-product` over scraping
 or LLM extraction on any Shopify storefront — it reads the store's own JSON, so
 prices and stock cannot be misread. Full field lists: [templates](references/templates.md).
 CLI: `crawlforge template github-repo https://github.com/owner/repo`.

package/src/skills/agent-skills/crawlforge-structured-extraction/references/templates.md

@@ -8,16 +8,21 @@ to enumerate at runtime. Cost: 1 credit per call.
 |-------------|--------|-------------------------|
 | `shopify-product` | Any Shopify storefront product page (`/products/<handle>`), custom domains included | title, price, compare-at price, on_sale, currency, price range, per-variant stock, options, images, tags — read from the store's `/products/<handle>.json`, not the rendered page |
 | `amazon-product` | Amazon product page (`/dp/...`) | title, price, currency, rating (number), review count (number), brand, ASIN, availability, full-size images, description, breadcrumbs. Breadcrumbs are empty on device pages, which genuinely have none |
-| `linkedin-profile` | LinkedIn public profile | name, headline, location, current role, about, experience |
 | `github-repo` | GitHub repository | name, owner, description, stars, forks, language, topics, README excerpt |
 | `youtube-video` | YouTube watch page | title, channel, views, likes, published date, description |
-| `tweet` | A single tweet/X post | author, handle, text, timestamp, likes, reposts |
-| `reddit-thread` | Reddit thread | title, subreddit, author, score, top comments |
+| `reddit-thread` | Reddit post (read from the Arctic Shift archive, not reddit.com, which blocks plain fetchers) | id, title, subreddit, author, score, upvote_ratio, num_comments, posted, body, link_url, flair, removed. Pass `id` to `reddit_search` with `mode:"thread"` for the comment tree |
 | `hacker-news-front-page` | HN front page | ranked stories: title, URL, points, author, comment count |
 | `producthunt-launch` | Product Hunt launch | name, tagline, upvotes, maker, description |
-| `stackoverflow-question` | Stack Overflow question | title, tags, votes, question body, accepted answer |
+| `stackoverflow-question` | Stack Overflow question (read from the keyless Stack Exchange API, not the page) | title, body, votes, views, tags, author, asked, answered, accepted_answer_id, answer_count, top 5 answers (accepted first: votes, author, posted, body) |
 | `npm-package` | npm package page | name, version, description, weekly downloads, license, repo link |
 
+**Retired (2026-08-30):** `linkedin-profile` and `tweet`. LinkedIn's robots.txt
+disallows every path for all agents but its own crawler and profiles sit behind
+an auth wall; X's robots.txt disallows every path for generic agents and its
+keyless embed endpoints are disallowed by their own robots.txt. Naming either
+returns the reason and fetches nothing. For X or LinkedIn data, use the
+platform's own API with the user's credentials.
+
 ## Usage
 
 ```json

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -675,19 +675,36 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // Prefer the post-action live page HTML captured during action execution.
       // This ensures the final content reflects clicks/typing/navigation rather
       // than re-fetching the original (pre-action) URL.
-      if (chainResult?.finalHtml) {
-        return await this.extractContentTool.execute({
+      const extractResult = chainResult?.finalHtml
+        ? await this.extractContentTool.execute({
           url: chainResult.finalUrl || params.url,
           html: chainResult.finalHtml,
           options
-        });
+        })
+        : await this.extractContentTool.execute({ url: params.url, options });
+
+      // extractContent never honoured customSelectors, so extractionOptions.
+      // selectors only ever surfaced inside captureIntermediateStates. Run them
+      // over the post-action DOM here, the one place every caller reads.
+      if (chainResult?.finalHtml && (params.extractionOptions?.selectors || params.formats?.includes('text'))) {
+        const $ = load(chainResult.finalHtml);
+        extractResult.content = extractResult.content || {};
+        if (params.extractionOptions?.selectors) {
+          extractResult.content.extracted = this.extractWithSelectors($, params.extractionOptions.selectors);
+        }
+        // Readability picks a "main" block and drops form controls; on a small
+        // post-action page (a demo form, a results panel) that leaves the
+        // footer alone, or a page minus the very buttons the actions produced.
+        // A short readable result means it found nothing substantial, so hand
+        // back the whole (equally short) body text instead.
+        const readable = extractResult.content.text || '';
+        const bodyText = $('body').text().replace(/\s+/g, ' ').trim();
+        if (readable.length < 300 && bodyText.length > readable.length) {
+          extractResult.content.text = bodyText;
+          extractResult.content.textSource = 'body';
+        }
       }
 
-      const extractResult = await this.extractContentTool.execute({
-        url: params.url,
-        options
-      });
-
       return extractResult;
     } catch (error) {
       this.log('warn', `Final content extraction failed: ${error.message}`);
@@ -706,6 +723,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
     if (formats.includes('json')) {
       content.json = {
         finalContent: finalContent.content || {},
+        ...(finalContent.content?.extracted ? { extracted: finalContent.content.extracted } : {}),
         metadata: finalContent.metadata || {},
         actionSummary: {
           totalActions: additionalData.actionResults.length,

package/src/tools/advanced/batchScrape/index.js

@@ -175,7 +175,7 @@ export class BatchScrapeTool extends EventEmitter {
       };
 
       if (this.enableResultCaching) {
-        this._cacheBatchResult(batchId, processedResults);
+        this._cacheBatchResult(batchId, processedResults, 'sync');
       }
 
       this.stats.completedBatches++;
@@ -237,7 +237,7 @@ export class BatchScrapeTool extends EventEmitter {
       if (Date.now() - cached.timestamp < cached.ttl) {
         const offset = (page - 1) * pageSize;
         return {
-          batchId, success: true,
+          batchId, success: true, status: 'completed', ...(cached.mode ? { mode: cached.mode } : {}),
           results: paginateResults(cached.results, offset, pageSize),
           pagination: { page, pageSize, totalResults: cached.results.length, totalPages: Math.ceil(cached.results.length / pageSize) },
           cached: true, timestamp: cached.timestamp
@@ -267,8 +267,11 @@ export class BatchScrapeTool extends EventEmitter {
       if (job.status === 'completed' && job.result) {
         const results = job.result.results || [];
         const offset = (page - 1) * pageSize;
+        // Same status/mode fields the pending and running branches carry, so
+        // a poller can key on `status` for the whole job lifecycle.
         return {
-          batchId, success: true, jobId: job.id,
+          batchId, success: true, jobId: job.id, status: 'completed', mode: 'async',
+          completedAt: job.completedAt,
           results: paginateResults(results, offset, pageSize),
           pagination: { page, pageSize, totalResults: results.length, totalPages: Math.ceil(results.length / pageSize) },
           cached: false, timestamp: job.completedAt
@@ -366,8 +369,8 @@ export class BatchScrapeTool extends EventEmitter {
    * Cache a batch's results, capped to maxCachedBatches. Map preserves
    * insertion order, so the oldest entry is evicted first (LRU by write time).
    */
-  _cacheBatchResult(batchId, results) {
-    this.batchResults.set(batchId, { results, timestamp: Date.now(), ttl: this.resultCacheTtl });
+  _cacheBatchResult(batchId, results, mode) {
+    this.batchResults.set(batchId, { results, mode, timestamp: Date.now(), ttl: this.resultCacheTtl });
     while (this.batchResults.size > this.maxCachedBatches) {
       const oldestKey = this.batchResults.keys().next().value;
       this.batchResults.delete(oldestKey);
@@ -430,7 +433,7 @@ export class BatchScrapeTool extends EventEmitter {
         };
 
         if (this.enableResultCaching) {
-          this._cacheBatchResult(batchId, processedResults);
+          this._cacheBatchResult(batchId, processedResults, 'async');
         }
 
         this.stats.totalUrls += results.length;

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -186,6 +186,9 @@ export class GenerateLLMsTxtTool {
     if (Array.isArray(outputOptions.customGuidelines) && outputOptions.customGuidelines.length > 0) {
       details.push(...outputOptions.customGuidelines);
     }
+    if (Array.isArray(outputOptions.customRestrictions) && outputOptions.customRestrictions.length > 0) {
+      details.push(`Do not access: ${outputOptions.customRestrictions.join(', ')}.`);
+    }
     if (details.length > 0) {
       lines.push(details.join(' '));
       lines.push('');
@@ -591,6 +594,13 @@ export class GenerateLLMsTxtTool {
     lines.push('- Form submissions and interactive content');
     lines.push('- Administrative and configuration areas');
     lines.push('');
+    if (Array.isArray(outputOptions.customRestrictions) && outputOptions.customRestrictions.length > 0) {
+      lines.push('### Additional Restrictions');
+      for (const restriction of outputOptions.customRestrictions) {
+        lines.push(`- ${restriction}`);
+      }
+      lines.push('');
+    }
 
     // Section 4: APIs and Data Sources
     lines.push('## 4. API and Data Sources');
@@ -669,6 +679,13 @@ export class GenerateLLMsTxtTool {
     lines.push('4. **Cache responsibly:** Cache content appropriately to reduce server load');
     lines.push('5. **Handle errors gracefully:** Implement proper error handling and retries');
     lines.push('');
+    if (Array.isArray(outputOptions.customGuidelines) && outputOptions.customGuidelines.length > 0) {
+      lines.push('### Site-Specific Guidelines');
+      for (const guideline of outputOptions.customGuidelines) {
+        lines.push(`- ${guideline}`);
+      }
+      lines.push('');
+    }
 
     if (analysis.structure && analysis.structure.robotsTxt) {
       lines.push('### Robots.txt Status');

package/src/tools/research/deepResearch.js

@@ -49,7 +49,7 @@ const DeepResearchSchema = z.object({
   
   // LLM Configuration
   llmConfig: z.object({
-    provider: z.enum(['auto', 'openai', 'anthropic']).optional().default('auto'),
+    provider: z.enum(['auto', 'openai', 'anthropic', 'ollama']).optional().default('auto'),
     openai: z.object({
       apiKey: z.string().optional(),
       model: z.string().optional().default('gpt-3.5-turbo'),
@@ -59,6 +59,10 @@ const DeepResearchSchema = z.object({
       apiKey: z.string().optional(),
       model: z.string().optional().default('claude-3-haiku-20240307')
     }).optional(),
+    ollama: z.object({
+      model: z.string().optional(),
+      embeddingModel: z.string().optional()
+    }).optional(),
     enableSemanticAnalysis: z.boolean().optional().default(true),
     enableIntelligentSynthesis: z.boolean().optional().default(true)
   }).optional(),
@@ -265,9 +269,23 @@ export class DeepResearchTool {
       ...baseConfig.searchConfig
     };
 
-    // Add LLM configuration if provided
+    // Add LLM configuration if provided. LLMManager selects on
+    // `defaultProvider`; the schema's `provider` was never mapped onto it, so
+    // an explicit choice was silently ignored and 'auto' always ran.
     if (params.llmConfig) {
-      baseConfig.llmConfig = params.llmConfig;
+      const { provider = 'auto', ...llmConfig } = params.llmConfig;
+      const cloudKey = {
+        openai: llmConfig.openai?.apiKey || process.env.OPENAI_API_KEY,
+        anthropic: llmConfig.anthropic?.apiKey || process.env.ANTHROPIC_API_KEY
+      };
+      if (provider in cloudKey && !cloudKey[provider]) {
+        throw new Error(
+          `llmConfig.provider "${provider}" needs an API key: pass llmConfig.${provider}.apiKey ` +
+          `or set ${provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY'}. ` +
+          'Omit provider (or pass "ollama") to use the local Ollama, which needs no key.'
+        );
+      }
+      baseConfig.llmConfig = { ...llmConfig, defaultProvider: provider };
     }
 
     // params.cacheResults was previously dropped entirely — only the

package/src/tools/search/redditSearch.js

@@ -15,9 +15,11 @@
  *   cross-subreddit full-text search, but has known post-2023 archive gaps
  *   and recurring outages.
  *
- * Routing: scoped searches go to Arctic Shift (fresher) with PullPush as an
- * error-only fallback; unscoped keyword searches can only go to PullPush.
- * Both services are free and need no credentials.
+ * Routing: scoped searches go to Arctic Shift. An unscoped keyword search
+ * finds posts through a site-restricted web search and then reads them — or
+ * searches their comments — in the archive by id. PullPush stopped serving
+ * automated clients in August 2026 and is used only on explicit request.
+ * Both archives are free and need no credentials.
  *
  * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
  * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
@@ -148,14 +150,15 @@ export class RedditSearchTool {
     // comment full-text search, so `comments` mode always uses the archives.
     const officialPossible = this.officialConfigured && (v.mode === 'thread' || v.mode === 'posts');
 
-    // A Reddit-wide keyword search for posts can be discovered through a web
-    // search and then read out of the archive by ID.
-    const discoveryPossible = v.mode === 'posts' && Boolean(v.query) && !subreddit && !author;
+    // A Reddit-wide keyword search can be discovered through a web search and
+    // then read out of the archive by ID — posts directly, comments by
+    // searching each discovered post's comments for the keywords.
+    const discoveryPossible = v.mode !== 'thread' && Boolean(v.query) && !scoped;
 
     let order; // backends to try, in order
     if (v.source === 'web_discovery') {
       if (!discoveryPossible) {
-        throw new Error('web_discovery only serves unscoped keyword searches in posts mode — it finds posts through a web search and then reads them from the archive');
+        throw new Error('web_discovery only serves unscoped keyword searches (posts or comments) — it finds posts through a web search and then reads them from the archive');
       }
       order = ['web_discovery'];
     } else if (v.source === 'reddit_api') {
@@ -187,14 +190,6 @@ export class RedditSearchTool {
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
 
-    if (order.length === 0) {
-      // Unscoped comment search was PullPush-only, and PullPush no longer
-      // serves automated clients. Web discovery identifies posts, not comments.
-      throw new Error(
-        `An unscoped keyword search of ${v.mode} has no available backend: Arctic Shift requires a subreddit or author scope, and PullPush no longer serves automated clients. Add a subreddit or author filter, or search mode:"posts", which finds posts through a web search and reads them from the archive.`
-      );
-    }
-
     const errors = [];
     for (const source of order) {
       try {
@@ -226,11 +221,12 @@ export class RedditSearchTool {
 
   /**
    * Reddit-wide keyword search, in two steps: find matching posts with a
-   * site-restricted web search, then read those posts out of the Arctic Shift
-   * archive by ID. Arctic Shift cannot keyword-search across all of Reddit and
-   * reddit.com blocks scrapers, so discovery has to come from somewhere else —
-   * but what comes back are real archive rows, the same shape a scoped search
-   * returns, not scraped search-engine snippets.
+   * site-restricted web search, then read them out of the Arctic Shift archive
+   * by ID — the posts themselves, or (comments mode) each post's comments that
+   * match the keywords. Arctic Shift cannot keyword-search across all of
+   * Reddit and reddit.com blocks scrapers, so discovery has to come from
+   * somewhere else — but what comes back are real archive rows, the same shape
+   * a scoped search returns, not scraped search-engine snippets.
    */
   async #searchWebDiscovery(v) {
     let found;
@@ -252,7 +248,9 @@ export class RedditSearchTool {
     }
 
     const notes = [
-      'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
+      v.mode === 'comments'
+        ? 'Reddit-wide comment search: posts were found with a site-restricted web search, then each post\'s comments were searched for the keywords in the Arctic Shift archive, in post relevance order.'
+        : 'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
       'Results are ordered by web-search relevance, not by score or date.',
       'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
     ];
@@ -262,11 +260,47 @@ export class RedditSearchTool {
       notes.push('after/before were NOT applied: discovery runs through a web search, which cannot filter by post date. Scope the search to a subreddit or author to use date filters.');
     }
 
+    const envelope = {
+      source: 'web_discovery', mode: v.mode,
+      query: v.query ?? null, subreddit: null, author: null,
+    };
     if (ids.length === 0) {
+      return { ...envelope, count: 0, results: [], notes, checkedAt: new Date().toISOString() };
+    }
+
+    if (v.mode === 'comments') {
+      // Arctic Shift's body search needs a scope, and link_id is one: search
+      // the discovered posts in relevance order until `limit` comments are in
+      // hand. A post whose search fails is skipped and counted, so one
+      // throttled request does not discard the others' results.
+      const results = [];
+      const skipped = [];
+      let postsSearched = 0;
+      for (const id of ids) {
+        if (results.length >= v.limit) break;
+        try {
+          const data = await this.#get(`${this.arcticBaseUrl}/api/comments/search`, {
+            link_id: id,
+            body: v.query,
+            limit: String(v.limit - results.length),
+            sort: v.sort,
+          });
+          postsSearched++;
+          const rows = Array.isArray(data.data) ? data.data : [];
+          results.push(...rows.map(normalizeComment));
+        } catch (error) {
+          skipped.push(`${id}: ${error.message}`);
+        }
+      }
+      if (postsSearched === 0) {
+        throw new Error(`comment search failed for every discovered post — ${skipped.join('; ')}`);
+      }
+      if (skipped.length > 0) {
+        notes.push(`${skipped.length} discovered post(s) could not be searched: ${skipped.join('; ')}`);
+      }
       return {
-        source: 'web_discovery', mode: 'posts',
-        query: v.query ?? null, subreddit: null, author: null,
-        count: 0, results: [],
+        ...envelope, count: results.length, results,
+        discovered: ids.length, posts_searched: postsSearched,
         notes, checkedAt: new Date().toISOString(),
       };
     }
@@ -280,9 +314,7 @@ export class RedditSearchTool {
     const ordered = ids.map(id => byId.get(id)).filter(Boolean);
 
     return {
-      source: 'web_discovery', mode: 'posts',
-      query: v.query ?? null, subreddit: null, author: null,
-      count: ordered.length, results: ordered.map(normalizePost),
+      ...envelope, count: ordered.length, results: ordered.map(normalizePost),
       discovered: ids.length,
       notes, checkedAt: new Date().toISOString(),
     };
@@ -327,13 +359,45 @@ export class RedditSearchTool {
       if (v.query) query.body = v.query;
       if (v.link_id) query.link_id = stripIdPrefix(v.link_id);
     }
-    const data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
+    let data;
+    let windowApplied = null;
+    try {
+      data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
+    } catch (error) {
+      // Arctic Shift sheds a scoped comment keyword search over its whole
+      // history with the same 422 "Timeout. Maybe slow down a bit" it uses for
+      // throttling (observed live 2026-08-30: r/webdev body=react failed
+      // unbounded and at 7d, answered at 3d). When the caller set no window,
+      // narrow it step by step rather than hand the caller the 422.
+      const narrowable = v.mode === 'comments' && Boolean(v.query) && !v.after;
+      if (!error.retryable || !narrowable) throw error;
+      let lastError = error;
+      for (const window of ['7d', '3d', '1d']) {
+        await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));
+        try {
+          data = await this.#getOnce(`${this.arcticBaseUrl}${path}?${new URLSearchParams({ ...query, after: window })}`);
+          windowApplied = window;
+          break;
+        } catch (retryError) {
+          lastError = retryError;
+          if (!retryError.retryable) throw retryError;
+        }
+      }
+      if (!data) {
+        throw new Error(
+          `${lastError.message} — Arctic Shift timed out at every window (full history, 7d, 3d, 1d), ` +
+          'so its comment search is under load right now. Retry later, pass a narrower after/before, or scope to one post with link_id.'
+        );
+      }
+      notes.push(`Arctic Shift timed out searching the full history; results are limited to the last ${windowApplied}. Pass after/before to choose the window.`);
+    }
     const rows = Array.isArray(data.data) ? data.data : [];
     const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
     return {
       source: 'arctic_shift', mode: v.mode,
       query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,
       count: results.length, results,
+      ...(windowApplied && { window_applied: windowApplied }),
       notes, checkedAt: new Date().toISOString(),
     };
   }

package/src/tools/templates/ScrapeTemplateTool.js

@@ -11,7 +11,7 @@
  * one call — the registry builds the URL, this tool fetches it.
  */
 
-import { TemplateRegistry } from 'crawlforge-extractors';
+import { TemplateRegistry, retiredTemplate } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -60,6 +60,12 @@ export class ScrapeTemplateTool {
       }
       const detected = this.registry.detect(url);
       if (!detected) {
+        // A URL a withdrawn template used to handle gets the reason it is
+        // gone, not "no template matches".
+        const retired = retiredTemplate(url);
+        if (retired) {
+          throw badRequest(`The "${retired.id}" template that handled ${url} was retired: ${retired.reason}`);
+        }
         throw badRequest(
           `No template matches ${url}. Pass template:"list" to see every template and the URLs ` +
           'each one handles, or name a template explicitly.'
@@ -74,6 +80,10 @@ export class ScrapeTemplateTool {
     // Validate template exists before making network call
     const tpl = this.registry.get(templateId);
     if (!tpl) {
+      const retired = retiredTemplate(templateId);
+      if (retired) {
+        throw badRequest(`Template "${templateId}" was retired: ${retired.reason}`);
+      }
       const available = this.registry.list().map(t => t.id).join(', ');
       throw new Error(`Unknown template "${templateId}". Available templates: ${available}`);
     }

package/src/tools/tracking/trackChanges/index.js

@@ -289,6 +289,7 @@ export class TrackChangesTool extends EventEmitter {
       ...(comparisonResult.warnings || fetchWarnings.length
         ? { warnings: [...(comparisonResult.warnings || []), ...fetchWarnings] }
         : {}),
+      ...(comparisonResult.alerts ? { alerts: comparisonResult.alerts } : {}),
       snapshot: snapshotInfo, timestamp: Date.now()
     };
   }
@@ -390,16 +391,36 @@ export class TrackChangesTool extends EventEmitter {
     const { url, scheduledMonitorOptions, monitoringOptions, trackingOptions, notificationOptions } = params;
     if (!url) throw new Error('create_scheduled_monitor requires a url');
     const opts = scheduledMonitorOptions || {};
+
+    // A preset fills in whatever the caller left unspecified; explicit values win.
+    let preset = null;
+    if (opts.templateId) {
+      preset = this.changeTracker.monitoringTemplates.get(opts.templateId);
+      if (!preset) {
+        throw new Error(
+          `Unknown monitoring template "${opts.templateId}". ` +
+          `Available: ${Array.from(this.changeTracker.monitoringTemplates.keys()).join(', ')} ` +
+          '(operation:"get_monitoring_templates" describes each).'
+        );
+      }
+    }
+    // Precedence: scheduledMonitorOptions > preset > monitoringOptions. The
+    // schema fills monitoringOptions.interval/notificationThreshold with
+    // defaults, so they cannot sit above a preset without always winning.
     const monitor = await this.scheduler.createMonitor({
       url,
-      interval: opts.interval ?? monitoringOptions?.interval,
+      interval: opts.interval ?? preset?.frequency ?? monitoringOptions?.interval,
       schedule: opts.schedule,
-      goal: opts.goal,
-      notificationThreshold: opts.notificationThreshold || monitoringOptions?.notificationThreshold || 'moderate',
-      trackingOptions,
+      goal: opts.goal ?? preset?.goal,
+      notificationThreshold: opts.notificationThreshold || preset?.notificationThreshold || monitoringOptions?.notificationThreshold || 'moderate',
+      trackingOptions: preset ? { ...preset.options, ...(trackingOptions || {}) } : trackingOptions,
       notificationOptions
     });
-    return { success: true, operation: 'create_scheduled_monitor', url, monitor, firingGuarantee: monitor.firingGuarantee, timestamp: Date.now() };
+    return {
+      success: true, operation: 'create_scheduled_monitor', url, monitor,
+      ...(preset ? { templateId: preset.id } : {}),
+      firingGuarantee: monitor.firingGuarantee, timestamp: Date.now()
+    };
   }
 
   async stopScheduledMonitor(params) {
@@ -424,8 +445,24 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async getMonitoringDashboard(params) {
-    const { dashboardOptions } = params;
+    // The schema defaults every flag to true, but only inside a dashboardOptions
+    // object the caller supplied; an omitted object must mean the same thing.
+    const dashboardOptions = { includeRecentAlerts: true, includeTrends: true, includeMonitorStatus: true, ...(params.dashboardOptions || {}) };
     const dashboard = this.changeTracker.getMonitoringDashboard();
+    // The tracker only knows tracked pages; the monitors that poll them live here.
+    const polling = Array.from(this.activeMonitors.entries()).map(([url, m]) => ({
+      id: `poll:${url}`, url, kind: 'polling', status: 'active',
+      interval: m.options?.interval ?? null, stats: m.stats
+    }));
+    if (!this.monitorStore._loaded) await this.monitorStore.load();
+    const scheduled = this.scheduler.list().map(m => ({
+      id: m.id, url: m.url, kind: 'scheduled',
+      status: m.enabled === false ? 'disabled' : m.scheduled ? 'scheduled' : 'idle',
+      interval: m.interval, goal: m.goal, nextDueAt: m.nextDueAt,
+      lastCheckAt: m.lastCheckAt, lastChangeAt: m.lastChangeAt, stats: m.stats
+    }));
+    dashboard.monitors = [...polling, ...scheduled];
+    dashboard.summary.activeMonitors = dashboard.monitors.length;
     if (!dashboardOptions?.includeRecentAlerts) delete dashboard.recentAlerts;
     if (!dashboardOptions?.includeTrends) delete dashboard.trends;
     if (!dashboardOptions?.includeMonitorStatus) {
@@ -441,16 +478,22 @@ export class TrackChangesTool extends EventEmitter {
   }
 
   async createAlertRule(params) {
-    const { alertRuleOptions } = params;
+    const { url, alertRuleOptions } = params;
     const ruleId = alertRuleOptions?.ruleId || `custom_rule_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-    const rule = {
-      condition: this._parseCondition(alertRuleOptions?.condition || 'significance === "major"'),
+    const rule = this.changeTracker.addAlertRule(ruleId, {
+      condition: alertRuleOptions?.condition,
+      url: url || null,
       actions: alertRuleOptions?.actions || ['webhook'],
-      throttle: alertRuleOptions?.throttle || 600000,
+      throttle: alertRuleOptions?.throttle ?? 600000,
       priority: alertRuleOptions?.priority || 'medium'
+    });
+    // Rules fire from compare: every later compare that matches returns an
+    // `alerts` array, and get_dashboard lists what fired.
+    return {
+      success: true, operation: 'create_alert_rule', ruleId,
+      rule: { id: rule.id, url: rule.url, condition: rule.conditionText, actions: rule.actions, throttle: rule.throttle, priority: rule.priority },
+      timestamp: Date.now()
     };
-    this.changeTracker.alertRules.set(ruleId, rule);
-    return { success: true, operation: 'create_alert_rule', ruleId, rule, timestamp: Date.now() };
   }
 
   async generateTrendReport(params) {
@@ -461,7 +504,11 @@ export class TrackChangesTool extends EventEmitter {
   async getMonitoringTemplates() {
     const templates = {};
     for (const [id, template] of this.changeTracker.monitoringTemplates.entries()) {
-      templates[id] = { name: template.name, frequency: template.frequency, options: template.options, alertRules: template.alertRules };
+      templates[id] = {
+        name: template.name, description: template.description, frequency: template.frequency,
+        notificationThreshold: template.notificationThreshold, goal: template.goal,
+        options: template.options, alertRules: template.alertRules
+      };
     }
     return { success: true, operation: 'get_monitoring_templates', templates, count: Object.keys(templates).length, timestamp: Date.now() };
   }
@@ -533,20 +580,6 @@ export class TrackChangesTool extends EventEmitter {
     stats.newestMonitor = Math.max(...all.map(v => v.started));
     return stats;
   }
-
-  _parseCondition(conditionString) {
-    return (changeResult) => {
-      try {
-        if (conditionString.includes('significance')) {
-          const match = conditionString.match(/significance\s*===\s*["'](\w+)["']/);
-          if (match) return changeResult.significance === match[1];
-        }
-        return false;
-      } catch {
-        return false;
-      }
-    };
-  }
 }
 
 export default TrackChangesTool;