crawlforge-mcp-server 5.4.3 → 5.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.4.3",
+  "version": "5.5.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.4.1",
+    "crawlforge-extractors": "^1.5.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -105,7 +105,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.4.3",
+  version: "5.5.0",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -492,7 +492,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search has no backend — scope it or search posts. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -504,7 +504,7 @@ registerToolIfEnabled("reddit_search", {
     before: z.string().optional().describe("Only content posted before this date — same formats as after"),
     limit: z.number().min(1).max(100).optional().describe("Max results (default 25; thread mode: max comments returned)"),
     sort: z.enum(["asc", "desc"]).optional().describe("Sort by post date (default desc = newest first)"),
-    source: z.enum(["auto", "arctic_shift", "pullpush", "reddit_api"]).optional().describe("Backend: auto routes + falls back (default). reddit_api uses the official Reddit Data API — only when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set; serves posts/thread, not comment search")
+    source: z.enum(["auto", "arctic_shift", "pullpush", "reddit_api", "web_discovery"]).optional().describe("Backend: auto routes + falls back (default). web_discovery serves only unscoped keyword searches (web search finds the posts, the archive supplies the rows). reddit_api uses the official Reddit Data API — only when REDDIT_CLIENT_ID/REDDIT_CLIENT_SECRET are set; serves posts/thread, not comment search")
   },
   outputSchema: OUTPUT_SCHEMAS.reddit_search
 }, withAuth("reddit_search", async ({ query, subreddit, author, mode, link_id, after, before, limit, sort, source }) => {
@@ -977,7 +977,7 @@ if (toolFilter.isEnabled("deep_research")) {
         maxVariations: z.number().min(1).max(20).optional().default(8)
       }).optional().describe("Query expansion settings for broader search coverage"),
       llmConfig: z.object({
-        provider: z.enum(['auto', 'openai', 'anthropic']).optional().default('auto'),
+        provider: z.enum(['auto', 'openai', 'anthropic', 'ollama']).optional().default('auto'),
         openai: z.object({
           apiKey: z.string().optional(),
           model: z.string().optional().default('gpt-3.5-turbo'),
@@ -987,9 +987,13 @@ if (toolFilter.isEnabled("deep_research")) {
           apiKey: z.string().optional(),
           model: z.string().optional().default('claude-3-haiku-20240307')
         }).optional(),
+        ollama: z.object({
+          model: z.string().optional(),
+          embeddingModel: z.string().optional()
+        }).optional(),
         enableSemanticAnalysis: z.boolean().optional().default(true),
         enableIntelligentSynthesis: z.boolean().optional().default(true)
-      }).optional().describe("LLM provider configuration for AI-powered analysis"),
+      }).optional().describe("LLM provider configuration for AI-powered analysis. provider 'auto' (default) uses a configured cloud key if there is one, else the local Ollama (http://localhost:11434, no key); 'ollama' forces the local model; 'openai'/'anthropic' need the matching API key"),
       concurrency: z.number().min(1).max(20).optional().default(5).describe("Number of concurrent research requests"),
       cacheResults: z.boolean().optional().default(true).describe("Cache research results for reuse"),
       webhook: z.object({
@@ -1534,10 +1538,30 @@ registerToolIfEnabled("localization", {
         if (!params.countryCode) throw new Error('countryCode is required for configure_country operation');
         result = await localizationManager.configureCountry(params.countryCode, params);
         break;
-      case 'localize_search':
+      case 'localize_search': {
         if (!params.searchParams) throw new Error('searchParams is required for localize_search operation');
-        result = await localizationManager.localizeSearchQuery(params.searchParams, params.countryCode);
+        const localizedParams = await localizationManager.localizeSearchQuery(params.searchParams, params.countryCode);
+        if (!params.searchParams.query) {
+          result = {
+            ...localizedParams,
+            note: 'No searchParams.query was given, so no search ran — these are the parameters a localized search would use.'
+          };
+          break;
+        }
+        // Run the search the caller asked for under the localized country and
+        // language, so the operation returns results rather than a config.
+        // Priced as a search_web call (AuthManager.getToolCost).
+        const countryCode = params.countryCode || localizationManager.getCurrentSettings().countryCode;
+        const search = await searchWebTool.execute({
+          query: params.searchParams.query,
+          limit: params.searchParams.limit,
+          offset: params.searchParams.offset,
+          lang: localizedParams.lang,
+          localization: { countryCode, language: params.language || localizedParams.lang }
+        });
+        result = { localizedParams, search };
         break;
+      }
       case 'localize_browser':
         if (!params.browserOptions) throw new Error('browserOptions is required for localize_browser operation');
         result = await localizationManager.localizeBrowserContext(params.browserOptions, params.countryCode);
@@ -1578,7 +1602,7 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package, linkedin-profile). List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
+  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package; reddit-thread reads the post from the Arctic Shift archive and reddit_search reads the comment tree). linkedin-profile and tweet are retired — those sites' robots.txt disallow every keyless path — and naming one returns the reason. List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     template: z.string().describe("Template ID (e.g. github-repo), \"auto\" to detect one from the url, or \"list\" to enumerate available templates"),

package/src/core/AuthManager.js

@@ -612,6 +612,12 @@ class AuthManager {
       if (bookkeepingOps.has(params?.operation)) return 1;
     }
 
+    // localize_search with a query runs a real web search through the same
+    // adapter search_web uses, so it is priced as one rather than undercutting it.
+    if (tool === 'localization' && params?.operation === 'localize_search' && params?.searchParams?.query) {
+      return costs.search_web;
+    }
+
     return costs[tool] ?? 1;
   }
 

package/src/schemas/toolOutputSchemas.js

@@ -189,6 +189,8 @@ const redditSearchShape = {
   count: z.number().optional(),
   results: z.array(z.union([redditPostShape, redditCommentShape])).optional().describe('posts/comments modes'),
   discovered: z.number().optional().describe('web_discovery: how many post ids the site-restricted web search surfaced before archive hydration'),
+  posts_searched: z.number().optional().describe('web_discovery comments mode: how many discovered posts had their comments searched before limit was reached'),
+  window_applied: z.string().optional().describe('arctic_shift comments mode: the after-window ("7d"/"3d"/"1d") the search was narrowed to after the full-history search timed out; absent when the caller set after or no narrowing was needed'),
   post: redditPostShape.nullable().optional().describe('thread mode: the post itself'),
   comments: z.array(z.unknown()).optional().describe('thread mode: nested comment tree ({...comment, replies:[...]}); collapsed branches appear as {more_count, more_ids}'),
   comment_count: z.number().optional(),

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -53,9 +53,11 @@ source and a ready-to-paste path.
 ```
 
 Pass `template:"list"` to enumerate templates. Supports `shopify-product`,
-`amazon-product`, `linkedin-profile`, `github-repo`, `youtube-video`, `tweet`,
-`reddit-thread`, `hacker-news-front-page`, `producthunt-launch`,
-`stackoverflow-question`, `npm-package`. Prefer `shopify-product` over scraping
+`amazon-product`, `github-repo`, `youtube-video`, `reddit-thread` (the post,
+from the Arctic Shift archive; `reddit_search` reads the comment tree),
+`hacker-news-front-page`, `producthunt-launch`, `stackoverflow-question`,
+`npm-package`. `linkedin-profile` and `tweet` are retired — those sites'
+robots.txt disallow every keyless path. Prefer `shopify-product` over scraping
 or LLM extraction on any Shopify storefront — it reads the store's own JSON, so
 prices and stock cannot be misread. Full field lists: [templates](references/templates.md).
 CLI: `crawlforge template github-repo https://github.com/owner/repo`.

package/src/skills/agent-skills/crawlforge-structured-extraction/references/templates.md

@@ -8,16 +8,21 @@ to enumerate at runtime. Cost: 1 credit per call.
 |-------------|--------|-------------------------|
 | `shopify-product` | Any Shopify storefront product page (`/products/<handle>`), custom domains included | title, price, compare-at price, on_sale, currency, price range, per-variant stock, options, images, tags — read from the store's `/products/<handle>.json`, not the rendered page |
 | `amazon-product` | Amazon product page (`/dp/...`) | title, price, currency, rating (number), review count (number), brand, ASIN, availability, full-size images, description, breadcrumbs. Breadcrumbs are empty on device pages, which genuinely have none |
-| `linkedin-profile` | LinkedIn public profile | name, headline, location, current role, about, experience |
 | `github-repo` | GitHub repository | name, owner, description, stars, forks, language, topics, README excerpt |
 | `youtube-video` | YouTube watch page | title, channel, views, likes, published date, description |
-| `tweet` | A single tweet/X post | author, handle, text, timestamp, likes, reposts |
-| `reddit-thread` | Reddit thread | title, subreddit, author, score, top comments |
+| `reddit-thread` | Reddit post (read from the Arctic Shift archive, not reddit.com, which blocks plain fetchers) | id, title, subreddit, author, score, upvote_ratio, num_comments, posted, body, link_url, flair, removed. Pass `id` to `reddit_search` with `mode:"thread"` for the comment tree |
 | `hacker-news-front-page` | HN front page | ranked stories: title, URL, points, author, comment count |
 | `producthunt-launch` | Product Hunt launch | name, tagline, upvotes, maker, description |
 | `stackoverflow-question` | Stack Overflow question (read from the keyless Stack Exchange API, not the page) | title, body, votes, views, tags, author, asked, answered, accepted_answer_id, answer_count, top 5 answers (accepted first: votes, author, posted, body) |
 | `npm-package` | npm package page | name, version, description, weekly downloads, license, repo link |
 
+**Retired (2026-08-30):** `linkedin-profile` and `tweet`. LinkedIn's robots.txt
+disallows every path for all agents but its own crawler and profiles sit behind
+an auth wall; X's robots.txt disallows every path for generic agents and its
+keyless embed endpoints are disallowed by their own robots.txt. Naming either
+returns the reason and fetches nothing. For X or LinkedIn data, use the
+platform's own API with the user's credentials.
+
 ## Usage
 
 ```json

package/src/tools/research/deepResearch.js

@@ -49,7 +49,7 @@ const DeepResearchSchema = z.object({
   
   // LLM Configuration
   llmConfig: z.object({
-    provider: z.enum(['auto', 'openai', 'anthropic']).optional().default('auto'),
+    provider: z.enum(['auto', 'openai', 'anthropic', 'ollama']).optional().default('auto'),
     openai: z.object({
       apiKey: z.string().optional(),
       model: z.string().optional().default('gpt-3.5-turbo'),
@@ -59,6 +59,10 @@ const DeepResearchSchema = z.object({
       apiKey: z.string().optional(),
       model: z.string().optional().default('claude-3-haiku-20240307')
     }).optional(),
+    ollama: z.object({
+      model: z.string().optional(),
+      embeddingModel: z.string().optional()
+    }).optional(),
     enableSemanticAnalysis: z.boolean().optional().default(true),
     enableIntelligentSynthesis: z.boolean().optional().default(true)
   }).optional(),
@@ -265,9 +269,23 @@ export class DeepResearchTool {
       ...baseConfig.searchConfig
     };
 
-    // Add LLM configuration if provided
+    // Add LLM configuration if provided. LLMManager selects on
+    // `defaultProvider`; the schema's `provider` was never mapped onto it, so
+    // an explicit choice was silently ignored and 'auto' always ran.
     if (params.llmConfig) {
-      baseConfig.llmConfig = params.llmConfig;
+      const { provider = 'auto', ...llmConfig } = params.llmConfig;
+      const cloudKey = {
+        openai: llmConfig.openai?.apiKey || process.env.OPENAI_API_KEY,
+        anthropic: llmConfig.anthropic?.apiKey || process.env.ANTHROPIC_API_KEY
+      };
+      if (provider in cloudKey && !cloudKey[provider]) {
+        throw new Error(
+          `llmConfig.provider "${provider}" needs an API key: pass llmConfig.${provider}.apiKey ` +
+          `or set ${provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY'}. ` +
+          'Omit provider (or pass "ollama") to use the local Ollama, which needs no key.'
+        );
+      }
+      baseConfig.llmConfig = { ...llmConfig, defaultProvider: provider };
     }
 
     // params.cacheResults was previously dropped entirely — only the

package/src/tools/search/redditSearch.js

@@ -15,9 +15,11 @@
  *   cross-subreddit full-text search, but has known post-2023 archive gaps
  *   and recurring outages.
  *
- * Routing: scoped searches go to Arctic Shift (fresher) with PullPush as an
- * error-only fallback; unscoped keyword searches can only go to PullPush.
- * Both services are free and need no credentials.
+ * Routing: scoped searches go to Arctic Shift. An unscoped keyword search
+ * finds posts through a site-restricted web search and then reads them — or
+ * searches their comments — in the archive by id. PullPush stopped serving
+ * automated clients in August 2026 and is used only on explicit request.
+ * Both archives are free and need no credentials.
  *
  * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
  * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
@@ -148,14 +150,15 @@ export class RedditSearchTool {
     // comment full-text search, so `comments` mode always uses the archives.
     const officialPossible = this.officialConfigured && (v.mode === 'thread' || v.mode === 'posts');
 
-    // A Reddit-wide keyword search for posts can be discovered through a web
-    // search and then read out of the archive by ID.
-    const discoveryPossible = v.mode === 'posts' && Boolean(v.query) && !subreddit && !author;
+    // A Reddit-wide keyword search can be discovered through a web search and
+    // then read out of the archive by ID — posts directly, comments by
+    // searching each discovered post's comments for the keywords.
+    const discoveryPossible = v.mode !== 'thread' && Boolean(v.query) && !scoped;
 
     let order; // backends to try, in order
     if (v.source === 'web_discovery') {
       if (!discoveryPossible) {
-        throw new Error('web_discovery only serves unscoped keyword searches in posts mode — it finds posts through a web search and then reads them from the archive');
+        throw new Error('web_discovery only serves unscoped keyword searches (posts or comments) — it finds posts through a web search and then reads them from the archive');
       }
       order = ['web_discovery'];
     } else if (v.source === 'reddit_api') {
@@ -187,14 +190,6 @@ export class RedditSearchTool {
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
 
-    if (order.length === 0) {
-      // Unscoped comment search was PullPush-only, and PullPush no longer
-      // serves automated clients. Web discovery identifies posts, not comments.
-      throw new Error(
-        `An unscoped keyword search of ${v.mode} has no available backend: Arctic Shift requires a subreddit or author scope, and PullPush no longer serves automated clients. Add a subreddit or author filter, or search mode:"posts", which finds posts through a web search and reads them from the archive.`
-      );
-    }
-
     const errors = [];
     for (const source of order) {
       try {
@@ -226,11 +221,12 @@ export class RedditSearchTool {
 
   /**
    * Reddit-wide keyword search, in two steps: find matching posts with a
-   * site-restricted web search, then read those posts out of the Arctic Shift
-   * archive by ID. Arctic Shift cannot keyword-search across all of Reddit and
-   * reddit.com blocks scrapers, so discovery has to come from somewhere else —
-   * but what comes back are real archive rows, the same shape a scoped search
-   * returns, not scraped search-engine snippets.
+   * site-restricted web search, then read them out of the Arctic Shift archive
+   * by ID — the posts themselves, or (comments mode) each post's comments that
+   * match the keywords. Arctic Shift cannot keyword-search across all of
+   * Reddit and reddit.com blocks scrapers, so discovery has to come from
+   * somewhere else — but what comes back are real archive rows, the same shape
+   * a scoped search returns, not scraped search-engine snippets.
    */
   async #searchWebDiscovery(v) {
     let found;
@@ -252,7 +248,9 @@ export class RedditSearchTool {
     }
 
     const notes = [
-      'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
+      v.mode === 'comments'
+        ? 'Reddit-wide comment search: posts were found with a site-restricted web search, then each post\'s comments were searched for the keywords in the Arctic Shift archive, in post relevance order.'
+        : 'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
       'Results are ordered by web-search relevance, not by score or date.',
       'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
     ];
@@ -262,11 +260,47 @@ export class RedditSearchTool {
       notes.push('after/before were NOT applied: discovery runs through a web search, which cannot filter by post date. Scope the search to a subreddit or author to use date filters.');
     }
 
+    const envelope = {
+      source: 'web_discovery', mode: v.mode,
+      query: v.query ?? null, subreddit: null, author: null,
+    };
     if (ids.length === 0) {
+      return { ...envelope, count: 0, results: [], notes, checkedAt: new Date().toISOString() };
+    }
+
+    if (v.mode === 'comments') {
+      // Arctic Shift's body search needs a scope, and link_id is one: search
+      // the discovered posts in relevance order until `limit` comments are in
+      // hand. A post whose search fails is skipped and counted, so one
+      // throttled request does not discard the others' results.
+      const results = [];
+      const skipped = [];
+      let postsSearched = 0;
+      for (const id of ids) {
+        if (results.length >= v.limit) break;
+        try {
+          const data = await this.#get(`${this.arcticBaseUrl}/api/comments/search`, {
+            link_id: id,
+            body: v.query,
+            limit: String(v.limit - results.length),
+            sort: v.sort,
+          });
+          postsSearched++;
+          const rows = Array.isArray(data.data) ? data.data : [];
+          results.push(...rows.map(normalizeComment));
+        } catch (error) {
+          skipped.push(`${id}: ${error.message}`);
+        }
+      }
+      if (postsSearched === 0) {
+        throw new Error(`comment search failed for every discovered post — ${skipped.join('; ')}`);
+      }
+      if (skipped.length > 0) {
+        notes.push(`${skipped.length} discovered post(s) could not be searched: ${skipped.join('; ')}`);
+      }
       return {
-        source: 'web_discovery', mode: 'posts',
-        query: v.query ?? null, subreddit: null, author: null,
-        count: 0, results: [],
+        ...envelope, count: results.length, results,
+        discovered: ids.length, posts_searched: postsSearched,
         notes, checkedAt: new Date().toISOString(),
       };
     }
@@ -280,9 +314,7 @@ export class RedditSearchTool {
     const ordered = ids.map(id => byId.get(id)).filter(Boolean);
 
     return {
-      source: 'web_discovery', mode: 'posts',
-      query: v.query ?? null, subreddit: null, author: null,
-      count: ordered.length, results: ordered.map(normalizePost),
+      ...envelope, count: ordered.length, results: ordered.map(normalizePost),
       discovered: ids.length,
       notes, checkedAt: new Date().toISOString(),
     };
@@ -327,13 +359,45 @@ export class RedditSearchTool {
       if (v.query) query.body = v.query;
       if (v.link_id) query.link_id = stripIdPrefix(v.link_id);
     }
-    const data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
+    let data;
+    let windowApplied = null;
+    try {
+      data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
+    } catch (error) {
+      // Arctic Shift sheds a scoped comment keyword search over its whole
+      // history with the same 422 "Timeout. Maybe slow down a bit" it uses for
+      // throttling (observed live 2026-08-30: r/webdev body=react failed
+      // unbounded and at 7d, answered at 3d). When the caller set no window,
+      // narrow it step by step rather than hand the caller the 422.
+      const narrowable = v.mode === 'comments' && Boolean(v.query) && !v.after;
+      if (!error.retryable || !narrowable) throw error;
+      let lastError = error;
+      for (const window of ['7d', '3d', '1d']) {
+        await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));
+        try {
+          data = await this.#getOnce(`${this.arcticBaseUrl}${path}?${new URLSearchParams({ ...query, after: window })}`);
+          windowApplied = window;
+          break;
+        } catch (retryError) {
+          lastError = retryError;
+          if (!retryError.retryable) throw retryError;
+        }
+      }
+      if (!data) {
+        throw new Error(
+          `${lastError.message} — Arctic Shift timed out at every window (full history, 7d, 3d, 1d), ` +
+          'so its comment search is under load right now. Retry later, pass a narrower after/before, or scope to one post with link_id.'
+        );
+      }
+      notes.push(`Arctic Shift timed out searching the full history; results are limited to the last ${windowApplied}. Pass after/before to choose the window.`);
+    }
     const rows = Array.isArray(data.data) ? data.data : [];
     const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
     return {
       source: 'arctic_shift', mode: v.mode,
       query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,
       count: results.length, results,
+      ...(windowApplied && { window_applied: windowApplied }),
       notes, checkedAt: new Date().toISOString(),
     };
   }

package/src/tools/templates/ScrapeTemplateTool.js

@@ -11,7 +11,7 @@
  * one call — the registry builds the URL, this tool fetches it.
  */
 
-import { TemplateRegistry } from 'crawlforge-extractors';
+import { TemplateRegistry, retiredTemplate } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
@@ -60,6 +60,12 @@ export class ScrapeTemplateTool {
       }
       const detected = this.registry.detect(url);
       if (!detected) {
+        // A URL a withdrawn template used to handle gets the reason it is
+        // gone, not "no template matches".
+        const retired = retiredTemplate(url);
+        if (retired) {
+          throw badRequest(`The "${retired.id}" template that handled ${url} was retired: ${retired.reason}`);
+        }
         throw badRequest(
           `No template matches ${url}. Pass template:"list" to see every template and the URLs ` +
           'each one handles, or name a template explicitly.'
@@ -74,6 +80,10 @@ export class ScrapeTemplateTool {
     // Validate template exists before making network call
     const tpl = this.registry.get(templateId);
     if (!tpl) {
+      const retired = retiredTemplate(templateId);
+      if (retired) {
+        throw badRequest(`Template "${templateId}" was retired: ${retired.reason}`);
+      }
       const available = this.registry.list().map(t => t.id).join(', ');
       throw new Error(`Unknown template "${templateId}". Available templates: ${available}`);
     }