crawlforge-mcp-server 5.2.2 → 5.2.3

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.2
+**Current Version:** 5.2.3
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.2",
+  "version": "5.2.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -113,7 +113,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.1.0",
+    "crawlforge-extractors": "^1.2.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.2",
+  version: "5.2.3",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -185,9 +185,13 @@ const searchWebTool = new SearchWebTool(searchWebToolConfig);
 // separate from CrawlForge billing — no getToolConfig needed. Degrades gracefully
 // when unconfigured (returns { configured: false } instead of throwing).
 const serpRankTool = new SerpRankTool();
-// reddit_search queries free community archives (Arctic Shift / PullPush) —
-// no credentials, no getToolConfig needed. reddit.com itself blocks scrapers.
-const redditSearchTool = new RedditSearchTool();
+// reddit_search reads the free Arctic Shift community archive — reddit.com
+// itself blocks scrapers. Arctic Shift cannot keyword-search across all of
+// Reddit, so a Reddit-wide search discovers posts through the same search
+// provider search_web uses, then reads those posts from the archive.
+const redditSearchTool = new RedditSearchTool({
+  searchApiKey: searchWebToolConfig.apiKey,
+});
 const crawlDeepTool = new CrawlDeepTool(getToolConfig('crawl_deep'));
 const mapSiteTool = new MapSiteTool(getToolConfig('map_site'));
 const extractContentTool = new ExtractContentTool();
@@ -450,7 +454,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this queries the Arctic Shift and PullPush community archives instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. Keyword search across ALL of Reddit routes to PullPush; subreddit/author-scoped searches use Arctic Shift (near-real-time) with PullPush fallback. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search has no backend — scope it or search posts. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -1402,7 +1406,7 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: shopify-product (any Shopify storefront, read from the store's own /products/<handle>.json rather than the rendered page), amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package. Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
+  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: shopify-product (any Shopify storefront, read from the store's own /products/<handle>.json rather than the rendered page), amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package (read from the npm registry API rather than the npmjs.com page, which blocks plain fetches). Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     template: z.string().describe("Template ID (e.g. github-repo) or list to enumerate available templates"),

package/src/tools/search/redditSearch.js

@@ -38,6 +38,7 @@ import {
   stripNamePrefix,
 } from './redditNormalize.js';
 import { RedditOfficialApiAdapter } from './adapters/redditOfficialApi.js';
+import { SearchProviderFactory } from './adapters/searchProviderFactory.js';
 
 const ARCTIC_SHIFT_BASE = 'https://arctic-shift.photon-reddit.com';
 const PULLPUSH_BASE = 'https://api.pullpush.io';
@@ -59,7 +60,7 @@ const RedditSearchSchema = z.object({
   before: z.string().min(1).optional(),
   limit: z.number().int().min(1).max(100).optional().default(25),
   sort: z.enum(['asc', 'desc']).optional().default('desc'),
-  source: z.enum(['auto', 'arctic_shift', 'pullpush', 'reddit_api']).optional().default('auto'),
+  source: z.enum(['auto', 'arctic_shift', 'pullpush', 'reddit_api', 'web_discovery']).optional().default('auto'),
 });
 
 /**
@@ -94,6 +95,20 @@ export class RedditSearchTool {
     this.officialConfigured = Boolean(this.redditClientId && this.redditClientSecret);
     // Lazily constructed on first official-path use; overridable for tests.
     this._officialAdapter = options.officialAdapter || null;
+
+    // Web discovery serves the one shape no archive can: a keyword search
+    // across all of Reddit. Arctic Shift requires a subreddit or author scope,
+    // and PullPush stopped serving automated clients in August 2026.
+    this.searchAdapter = options.searchAdapter || null;
+    this.searchApiKey = options.searchApiKey || null;
+  }
+
+  /** The web-search adapter used to discover posts, built once on first use. */
+  #search() {
+    if (!this.searchAdapter) {
+      this.searchAdapter = SearchProviderFactory.createAdapter(this.searchApiKey);
+    }
+    return this.searchAdapter;
   }
 
   /** The official-API adapter, built once from the configured credentials. */
@@ -127,8 +142,17 @@ export class RedditSearchTool {
     // comment full-text search, so `comments` mode always uses the archives.
     const officialPossible = this.officialConfigured && (v.mode === 'thread' || v.mode === 'posts');
 
+    // A Reddit-wide keyword search for posts can be discovered through a web
+    // search and then read out of the archive by ID.
+    const discoveryPossible = v.mode === 'posts' && Boolean(v.query) && !subreddit && !author;
+
     let order; // backends to try, in order
-    if (v.source === 'reddit_api') {
+    if (v.source === 'web_discovery') {
+      if (!discoveryPossible) {
+        throw new Error('web_discovery only serves unscoped keyword searches in posts mode — it finds posts through a web search and then reads them from the archive');
+      }
+      order = ['web_discovery'];
+    } else if (v.source === 'reddit_api') {
       if (!this.officialConfigured) {
         throw new Error('source:"reddit_api" needs Reddit app credentials — set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET (create a "script" app at https://www.reddit.com/prefs/apps)');
       }
@@ -147,12 +171,24 @@ export class RedditSearchTool {
     } else {
       // auto: prefer the user's own official API (live, authoritative) when it
       // can serve this request, then fall back to the community archives.
+      // PullPush is no longer tried automatically: every request now returns
+      // 429 "This website does not provide free scraping resources for agents",
+      // or a Cloudflare 403 challenge. It stays available on explicit request.
       const archives = v.mode === 'thread' ? ['arctic_shift']
-        : arcticPossible ? ['arctic_shift', 'pullpush']
-        : ['pullpush'];
+        : arcticPossible ? ['arctic_shift']
+        : discoveryPossible ? ['web_discovery']
+        : [];
       order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
 
+    if (order.length === 0) {
+      // Unscoped comment search was PullPush-only, and PullPush no longer
+      // serves automated clients. Web discovery identifies posts, not comments.
+      throw new Error(
+        `An unscoped keyword search of ${v.mode} has no available backend: Arctic Shift requires a subreddit or author scope, and PullPush no longer serves automated clients. Add a subreddit or author filter, or search mode:"posts", which finds posts through a web search and reads them from the archive.`
+      );
+    }
+
     const errors = [];
     for (const source of order) {
       try {
@@ -161,6 +197,8 @@ export class RedditSearchTool {
           result = v.mode === 'thread'
             ? await this.#official().getThread(v)
             : await this.#official().searchPosts(v, { subreddit, author });
+        } else if (source === 'web_discovery') {
+          result = await this.#searchWebDiscovery(v);
         } else if (source === 'arctic_shift') {
           result = await this.#searchArcticShift(v, { subreddit, author });
         } else {
@@ -172,14 +210,78 @@ export class RedditSearchTool {
         errors.push(`${source}: ${error.message}`);
       }
     }
-    // Unscoped keyword searches have no Arctic Shift fallback (it requires a
-    // scope — verified live: HTTP 400 without one), so point at the fix.
-    const hint = order.length === 1 && order[0] === 'pullpush' && v.source === 'auto'
-      ? ' Tip: add a subreddit or author filter to route to the more reliable Arctic Shift archive.'
+    // Arctic Shift requires a scope (verified live: HTTP 400 without one), so an
+    // unscoped query that got this far has exhausted its only route.
+    const hint = order.includes('web_discovery')
+      ? ' Tip: add a subreddit or author filter to search the Arctic Shift archive directly.'
       : '';
     throw new Error(`All Reddit sources failed — ${errors.join('; ')}.${hint}`);
   }
 
+  /**
+   * Reddit-wide keyword search, in two steps: find matching posts with a
+   * site-restricted web search, then read those posts out of the Arctic Shift
+   * archive by ID. Arctic Shift cannot keyword-search across all of Reddit and
+   * reddit.com blocks scrapers, so discovery has to come from somewhere else —
+   * but what comes back are real archive rows, the same shape a scoped search
+   * returns, not scraped search-engine snippets.
+   */
+  async #searchWebDiscovery(v) {
+    let found;
+    try {
+      found = await this.#search().search({
+        query: `site:reddit.com ${v.query}`,
+        num: Math.min(Math.max(v.limit, 1), 10), // one call caps at 10 results
+        start: 1,
+      });
+    } catch (error) {
+      throw new Error(`web search failed: ${error.message}`);
+    }
+
+    // /r/<sub>/comments/<id>/<slug> is the only Reddit URL shape naming a post.
+    const ids = [];
+    for (const item of found?.items ?? []) {
+      const id = /\/comments\/([a-z0-9]+)/i.exec(item?.link ?? item?.url ?? '')?.[1];
+      if (id && !ids.includes(id)) ids.push(id);
+    }
+
+    const notes = [
+      'Reddit-wide keyword search: posts were found with a site-restricted web search, then read from the Arctic Shift archive by ID.',
+      'Results are ordered by web-search relevance, not by score or date.',
+      'Arctic Shift cannot keyword-search across all of Reddit, and PullPush no longer serves automated clients — scope the search to a subreddit or author to query the archive directly.',
+    ];
+    if (v.after || v.before) {
+      // Silently dropping a date filter would return results the caller
+      // believes were filtered.
+      notes.push('after/before were NOT applied: discovery runs through a web search, which cannot filter by post date. Scope the search to a subreddit or author to use date filters.');
+    }
+
+    if (ids.length === 0) {
+      return {
+        source: 'web_discovery', mode: 'posts',
+        query: v.query ?? null, subreddit: null, author: null,
+        count: 0, results: [],
+        notes, checkedAt: new Date().toISOString(),
+      };
+    }
+
+    const data = await this.#get(`${this.arcticBaseUrl}/api/posts/ids`, {
+      ids: ids.slice(0, v.limit).join(','),
+    });
+    const rows = Array.isArray(data.data) ? data.data : [];
+    // Restore the web-search ordering; the archive answers in its own order.
+    const byId = new Map(rows.map(row => [stripIdPrefix(String(row.id ?? '')), row]));
+    const ordered = ids.map(id => byId.get(id)).filter(Boolean);
+
+    return {
+      source: 'web_discovery', mode: 'posts',
+      query: v.query ?? null, subreddit: null, author: null,
+      count: ordered.length, results: ordered.map(normalizePost),
+      discovered: ids.length,
+      notes, checkedAt: new Date().toISOString(),
+    };
+  }
+
   async #searchArcticShift(v, { subreddit, author }) {
     const notes = [
       'Data from the Arctic Shift community archive (arctic-shift.photon-reddit.com), not reddit.com (which blocks scrapers).',