crawlforge-mcp-server 6.20.0 → 6.21.0

package/src/tools/crawl/mapSite.js

@@ -333,9 +333,10 @@ export class MapSiteTool {
     // Extract all links
     $('a[href]').each((_, element) => {
       const href = $(element).attr('href');
-      if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+      if (href && !href.startsWith('#')) {
         try {
           const absoluteUrl = new URL(href, url);
+          if (['javascript:', 'data:', 'vbscript:'].includes(absoluteUrl.protocol)) return;
           // Only include URLs from the same domain
           if (absoluteUrl.origin === new URL(baseUrl).origin) {
             const urlString = absoluteUrl.toString();

package/src/tools/scrape/unifiedScrape.js

@@ -111,9 +111,11 @@ function extractLinksFromDom($, pageUrl, docBaseUrl) {
     const href = $(el).attr('href');
     const text = $(el).text().trim();
     if (!href) return;
-    if (href.startsWith('#') || href.startsWith('javascript:')) return;
+    if (href.startsWith('#')) return;
     try {
-      const absoluteUrl = new URL(href, resolveBase).toString();
+      const resolved = new URL(href, resolveBase);
+      if (['javascript:', 'data:', 'vbscript:'].includes(resolved.protocol)) return;
+      const absoluteUrl = resolved.toString();
       const isExternal = new URL(absoluteUrl).origin !== pageOrigin;
       if (!seen.has(absoluteUrl)) {
         seen.add(absoluteUrl);

package/src/tools/search/adapters/braveSearch.js

@@ -0,0 +1,160 @@
+/**
+ * Brave Search API Adapter
+ *
+ * Creator Mode's direct search backend when BRAVE_SEARCH_API_KEY is set,
+ * replacing Google Custom Search, which Google discontinues on 2027-01-01.
+ * Production users never reach this: their searches go through the CrawlForge
+ * API, which runs the same Brave mapping server-side
+ * (crawlforge-website src/lib/search/providers/brave.ts).
+ *
+ * Takes the Custom Search parameters SearchWebTool sends and returns Custom
+ * Search's response shape, so the rest of the pipeline is unchanged.
+ *
+ * Checked against the live API on 2026-10-19:
+ * - `count` is at most 20, and `offset` is a PAGE index from 0 to 9.
+ * - `search_lang` and `country` are validated: `zh`, `pt` and `EN` are a 422
+ *   where `zh-hans`, `pt-br` and `en` pass. Common forms are normalised, and a
+ *   locale still refused is retried once without it.
+ * - `description` carries `<strong>` highlighting; there is no total count.
+ */
+
+const ENDPOINT = 'https://api.search.brave.com/res/v1/web/search';
+const MAX_COUNT = 20;
+const MAX_PAGE = 9;
+const TIMEOUT_MS = 15000;
+
+const FRESHNESS = { d1: 'pd', w1: 'pw', m1: 'pm', y1: 'py' };
+
+const LANGUAGE_ALIASES = {
+  zh: 'zh-hans', 'zh-cn': 'zh-hans', 'zh-sg': 'zh-hans',
+  'zh-tw': 'zh-hant', 'zh-hk': 'zh-hant',
+  pt: 'pt-br',
+};
+const REGIONAL_LANGUAGES = new Set(['zh-hans', 'zh-hant', 'pt-br', 'pt-pt', 'en-gb']);
+
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
+
+/** `lang_en`, `lang_zh-CN`, `pt` → the code Brave validates; undefined to send none. */
+export function braveLanguage(lr) {
+  const code = String(lr ?? '').replace(/^lang_/, '').trim().toLowerCase();
+  if (!code) return undefined;
+  if (LANGUAGE_ALIASES[code]) return LANGUAGE_ALIASES[code];
+  if (REGIONAL_LANGUAGES.has(code)) return code;
+  return code.split('-')[0];
+}
+
+/** Brave's highlighted HTML as the plain text Custom Search puts in `snippet`. */
+export function plainText(html) {
+  return String(html ?? '')
+    .replace(/<[^>]*>/g, '')
+    .replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, name) => {
+      if (name[0] === '#') {
+        const code = name[1] === 'x' || name[1] === 'X' ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10);
+        return code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : entity;
+      }
+      return ENTITIES[name.toLowerCase()] ?? entity;
+    })
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function toItem(result) {
+  if (!result?.url) return [];
+  const pagemap = {};
+  if (result.thumbnail?.src) pagemap.cse_thumbnail = [{ src: result.thumbnail.src }];
+  if (result.page_age) pagemap.metatags = [{ 'article:published_time': result.page_age }];
+
+  let displayLink = result.meta_url?.hostname;
+  if (!displayLink) {
+    try { displayLink = new URL(result.url).hostname; } catch { displayLink = ''; }
+  }
+  return [{
+    title: plainText(result.title),
+    link: result.url,
+    snippet: plainText(result.description),
+    displayLink,
+    formattedUrl: result.url,
+    htmlTitle: result.title ?? '',
+    htmlSnippet: result.description ?? '',
+    ...(Object.keys(pagemap).length > 0 ? { pagemap } : {}),
+  }];
+}
+
+export class BraveSearchAdapter {
+  constructor(apiKey) {
+    if (!apiKey) {
+      throw new Error('Brave Search API key is required');
+    }
+    this.apiKey = apiKey;
+  }
+
+  /**
+   * @param {Object} params - Custom Search parameters, as SearchWebTool sends them
+   *   (query, num, start, lr, safe, dateRestrict, cr)
+   * @returns {Promise<Object>} Custom Search-shaped results, plus `provider: 'brave'`
+   */
+  async search(params) {
+    const startedAt = Date.now();
+    const count = Math.min(Math.max(1, params.num || 10), MAX_COUNT);
+    const page = Math.floor(Math.max(0, (params.start || 1) - 1) / count);
+    if (page > MAX_PAGE) return this.toResult([], startedAt);
+
+    const base = new URLSearchParams({
+      q: params.query,
+      count: String(count),
+      safesearch: params.safe === 'active' ? 'strict' : 'off',
+    });
+    if (page > 0) base.append('offset', String(page));
+    if (FRESHNESS[params.dateRestrict]) base.append('freshness', FRESHNESS[params.dateRestrict]);
+
+    const localized = new URLSearchParams(base);
+    const language = braveLanguage(params.lr);
+    const country = String(params.cr ?? '').replace(/^country/, '').trim().toUpperCase();
+    if (language) localized.append('search_lang', language);
+    if (/^[A-Z]{2}$/.test(country)) localized.append('country', country);
+
+    let response = await this.send(localized);
+    if (response.status === 422 && localized.toString() !== base.toString()) {
+      response = await this.send(base);
+    }
+
+    if (!response.ok) {
+      let detail = response.statusText;
+      try { detail = (await response.json())?.error?.detail || detail; } catch { /* no body */ }
+      throw new Error(
+        response.status === 401 || response.status === 403
+          ? 'Invalid Brave Search API key'
+          : response.status === 429
+            ? 'Brave Search API rate limit or credit balance reached. Try again later.'
+            : `Brave Search failed with status ${response.status}: ${detail}`
+      );
+    }
+
+    const data = await response.json();
+    return this.toResult((data?.web?.results ?? []).flatMap(toItem), startedAt);
+  }
+
+  async send(params) {
+    try {
+      return await fetch(`${ENDPOINT}?${params.toString()}`, {
+        method: 'GET',
+        headers: { Accept: 'application/json', 'X-Subscription-Token': this.apiKey },
+        signal: AbortSignal.timeout(TIMEOUT_MS),
+      });
+    } catch (error) {
+      throw new Error(`Network error connecting to Brave Search API: ${error.message}`);
+    }
+  }
+
+  toResult(items, startedAt) {
+    return {
+      items,
+      searchInformation: { totalResults: null, searchTime: (Date.now() - startedAt) / 1000 },
+      queries: {},
+      context: {},
+      provider: 'brave',
+    };
+  }
+}
+
+export default BraveSearchAdapter;

package/src/tools/search/adapters/crawlforgeSearch.js

@@ -1,7 +1,8 @@
 /**
  * CrawlForge Search Adapter
  * 
- * Proxies search requests through CrawlForge.dev API which uses Google Search.
+ * Proxies search requests through the CrawlForge.dev API, which runs them on its
+ * configured backend (Brave Search API since 2026-10) and reports which in `provider`.
  * Users only need their CrawlForge API key - no Google credentials required.
  * 
  * Credit Cost: 5 credits per search (SEARCH_WEB_CREDITS in ../batchSearch.js)
@@ -90,7 +91,9 @@ export class CrawlForgeSearchAdapter {
           searchTime: 0
         },
         queries: data.queries || {},
-        context: data.context || {}
+        context: data.context || {},
+        // 'brave' | 'google'; absent from deployments older than 2026-10-19
+        provider: data.provider
       };
     } catch (error) {
       // Network errors or fetch failures

package/src/tools/search/adapters/searchProviderFactory.js

@@ -2,12 +2,14 @@
  * Search Provider Factory
  *
  * Creates search adapter instances.
- * - Production: Uses CrawlForge proxy for Google Search (users with CrawlForge API key)
- * - Creator Mode: Uses Google Search API directly (requires GOOGLE_API_KEY & GOOGLE_SEARCH_ENGINE_ID)
+ * - Production: Uses the CrawlForge API's web search (users with CrawlForge API key)
+ * - Creator Mode: Uses Brave Search API directly when BRAVE_SEARCH_API_KEY is set, else Google
+ *   Custom Search (GOOGLE_API_KEY & GOOGLE_SEARCH_ENGINE_ID; discontinued by Google on 2027-01-01)
  */
 
 import { CrawlForgeSearchAdapter } from './crawlforgeSearch.js';
 import { GoogleSearchAdapter } from './googleSearch.js';
+import { BraveSearchAdapter } from './braveSearch.js';
 import { SEARCH_WEB_CREDITS } from '../batchSearch.js';
 
 export class SearchProviderFactory {
@@ -19,18 +21,25 @@ export class SearchProviderFactory {
    * @param {boolean} options.creatorMode - Whether Creator Mode is enabled
    * @param {string} options.googleApiKey - Google API key (for Creator Mode)
    * @param {string} options.googleSearchEngineId - Google Search Engine ID (for Creator Mode)
-   * @returns {CrawlForgeSearchAdapter|GoogleSearchAdapter} Search adapter instance
+   * @returns {CrawlForgeSearchAdapter|BraveSearchAdapter|GoogleSearchAdapter} Search adapter instance
    */
   static createAdapter(apiKey, options = {}) {
-    // In Creator Mode without CrawlForge API key, use Google Search API directly
+    // In Creator Mode without CrawlForge API key, call a search API directly:
+    // Brave when its key is set, Google Custom Search until that is switched off.
     if (!apiKey && options.creatorMode) {
+      const braveApiKey = options.braveApiKey || process.env.BRAVE_SEARCH_API_KEY;
+      if (braveApiKey) {
+        console.error('🔍 Creator Mode: Using Brave Search API directly');
+        return new BraveSearchAdapter(braveApiKey);
+      }
+
       const googleApiKey = options.googleApiKey || process.env.GOOGLE_API_KEY;
       const googleSearchEngineId = options.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;
 
       if (!googleApiKey || !googleSearchEngineId) {
         throw new Error(
-          'Creator Mode requires Google Search API credentials. ' +
-          'Set GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID environment variables.\n' +
+          'Creator Mode requires search API credentials. ' +
+          'Set BRAVE_SEARCH_API_KEY, or GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID.\n' +
           'Get credentials at:\n' +
           '  - API Key: https://console.cloud.google.com/apis/credentials\n' +
           '  - Search Engine ID: https://programmablesearchengine.google.com/'
@@ -72,8 +81,9 @@ export class SearchProviderFactory {
     // CrawlForge API key available
     if (apiKey) return true;
 
-    // Creator Mode with Google credentials
+    // Creator Mode with Brave or Google credentials
     if (options.creatorMode) {
+      if (options.braveApiKey || process.env.BRAVE_SEARCH_API_KEY) return true;
       const googleApiKey = options.googleApiKey || process.env.GOOGLE_API_KEY;
       const googleSearchEngineId = options.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;
       return !!(googleApiKey && googleSearchEngineId);
@@ -141,8 +151,8 @@ export class SearchProviderFactory {
       rateLimit: 'Based on your CrawlForge plan',
       creditCost: `${SEARCH_WEB_CREDITS} credits per search`,
       features: [
-        'Google Search results via CrawlForge proxy',
-        'No Google API credentials needed',
+        'Web search results via the CrawlForge API (Brave Search API)',
+        'No search API credentials needed',
         'Full text search',
         'Image metadata',
         'Exact phrase matching',
@@ -154,14 +164,13 @@ export class SearchProviderFactory {
         'Country targeting',
         'Safe search',
         'Localization support',
-        'Related searches',
         'Rich snippets'
       ],
       benefits: [
         'Simplified authentication (one API key)',
         'Unified billing through CrawlForge',
         'Enterprise-grade reliability',
-        'No Google API quota management',
+        'No search API quota management',
         'Built-in rate limiting',
         'Credit-based pricing'
       ]

package/src/tools/search/providers/searxng.js

@@ -9,6 +9,11 @@
  *
  * Result shape is normalised to match the CrawlForge/Google adapter format so
  * the rest of the search pipeline (ranking, deduplication, caching) is unaffected.
+ *
+ * SearXNG's `google` engine scrapes google.com, so it meets Google's
+ * anti-scraping measures: 429s, CAPTCHAs, and decoy result pages that look
+ * valid but are unrelated to the query (seen 2026-10 by SEO rank trackers).
+ * Prefer the `brave` and `duckduckgo` engines in settings.yml.
  */
 
 /**

package/src/tools/search/redditSearch.js

@@ -166,7 +166,7 @@ export class RedditSearchTool {
       order = ['web_discovery'];
     } else if (v.source === 'reddit_api') {
       if (!this.officialConfigured) {
-        throw new Error('source:"reddit_api" needs Reddit app credentials — set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET (create a "script" app at https://www.reddit.com/prefs/apps)');
+        throw new Error('source:"reddit_api" needs Reddit app credentials — set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET. Reddit stopped issuing new app credentials without an approval ticket on 2025-11-11, so this needs an existing app or an approved request (see docs/reddit-access-and-oauth.md). Without it, drop source and auto reads the community archives');
       }
       if (v.mode === 'comments') {
         throw new Error('the official Reddit API has no comment full-text search — use comments mode with source:"arctic_shift" (scoped) or "web_discovery" (unscoped), or read a whole thread with mode:"thread"');

package/src/tools/search/searchWeb.js

@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { SearchProviderFactory } from './adapters/searchProviderFactory.js';
+import { BraveSearchAdapter } from './adapters/braveSearch.js';
 import { CacheManager } from '../../core/cache/CacheManager.js';
 import { QueryExpander } from './queryExpander.js';
 import { ResultRanker } from './ranking/ResultRanker.js';
@@ -8,6 +9,14 @@ import { SearchResultCache } from './ranking/SearchResultCache.js';
 import LocalizationManager from '../../core/LocalizationManager.js';
 import { isCreatorModeVerified } from '../../core/creatorMode.js';
 import { searchViaSearxng } from './providers/searxng.js';
+
+/** The CrawlForge API's `provider` field, as a backend name. */
+const CRAWLFORGE_BACKENDS = { brave: 'Brave Search API', google: 'Google Custom Search API' };
+
+/** The backend the CrawlForge API reported serving a search with. */
+function crawlforgeBackend(provider) {
+  return CRAWLFORGE_BACKENDS[provider] ?? 'CrawlForge web search';
+}
 import { MAX_SEARCH_QUERIES, SEARCH_WEB_CREDITS, EXACTLY_ONE_QUERY_MESSAGE } from './batchSearch.js';
 import { setActualCost } from '../../server/requestContext.js';
 
@@ -405,12 +414,12 @@ export class SearchWebTool {
         // is not repeated here: it is the same on every call, and a
         // queries:[...] batch carried one copy per query (R24 4.4).
         provider: this.isCreatorModeFallback ? {
-          name: 'google',
-          backend: 'Google Custom Search API (Creator Mode)',
-          note: 'Using Google Search API directly. Production users use CrawlForge API.'
+          name: this.creatorBackendName(),
+          backend: `${CRAWLFORGE_BACKENDS[this.creatorBackendName()]} (Creator Mode)`,
+          note: 'Calling the search API directly. Production users use CrawlForge API.'
         } : {
           name: 'crawlforge',
-          backend: 'Google Search'
+          backend: crawlforgeBackend(bestResults?.provider)
         },
         
         // Add localization information
@@ -664,12 +673,12 @@ export class SearchWebTool {
   getStats() {
     return {
       provider: this.isCreatorModeFallback ? {
-        name: 'google',
-        backend: 'Google Custom Search API (Creator Mode)',
-        note: 'Using Google Search API directly'
+        name: this.creatorBackendName(),
+        backend: `${CRAWLFORGE_BACKENDS[this.creatorBackendName()]} (Creator Mode)`,
+        note: 'Calling the search API directly'
       } : {
         name: 'crawlforge',
-        backend: 'Google Search',
+        backend: 'Web search via CrawlForge API',
         capabilities: SearchProviderFactory.getProviderCapabilities('crawlforge')
       },
       creatorMode: this.isCreatorModeFallback || false,
@@ -680,12 +689,17 @@ export class SearchWebTool {
     };
   }
 
+  /** Which search API Creator Mode calls directly: 'brave' or 'google'. */
+  creatorBackendName() {
+    return this.searchAdapter instanceof BraveSearchAdapter ? 'brave' : 'google';
+  }
+
   getProviderInfo() {
     return {
-      activeProvider: this.isCreatorModeFallback ? 'google' : 'crawlforge',
+      activeProvider: this.isCreatorModeFallback ? this.creatorBackendName() : 'crawlforge',
       backend: this.isCreatorModeFallback
-        ? 'Google Custom Search API (Creator Mode)'
-        : 'Google Search via CrawlForge API',
+        ? `${CRAWLFORGE_BACKENDS[this.creatorBackendName()]} (Creator Mode)`
+        : 'Web search via CrawlForge API',
       capabilities: SearchProviderFactory.getProviderCapabilities(
         this.isCreatorModeFallback ? 'google' : 'crawlforge'
       ),

package/src/tools/search/serpRank.js

@@ -2,7 +2,7 @@
  * serp_rank tool
  *
  * Reports where a target domain ranks in Google's REAL organic results for a
- * keyword — the SERP position that Google Custom Search (search_web) cannot
+ * keyword — the SERP position that search_web, a general web search, cannot
  * give you. Backed by the DataForSEO SERP adapter.
  *
  * Honesty rule: if DataForSEO is not configured, this returns an explained
@@ -12,6 +12,46 @@
 import { z } from 'zod';
 import { DataForSEOSearchAdapter } from './adapters/dataforseoSearch.js';
 
+/** Words too common to tell a relevant result from an unrelated one. */
+const STOPWORDS = new Set([
+  'the', 'and', 'for', 'with', 'how', 'what', 'why', 'who', 'when', 'where',
+  'are', 'you', 'your', 'best', 'from', 'near', 'can', 'does', 'that', 'this',
+]);
+
+/** Below this share of top results mentioning the keyword, a SERP is flagged. */
+const DECOY_THRESHOLD = 0.2;
+/** Fewer results than this is too little to judge. */
+const DECOY_MIN_RESULTS = 3;
+
+/** The keyword's distinctive words: lowercased, 3+ characters, stopwords out. */
+function keywordTokens(keyword) {
+  const words = String(keyword).toLowerCase().split(/[^\p{L}\p{N}]+/u);
+  return [...new Set(words.filter((w) => w.length >= 3 && !STOPWORDS.has(w)))];
+}
+
+/**
+ * Whether a SERP looks like a decoy. Google serves some scrapers real-looking
+ * results that have nothing to do with the query, and DataForSEO passes such
+ * a page on as a valid one: on 2026-10-19 "crawlforge" came back as 19
+ * crawl/MCP pages, none naming CrawlForge, while Search Console had the site
+ * at #1 and a second lookup a minute later did too. So check whether the top
+ * results mention the keyword at all.
+ *
+ * @returns {{ checked: number, matching: number, suspect: boolean } | null}
+ *   null when the keyword has no usable word or there are too few results.
+ */
+export function assessSerpRelevance(keyword, results) {
+  const tokens = keywordTokens(keyword);
+  const top = results.slice(0, RESULTS_LIMIT);
+  if (tokens.length === 0 || top.length < DECOY_MIN_RESULTS) return null;
+
+  const matching = top.filter((r) => {
+    const text = `${r.title ?? ''} ${r.url ?? ''} ${r.snippet ?? ''}`.toLowerCase();
+    return tokens.some((t) => text.includes(t));
+  }).length;
+  return { checked: top.length, matching, suspect: matching / top.length < DECOY_THRESHOLD };
+}
+
 /** How many top organic results to return as the SERP listing (`results`).
  * The first Google page is ~10; bounding it keeps the tool payload small while
  * still surfacing the competitors that matter. */
@@ -121,6 +161,9 @@ export class SerpRankTool {
         snippet: it.snippet,
       }));
 
+    const relevance = assessSerpRelevance(validated.keyword, results);
+    const suspect = relevance?.suspect ?? false;
+
     return {
       configured: true,
       keyword: validated.keyword,
@@ -140,6 +183,16 @@ export class SerpRankTool {
       checkUrl: meta.checkUrl, // link to view the real SERP
       cost: meta.cost, // USD charged by DataForSEO for this lookup
       checkedAt: new Date().toISOString(),
+      // true when the top results barely mention the keyword: a likely decoy SERP
+      suspect,
+      ...(suspect
+        ? {
+            warnings: [
+              `These results look unrelated to the keyword (${relevance.matching} of the top ${relevance.checked} mention it). ` +
+                'Google may have served a decoy results page. Re-run the lookup, or check Search Console, before acting on this rank.',
+            ],
+          }
+        : {}),
     };
   }
 }

package/src/utils/ErrorHandlingConfig.js

@@ -38,7 +38,7 @@ export const DEFAULT_CONFIGS = {
     }
   },
 
-  // API operations (Google Search, etc.)
+  // API operations (web search, etc.)
   api: {
     retry: {
       maxRetries: 4,

package/src/utils/urlNormalizer.js

@@ -94,9 +94,10 @@ export function extractLinks(html, baseUrl) {
   while ((match = hrefRegex.exec(html)) !== null) {
     const href = match[1];
     
-    if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+    if (href && !href.startsWith('#')) {
       try {
         const absoluteUrl = new URL(href, baseUrl);
+        if (['javascript:', 'data:', 'vbscript:'].includes(absoluteUrl.protocol)) continue;
         links.add(absoluteUrl.toString());
       } catch {
         // Invalid URL, skip it