crawlforge-mcp-server 6.19.0 → 6.19.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.19.0",
+  "version": "6.19.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.19.0",
+  version: "6.19.2",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -597,7 +597,7 @@ registerToolIfEnabled("serp_rank", {
     location_code: z.number().optional().describe("Numeric DataForSEO location code (overrides location_name)"),
     language_code: z.string().optional().describe("Language code (e.g. 'en')"),
     device: z.enum(["desktop", "mobile"]).optional().describe("Device to emulate"),
-    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 20; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
+    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 30; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
   },
   outputSchema: OUTPUT_SCHEMAS.serp_rank
 }, withAuth("serp_rank", async ({ keyword, target, location_name, location_code, language_code, device, depth }) => {

package/src/core/LocalizationManager.js

@@ -399,11 +399,15 @@ export class LocalizationManager extends EventEmitter {
       // Configure geolocation
       geolocation: config.geoLocation || this.getCountryCoordinates(targetCountry) || undefined,
       
-      // Set HTTP headers
+      // Set HTTP headers. A context's extraHTTPHeaders go on every request, so
+      // each must be one CORS ignores (a forbidden name: Accept-Encoding, DNT)
+      // or safelisted (Accept-Language). Cache-Control is neither: it turned
+      // every cross-origin CORS fetch into a preflighted one, which a server
+      // answering no OPTIONS refuses — the stealth headers broke Cloudflare's
+      // Turnstile the same way (R24).
       extraHTTPHeaders: {
         'Accept-Language': config.acceptLanguage,
         'Accept-Encoding': 'gzip, deflate, br',
-        'Cache-Control': 'no-cache',
         'DNT': '1',
         ...config.customHeaders,
         ...browserOptions.extraHTTPHeaders
@@ -963,7 +967,6 @@ export class LocalizationManager extends EventEmitter {
     return {
       "Accept-Language": this.generateAcceptLanguageHeader(config.language),
       "Accept-Encoding": "gzip, deflate, br",
-      "Cache-Control": "no-cache",
       "DNT": "1"
     };
   }

package/src/tools/search/adapters/dataforseoSearch.js

@@ -12,8 +12,8 @@
  *   POST https://api.dataforseo.com/v3/serp/google/organic/live/advanced
  *
  * Cost: US$0.002 per 10 results of `depth` on Live Advanced — measured live,
- * not estimated: depth 10 bills $0.002, the depth 20 default bills $0.004, and
- * depth 100 bills $0.02. Deeper scans are also slower (see the timeout note
+ * not estimated: depth 10 bills $0.002, depth 20 bills $0.004, and depth 100
+ * bills $0.02, so the depth 30 default is $0.006. Deeper scans are also slower (see the timeout note
  * below), so raise `depth` only when a rank below the default is worth paying
  * for. For high-volume scheduled tracking, DataForSEO's task-based "Standard"
  * queue (task_post → tasks_ready → task_get) is cheaper; swap the endpoint +
@@ -51,7 +51,7 @@ export class DataForSEOSearchAdapter {
    * @param {number} [params.locationCode] - Numeric DataForSEO location code (overrides locationName)
    * @param {string} [params.languageCode='en'] - Language code
    * @param {('desktop'|'mobile')} [params.device='desktop'] - Device to emulate
-   * @param {number} [params.depth=20] - How many results to scan (billed per 10)
+   * @param {number} [params.depth=30] - How many results to scan (billed per 10)
    * @returns {Promise<{items: Array<Object>, meta: Object}>} Normalized organic results + metadata
    */
   async search(params) {
@@ -61,7 +61,7 @@ export class DataForSEOSearchAdapter {
       locationCode,
       languageCode = 'en',
       device = 'desktop',
-      depth = 20,
+      depth = 30,
     } = params;
 
     if (!keyword) {

package/src/tools/search/serpRank.js

@@ -27,7 +27,7 @@ const SerpRankSchema = z.object({
   // DataForSEO bills per 10 results scanned and gets slower the deeper it goes,
   // so the default stays shallow: 20 covers Google's first two pages for $0.004.
   // Raise it (max 200, their cap) when a deeper position is worth the spend.
-  depth: z.number().int().min(10).max(200).optional().default(20),
+  depth: z.number().int().min(10).max(200).optional().default(30),
 });
 
 /** Reduce a domain or URL to a bare, comparable host: "https://www.Example.com/x" → "example.com".