crawlforge-mcp-server 6.19.1 → 6.20.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.19.1",
+  "version": "6.20.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.19.1",
+  version: "6.20.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -597,7 +597,7 @@ registerToolIfEnabled("serp_rank", {
     location_code: z.number().optional().describe("Numeric DataForSEO location code (overrides location_name)"),
     language_code: z.string().optional().describe("Language code (e.g. 'en')"),
     device: z.enum(["desktop", "mobile"]).optional().describe("Device to emulate"),
-    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 20; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
+    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 30; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
   },
   outputSchema: OUTPUT_SCHEMAS.serp_rank
 }, withAuth("serp_rank", async ({ keyword, target, location_name, location_code, language_code, device, depth }) => {

package/src/core/ActionExecutor.js

@@ -8,12 +8,16 @@ import BrowserProcessor from './processing/BrowserProcessor.js';
 import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
 import { assertUrlAllowed, assertNavigationAllowed } from '../utils/ssrfGuard.js';
-import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode } from '../utils/robotsGate.js';
+import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode, paceBrowserAction } from '../utils/robotsGate.js';
 import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
 import { settlePage } from './browser/settle.js';
 import { handleConsent } from './browser/consent.js';
 import { stealthDocumentVerdict } from '../utils/stealthVerdict.js';
 
+// Actions that can make the page send a request (follow a link, submit a form,
+// trigger a load): each waits out the page host's Crawl-delay first.
+const REQUESTING_ACTION_TYPES = new Set(['click', 'press', 'select', 'check']);
+
 // executeJavaScript hardening limits (only relevant when the deploy-time flag
 // ALLOW_JAVASCRIPT_EXECUTION=true is set; JS execution stays off by default).
 const JS_MAX_SCRIPT_LENGTH = parseInt(process.env.JS_MAX_SCRIPT_LENGTH || '10000', 10);
@@ -799,6 +803,9 @@ export class ActionExecutor extends EventEmitter {
     // A navigate leaves the page it is on and gates the URL it is sent to.
     if (action.type !== 'navigate') await this.assertPageAllowed(page, browserOptions);
     const urlBefore = page.url();
+    // A navigate waits its turn in its own pre-flight; these can send the page
+    // somewhere too, and the request goes out before anything can ask.
+    if (REQUESTING_ACTION_TYPES.has(action.type)) await paceBrowserAction(urlBefore);
     const actionResult = await this.executeActionInternal(page, action, executionContext);
     await this.assertPageAllowed(page, browserOptions);
     // A click that followed a link or a submit that posted a form is a

package/src/core/AuthManager.js

@@ -11,7 +11,7 @@ import { randomUUID } from 'crypto';
 import { isCreatorModeVerified } from './creatorMode.js';
 import { resolveApiEndpoint } from './endpointGuard.js';
 import { logger } from '../utils/Logger.js';
-import { maskSecrets } from '../utils/secretMask.js';
+import { maskSecrets, maskLocalPaths } from '../utils/secretMask.js';
 import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
 import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
 import { agentEscalationSurcharge } from '../tools/agent/escalation.js';
@@ -385,7 +385,8 @@ class AuthManager {
       creditsUsed,
       // Never send raw tool params to the backend — they can carry third-party
       // API keys, auth headers, or webhook secrets. Mask before it leaves the process.
-      requestData: maskSecrets(requestData),
+      // Local file paths too: they name the customer's clients and folders.
+      requestData: maskLocalPaths(maskSecrets(requestData)),
       responseStatus,
       processingTime,
       timestamp: new Date().toISOString(),

package/src/tools/scrape/formats.js

@@ -51,6 +51,23 @@ export const FormatSchema = z.union([
 
 const QUERY_FORMATS = new Set(['highlights', 'question']);
 
+/**
+ * True when `formats` holds at most one `highlights` and one `question`. The
+ * result has one slot of each, so a second entry overwrote the first while
+ * the add-on still charged; the schema refuses it instead.
+ *
+ * @param {Array<unknown>} formats
+ * @returns {boolean}
+ */
+export function oneOfEachQueryFormat(formats) {
+  return [...QUERY_FORMATS].every(
+    (type) => formats.filter((fmt) => fmt && typeof fmt === 'object' && fmt.type === type).length <= 1
+  );
+}
+
+export const ONE_OF_EACH_QUERY_FORMAT_MESSAGE =
+  "At most one 'highlights' and one 'question' format per call; make another call for another query";
+
 /**
  * What the query-scoped formats add to `scrape`'s base price: 1 credit once
  * per call when any `highlights` or `question` format is present, 3 more

package/src/tools/scrape/unifiedScrape.js

@@ -19,6 +19,7 @@ import { segmentUnits, rankUnits } from 'crawlforge-extractors';
 import { stealthDocumentVerdict as documentVerdict } from '../../utils/stealthVerdict.js';
 import {
   SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema,
+  oneOfEachQueryFormat, ONE_OF_EACH_QUERY_FORMAT_MESSAGE,
   scrapeFormatSurcharge
 } from './formats.js';
 import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from './escalation.js';
@@ -57,7 +58,9 @@ const isQueryFormat = (fmt) => Boolean(fmt) && typeof fmt === 'object' && (fmt.t
 // keeping its own copy (Phase 0, 0.3).
 export const SCRAPE_INPUT_SHAPE = {
   url: z.string().url().describe('The URL to scrape'),
-  formats: z.array(FormatSchema).min(1).optional().default(['markdown']).describe('Formats to return (default: ["markdown"])'),
+  formats: z.array(FormatSchema).min(1)
+    .refine(oneOfEachQueryFormat, ONE_OF_EACH_QUERY_FORMAT_MESSAGE)
+    .optional().default(['markdown']).describe('Formats to return (default: ["markdown"]); at most one highlights and one question format per call'),
   onlyMainContent: z.boolean().optional().default(true).describe('Strip boilerplate via Readability (default: true)'),
   // Pass-through to fetchAndParse
   timeoutMs: z.number().min(1000).max(60000).optional().default(15000).describe('Fetch timeout in ms'),

package/src/tools/search/adapters/dataforseoSearch.js

@@ -12,8 +12,8 @@
  *   POST https://api.dataforseo.com/v3/serp/google/organic/live/advanced
  *
  * Cost: US$0.002 per 10 results of `depth` on Live Advanced — measured live,
- * not estimated: depth 10 bills $0.002, the depth 20 default bills $0.004, and
- * depth 100 bills $0.02. Deeper scans are also slower (see the timeout note
+ * not estimated: depth 10 bills $0.002, depth 20 bills $0.004, and depth 100
+ * bills $0.02, so the depth 30 default is $0.006. Deeper scans are also slower (see the timeout note
  * below), so raise `depth` only when a rank below the default is worth paying
  * for. For high-volume scheduled tracking, DataForSEO's task-based "Standard"
  * queue (task_post → tasks_ready → task_get) is cheaper; swap the endpoint +
@@ -51,7 +51,7 @@ export class DataForSEOSearchAdapter {
    * @param {number} [params.locationCode] - Numeric DataForSEO location code (overrides locationName)
    * @param {string} [params.languageCode='en'] - Language code
    * @param {('desktop'|'mobile')} [params.device='desktop'] - Device to emulate
-   * @param {number} [params.depth=20] - How many results to scan (billed per 10)
+   * @param {number} [params.depth=30] - How many results to scan (billed per 10)
    * @returns {Promise<{items: Array<Object>, meta: Object}>} Normalized organic results + metadata
    */
   async search(params) {
@@ -61,7 +61,7 @@ export class DataForSEOSearchAdapter {
       locationCode,
       languageCode = 'en',
       device = 'desktop',
-      depth = 20,
+      depth = 30,
     } = params;
 
     if (!keyword) {

package/src/tools/search/serpRank.js

@@ -27,7 +27,7 @@ const SerpRankSchema = z.object({
   // DataForSEO bills per 10 results scanned and gets slower the deeper it goes,
   // so the default stays shallow: 20 covers Google's first two pages for $0.004.
   // Raise it (max 200, their cap) when a deeper position is worth the spend.
-  depth: z.number().int().min(10).max(200).optional().default(20),
+  depth: z.number().int().min(10).max(200).optional().default(30),
 });
 
 /** Reduce a domain or URL to a bare, comparable host: "https://www.Example.com/x" → "example.com".

package/src/utils/hostRateLimiter.js

@@ -175,9 +175,11 @@ export async function throttleHost(url, options = {}) {
       waits.push(state.lastRequestAt + crawlDelayMs - Date.now());
     }
     const wait = Math.max(0, ...waits);
+    // Recorded before the sleep, as the time this request will go out: a
+    // concurrent caller then queues a delay behind it instead of reading the
+    // same lastRequestAt and waking at the same moment.
+    state.lastRequestAt = Date.now() + wait;
     if (wait > 0) await sleep(wait);
-
-    state.lastRequestAt = Date.now();
   }
 
   if (config.rateLimit.perDomain === false) return; // our own throttle disabled

package/src/utils/robotsGate.js

@@ -362,6 +362,33 @@ export async function browserPreflight(url, options = {}) {
   return [...decision.warnings, ...crawlDelayWarning(url, decision.crawlDelayMs)];
 }
 
+/**
+ * Wait the page's host out before a browser action that can send a request:
+ * a click on a link, a submit, an Enter press, a select or checkbox wired to
+ * a load. The browser makes that request the moment the action runs and the
+ * gate only sees where it led afterwards (`alreadyRequested`), so the wait
+ * has to come first. Spaced by the CURRENT page's host — where the action will
+ * lead is not known until it has run.
+ *
+ * Reads Crawl-delay from the cached robots.txt with no allow/deny decision
+ * and no audit row: the page itself already passed the gate, and an override
+ * is recorded once per URL, not once per click. An action that sends nothing
+ * still takes a slot; overspacing a polite crawl is the safe way to be wrong.
+ *
+ * @param {string} url the page's current URL
+ * @returns {Promise<void>}
+ */
+export async function paceBrowserAction(url) {
+  if (!/^https?:/i.test(url || '')) return;
+  let crawlDelayMs = 0;
+  try {
+    crawlDelayMs = (await checkerFor(resolveUserAgent()).fetchCrawlDelay(url)) * 1000;
+  } catch {
+    // Unreadable robots.txt asks for no delay (see robotsPreflight).
+  }
+  await throttleHost(url, { crawlDelayMs });
+}
+
 /** Test/diagnostic hook: drop every cached robots.txt. */
 export function _resetRobotsGate() {
   checkers.clear();

package/src/utils/secretMask.js

@@ -81,6 +81,43 @@ export function maskSecrets(obj, depth = 0) {
   return obj;
 }
 
+// A string that names a place on the customer's disk: home-relative, a file
+// URL, a Windows drive or UNC share, or an absolute POSIX path under a root
+// that holds user files. A bare "/" or a URL path like "/pricing" is not one.
+const LOCAL_PATH_RE = /^(?:~[\\/]|file:\/\/|[A-Za-z]:[\\/]|\\\\|\/(?:Users|home|root|tmp|private|var|mnt|Volumes|media|opt|srv)\/)/;
+// process_document reads `source` from disk for these, relative paths included.
+const LOCAL_SOURCE_TYPES = new Set(['file', 'pdf_file']);
+
+/** "[local file]" plus the extension, so usage stays countable by file type. */
+function maskPath(value) {
+  const ext = /\.([A-Za-z0-9]{1,8})$/.exec(value.split(/[\\/]/).pop() || '');
+  return ext ? `[local file].${ext[1].toLowerCase()}` : '[local file]';
+}
+
+/**
+ * Deep-clone obj with every local file path reduced to its extension. For the
+ * usage report, which leaves the machine: a path names the customer's clients
+ * and folders, and maskSecrets matches key names, so it never saw one. Local
+ * logs keep the path. Does NOT mutate the original.
+ * @param {*} obj
+ * @param {number} depth - internal recursion guard
+ * @returns {*}
+ */
+export function maskLocalPaths(obj, depth = 0) {
+  if (depth > 10) return obj;
+  if (typeof obj === 'string') return LOCAL_PATH_RE.test(obj) ? maskPath(obj) : obj;
+  if (Array.isArray(obj)) return obj.map(item => maskLocalPaths(item, depth + 1));
+  if (obj !== null && typeof obj === 'object') {
+    const localSource = LOCAL_SOURCE_TYPES.has(obj.sourceType) && typeof obj.source === 'string';
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = key === 'source' && localSource ? maskPath(value) : maskLocalPaths(value, depth + 1);
+    }
+    return result;
+  }
+  return obj;
+}
+
 /**
  * Redact secrets from an Error's message and stack.
  * Returns a new plain-object representation safe for logging.