crawlforge-mcp-server 6.11.0 → 6.12.0

package/CLAUDE.md

@@ -236,6 +236,11 @@ CRAWLFORGE_STEALTH_PROXIES=          # comma-separated proxy URLs used by the es
                                      # from the PROXY_ROTATION_* family, which belongs to
                                      # localization. The agent's automatic stealth retry
                                      # (stealth review Phase 3) reads it too.
+CRAWLFORGE_IMPIT=                    # `off` skips escalation's impit step (a Chrome TLS handshake with
+                                     # the honest UA, tried before the browser; stealth review Phase 7).
+                                     # Exit-IP dependent like the engine: it cleared indeed.com from a
+                                     # residential IP and nothing from the hosted datacenter IP, so
+                                     # render.yaml sets it off there. Measure with scripts/impit-probe.mjs.
 CRAWLFORGE_CLEARANCE_JAR=            # `off` disables the clearance jar (stealth review Phase 4):
                                      # cf_clearance/__cf_bm/datadome cookies a stealth render earned
                                      # are replayed to the next stealth context with the same engine,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.11.0",
+  "version": "6.12.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.9.0",
+    "crawlforge-extractors": "^1.10.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -61,7 +61,7 @@ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
 import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
-import { loadImpit, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
+import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -110,7 +110,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.11.0",
+  version: "6.12.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -259,7 +259,8 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
   // honest User-Agent is tried before any browser launches. A caller who
   // named an engine asked for that browser, so it is not tried for them.
-  if (engine === 'auto' && await loadImpit()) {
+  // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
+  if (engine === 'auto' && impitEnabled() && await loadImpit()) {
     recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
     const page = await impitFetchPage(url);
     if (page) return { ...page, warnings };

package/src/utils/impitRung.js

@@ -48,6 +48,19 @@ const MIN_UNRENDERED_TEXT_CHARS = 200;
 
 let impitClass; // undefined: not tried yet; null: unavailable
 
+/**
+ * Whether this deployment runs the step. `CRAWLFORGE_IMPIT=off` turns it off.
+ * Like the stealth engine, what it can clear depends on the exit IP: from a
+ * residential IP it cleared indeed.com, and from the hosted instance's
+ * datacenter IP (AS14618, Ashburn) it cleared nothing in 3 runs (2026-09-27),
+ * so render.yaml sets it off there. Read on every call, like the stealth
+ * proxy list, so an operator can change it without a restart.
+ * @returns {boolean}
+ */
+export function impitEnabled() {
+  return String(process.env.CRAWLFORGE_IMPIT || '').trim().toLowerCase() !== 'off';
+}
+
 /**
  * The Impit class, or null when the optional dependency is absent or its
  * native bindings do not load on this platform. Tried once per process.