crawlforge-mcp-server 6.7.0 → 6.9.0

package/src/utils/challengeDetection.js

@@ -1,10 +1,93 @@
 /**
- * challengeDetection.js — re-export of the challenge-vendor tables.
+ * challengeDetection.js — the challenge-vendor tables, refined for widgets.
  *
  * The tables moved to crawlforge-extractors 1.7.0 so the REST `scrape` route
  * reaches the same verdict as the MCP server (Phase 0, 0.1). Callers keep
  * importing from here.
+ *
+ * Upstream treats a vendor marker on a page under 4000 visible characters as
+ * definitive, which reports a block on a real page that merely embeds a
+ * Turnstile widget: quora.com rendered its real 397-character login page on
+ * both engines and the verdict layer still called it blocked (2026-09-21
+ * benchmark, finding 6). A widget embed and an interstitial are separable by
+ * what else is in the document, not by length — the interstitial carries
+ * Cloudflare's challenge bootstrap or the prose it shows while it holds the
+ * request, and a page that merely embeds the widget carries neither. So the
+ * refinement below suppresses that one case and leaves every other path
+ * upstream owns (a challenge title, Amazon's validateCaptcha form, DataDome,
+ * PerimeterX, Akamai) exactly as it is. Worth upstreaming.
+ */
+
+import { detectChallengePage as detectUpstream } from 'crawlforge-extractors';
+
+// The bootstrap a real interstitial ships and a page embedding a widget never
+// does: Cloudflare's challenge options blob and its cf-chl- element ids,
+// Vercel's checkpoint script and token. Two Cloudflare tokens are deliberately
+// NOT here, because a real page legitimately carries both:
+//   - challenges.cloudflare.com, the widget's own script host;
+//   - cf-chl-widget-*, the hidden response input Turnstile names itself.
+// quora.com's login page carries exactly one cf-chl- token and it is
+// "cf-chl-widget-gaztz_response" (measured 2026-09-21), so a bare cf-chl- test
+// matches every widget embed there is. Hence the lookahead — do not put
+// cf-chl- back.
+const INTERSTITIAL_MARKERS = [
+  { vendor: 'cloudflare', markers: /_cf_chl_opt|window\._cf_chl|cf-chl-(?!widget)|cf_chl_rc_/i, evidence: 'the Cloudflare challenge bootstrap' },
+  { vendor: 'vercel', markers: /_vercel\/challenge|vercel\.link\/security-checkpoint|x-vercel-challenge-token/i, evidence: 'the Vercel Security Checkpoint bootstrap' }
+];
+
+// What an interstitial says to the reader while it holds the request.
+const INTERSTITIAL_PROSE = /verifying you are human|checking your browser|needs to review the security of your connection|enable javascript and cookies to continue/i;
+
+// The floor for "this document has a real body". quora.com's real login page
+// is 397 visible characters, so the floor has to sit below that; Cloudflare's
+// own interstitial renders well under 200 characters of prose ("Verifying you
+// are human…", "Enable JavaScript and cookies to continue"), so a genuinely
+// empty wall carrying only a widget marker is still short enough to be caught.
+const REAL_BODY_MIN_CHARS = 200;
+
+/**
+ * Whether a document is a challenge wall rather than the page: a known
+ * challenge title OR a challenge bootstrap marker, regardless of the title and
+ * regardless of page length, so a custom-titled interstitial (nowsecure.nl,
+ * whose title is just "nowsecure.nl") is recognised too.
+ *
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function looksLikeInterstitial(page = {}) {
+  const { title = '', html = '' } = page;
+  // A title on its own reaches only upstream's title branch — its marker
+  // branch needs html and its definitive branch a form — so the challenge
+  // titles stay in one place rather than being copied here.
+  const byTitle = detectUpstream({ title });
+  if (byTitle) return byTitle;
+  for (const { vendor, markers, evidence } of INTERSTITIAL_MARKERS) {
+    if (markers.test(html)) return { vendor, evidence };
+  }
+  return null;
+}
+
+/** A title, a body a reader could read, and none of the interstitial's prose. */
+function looksLikeRealPage({ title = '', text = '' }) {
+  if (!String(title).trim()) return false;
+  const visible = String(text).replace(/\s+/g, ' ').trim();
+  if (visible.length < REAL_BODY_MIN_CHARS) return false;
+  return !INTERSTITIAL_PROSE.test(`${title} ${visible}`);
+}
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
  */
+export function detectChallengePage(page = {}) {
+  const hit = detectUpstream(page);
+  // Cloudflare is the only vendor whose markers a real page can carry, so it is
+  // the only one refined. Reaching here with a cloudflare hit that is not an
+  // interstitial means the evidence was the widget's own script host or
+  // response input on a short page.
+  if (!hit || hit.vendor !== 'cloudflare') return hit;
+  if (looksLikeInterstitial(page)) return hit;
+  return looksLikeRealPage(page) ? null : hit;
+}
 
-export { detectChallengePage } from 'crawlforge-extractors';
-export { detectChallengePage as default } from 'crawlforge-extractors';
+export default detectChallengePage;

package/src/utils/stealthVerdict.js

@@ -1,10 +1,44 @@
 /**
- * stealthVerdict.js — re-export of the document verdict.
+ * stealthVerdict.js — the document verdict, with the widget refinement.
  *
  * `stealthDocumentVerdict` moved to crawlforge-extractors 1.7.0 as
  * `documentVerdict` — the same behaviour and messages for the stealth callers,
  * plus a `fetcher` option so the plain `scrape` path on both surfaces reaches
  * the same verdict (Phase 0, 0.1). Callers keep importing from here.
+ *
+ * Upstream decides the block before every other rule it owns, so the widget
+ * refinement in challengeDetection.js would never reach the verdict the tools
+ * and the benchmark actually print. The wrapper below re-runs upstream on the
+ * same document with the widget marker neutralised whenever the refinement
+ * clears the block, so an HTTP error page, an empty shell or a short
+ * error-titled placeholder that happens to embed a widget is still named by
+ * upstream and still decides (Phase 1, finding 6).
  */
 
-export { documentVerdict as stealthDocumentVerdict, SOFT_ERROR_MAX_CHARS } from 'crawlforge-extractors';
+import { documentVerdict, SOFT_ERROR_MAX_CHARS } from 'crawlforge-extractors';
+import { detectChallengePage } from './challengeDetection.js';
+
+export { SOFT_ERROR_MAX_CHARS };
+
+// The two Turnstile markers challengeDetection.js clears: the widget's script
+// host, and the hidden response input the widget names itself
+// (id="cf-chl-widget-<id>_response" — quora.com, measured 2026-09-21).
+// Upstream's Cloudflare pattern matches a bare cf-chl-, so neutralising the
+// host alone left the re-run blocking the same document a second time. Both
+// are swapped for inert text; the bootstrap markers are never touched, so a
+// document that carries one is still upstream's block to make.
+const TURNSTILE_WIDGET = /challenges\.cloudflare\.com|cf-chl-widget/gi;
+
+/**
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean, fetcher?: string, rendered?: boolean, contentReturned?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function stealthDocumentVerdict(scraped, options) {
+  const verdict = documentVerdict(scraped, options);
+  if (!verdict.blocked || detectChallengePage(scraped || {})) return verdict;
+  // The neutralised copy is read by upstream and discarded; the caller's own
+  // document is never rewritten.
+  const html = String(scraped?.html || '').replace(TURNSTILE_WIDGET, 'turnstile-widget');
+  return documentVerdict({ ...scraped, html }, options);
+}