crawlforge-extractors 1.11.0 → 1.12.0

package/README.md

@@ -202,7 +202,7 @@ URL is reported in `reason` as a retired handle.
 ### Recognising a blocked page
 
 `documentVerdict` says what a fetched document is: the page, a bot-wall
-interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel), an
+interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF), an
 HTTP error page, an empty shell, a short error-titled placeholder, or a
 client-rendered app's own short error fallback under a normal title (quora.com's
 "Something went wrong. Wait a moment and try again."). A wall

package/index.d.ts

@@ -308,7 +308,7 @@ export default TemplateRegistry;
 
 /** The bot-defence vendor whose interstitial a document is, and what gave it away. */
 export interface ChallengeVerdict {
-  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel';
+  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel' | 'aws-waf';
   evidence: string;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/blockedPage.js

@@ -1,7 +1,7 @@
 /**
  * blockedPage.js — decide whether a fetched document is the page or a wall.
  *
- * Cloudflare, Amazon, DataDome, PerimeterX, Akamai and Vercel all answer a
+ * Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel and AWS WAF answer a
  * blocked request with HTTP 200 and a page of their own: a title, some prose
  * and a challenge script. Reported as a successful scrape, that page hides
  * the block — producthunt.com came back "success:true, title: Just a
@@ -57,6 +57,17 @@ const CHALLENGES = [
     title: /^vercel security checkpoint/i,
     markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
     evidence: 'a Vercel Security Checkpoint page'
+  },
+  {
+    // AWS WAF's challenge action answers with HTTP 202 and a page with no
+    // title and no text: `window.gokuProps = {…}` and a token.awswaf.com
+    // challenge.js that reloads into the real page about 0.4 s later
+    // (amazon.com to headless Chromium, 4 of 4 runs, 2026-10-03). gokuProps
+    // is the payload the WAF injects into its own page; a real page that
+    // integrates the WAF SDK loads challenge.js without it.
+    vendor: 'aws-waf',
+    markers: /window\.gokuProps\s*=/,
+    evidence: 'an AWS WAF challenge interstitial'
   }
 ];