crawlforge-extractors 1.13.0 → 1.14.0

package/README.md

@@ -202,7 +202,7 @@ URL is reported in `reason` as a retired handle.
 ### Recognising a blocked page
 
 `documentVerdict` says what a fetched document is: the page, a bot-wall
-interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF), an
+interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF, F5), an
 HTTP error page, an empty shell, a short error-titled placeholder, or a
 client-rendered app's own short error fallback under a normal title (quora.com's
 "Something went wrong. Wait a moment and try again."). A wall

package/index.d.ts

@@ -308,7 +308,7 @@ export default TemplateRegistry;
 
 /** The bot-defence vendor whose interstitial a document is, and what gave it away. */
 export interface ChallengeVerdict {
-  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel' | 'aws-waf';
+  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel' | 'aws-waf' | 'f5';
   evidence: string;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.13.0",
+  "version": "1.14.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/blockedPage.js

@@ -1,8 +1,8 @@
 /**
  * blockedPage.js — decide whether a fetched document is the page or a wall.
  *
- * Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel and AWS WAF answer a
- * blocked request with HTTP 200 and a page of their own: a title, some prose
+ * Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF and F5 answer
+ * a blocked request with a page of their own, often under HTTP 200: a title, some prose
  * and a challenge script. Reported as a successful scrape, that page hides
  * the block — producthunt.com came back "success:true, title: Just a
  * moment..." for three regression rounds (R10 Q1 → R15, 2026-09-04). The
@@ -68,6 +68,18 @@ const CHALLENGES = [
     vendor: 'aws-waf',
     markers: /window\.gokuProps\s*=/,
     evidence: 'an AWS WAF challenge interstitial'
+  },
+  {
+    // F5 BIG-IP ASM / Advanced WAF's default blocking page: the title
+    // "Request Rejected", one sentence and a support ID, about 250 bytes.
+    // walmart.com sent it with HTTP 444 to a plain fetch (2026-10-03). Its
+    // `server-timing: ak_p` header named Akamai, but walmart's normal 200s
+    // carry the same header (it is the CDN), so the page, not the header,
+    // says who refused the request.
+    vendor: 'f5',
+    title: /^request rejected$/i,
+    markers: /the requested url was rejected\.?\s*please consult with your administrator/i,
+    evidence: 'an F5 "Request Rejected" blocking page'
   }
 ];