imperium-crawl 2.3.1 → 2.5.1

package/dist/stealth/detector.js

@@ -1,11 +1,9 @@
 const BLOCKED_INDICATORS = [
     "access denied",
-    "403 forbidden",
     "captcha",
     "cf-challenge",
     "just a moment",
     "checking your browser",
-    "blocked",
     "rate limit",
     "too many requests",
     // Extended soft-block patterns
@@ -40,43 +38,137 @@ const SPA_INDICATORS = [
     "ng-app",
     'id="__nuxt"',
 ];
+// ── Anti-bot signal headers that confirm a status code is actually a block ──
+const ANTI_BOT_HEADERS = [
+    "cf-ray",
+    "cf-mitigated",
+    "x-datadome",
+    "x-datadome-cid",
+    "x-kpsdk-ct",
+    "x-sucuri-id",
+    "x-distil-cs",
+];
+/**
+ * Extract text content from HTML, stripping script/style tags.
+ * Returns the text length and the cleaned text.
+ */
+function extractTextContent(html) {
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    const content = bodyMatch ? bodyMatch[1] : html;
+    const text = content
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]*>/g, "")
+        .trim();
+    return { text, length: text.length };
+}
+/**
+ * Count links in HTML.
+ */
+function countLinks(html) {
+    const matches = html.match(/<a\s/gi);
+    return matches ? matches.length : 0;
+}
+/**
+ * Check if response headers contain known anti-bot signals.
+ */
+function hasAntiBotHeaders(headers) {
+    const lowerHeaders = Object.keys(headers).map((k) => k.toLowerCase());
+    return ANTI_BOT_HEADERS.some((h) => lowerHeaders.includes(h));
+}
+/**
+ * Check if the page has challenge scripts (Cloudflare, DataDome, etc.)
+ */
+function hasChallengeScript(html) {
+    return (html.includes("/cdn-cgi/challenge-platform/") ||
+        html.includes("js.datadome.co") ||
+        html.includes("client.perimeterx.net") ||
+        html.includes("awswaf") ||
+        html.includes("_kpsdk"));
+}
+// ── Legitimate page threshold: pages above this are almost never block pages ──
+const LEGITIMATE_TEXT_THRESHOLD = 5000; // 5KB of text content
+const SMALL_PAGE_THRESHOLD = 5000; // Only check content indicators on small pages
+const MIN_LINKS_FOR_LEGITIMATE = 10;
 /**
  * Detect if a response is blocked by anti-bot protection.
- * Checks HTTP status, page content, and structural signals.
+ *
+ * Uses a layered approach to minimize false positives:
+ * 1. Status codes (403/429/503) only trigger block if paired with anti-bot signals
+ * 2. Content-based checks only run on small pages (<5KB text)
+ * 3. Large pages with navigation are treated as legitimate regardless
  */
 export function isBlocked(html, status, headers) {
-    // Hard blocks by status
-    if (status === 403 || status === 429 || status === 503)
+    const responseHeaders = headers || {};
+    const hasAntiBot = hasAntiBotHeaders(responseHeaders);
+    const hasChallengeJs = hasChallengeScript(html);
+    const { text: bodyText, length: textLength } = extractTextContent(html);
+    // ── cf-mitigated header is an explicit block signal ──
+    if (responseHeaders["cf-mitigated"])
         return true;
-    // cf-mitigated header
-    if (headers && headers["cf-mitigated"])
-        return true;
-    const lower = html.toLowerCase();
-    // Content-based indicators
-    if (BLOCKED_INDICATORS.some((indicator) => lower.includes(indicator)))
-        return true;
-    // Challenge page title patterns
+    // ── Status-based detection (only with anti-bot confirmation) ──
+    if (status === 403 || status === 429 || status === 503) {
+        // If anti-bot headers or challenge scripts present → definitely blocked
+        if (hasAntiBot || hasChallengeJs)
+            return true;
+        // If response body is very small, likely a block page
+        if (textLength < SMALL_PAGE_THRESHOLD) {
+            // Check title for challenge patterns
+            const titleMatch = html.match(/<title[^>]*>(.*?)<\/title>/i);
+            if (titleMatch) {
+                const title = titleMatch[1].trim();
+                if (CHALLENGE_TITLE_PATTERNS.some((p) => p.test(title)))
+                    return true;
+            }
+            // Small body + error status + blocked indicators in body → block
+            const lower = bodyText.toLowerCase();
+            if (BLOCKED_INDICATORS.some((indicator) => lower.includes(indicator)))
+                return true;
+        }
+        // Large 403/503 without anti-bot signals → NOT a block (e.g. custom error page, maintenance)
+        return false;
+    }
+    // ── Legitimate page signals — bail early for large, link-rich pages ──
+    if (textLength > LEGITIMATE_TEXT_THRESHOLD && countLinks(html) > MIN_LINKS_FOR_LEGITIMATE) {
+        return false;
+    }
+    // ── Challenge page title patterns (any status) ──
     const titleMatch = html.match(/<title[^>]*>(.*?)<\/title>/i);
     if (titleMatch) {
         const title = titleMatch[1].trim();
         if (CHALLENGE_TITLE_PATTERNS.some((p) => p.test(title)))
             return true;
     }
-    // Unusually small response body for what should be a full page
-    // (challenge pages are typically <1KB of real content)
-    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-    if (bodyMatch) {
-        const bodyContent = bodyMatch[1]
-            .replace(/<script[\s\S]*?<\/script>/gi, "")
-            .replace(/<style[\s\S]*?<\/style>/gi, "")
-            .replace(/<[^>]*>/g, "")
-            .trim();
-        // If page has a title but almost no body text, likely a block page
-        if (titleMatch && bodyContent.length < 50 && status >= 400)
-            return true;
+    // ── Content-based indicators (only for small pages to avoid article false positives) ──
+    if (textLength < SMALL_PAGE_THRESHOLD) {
+        const lower = bodyText.toLowerCase();
+        if (BLOCKED_INDICATORS.some((indicator) => lower.includes(indicator))) {
+            // Extra check: if page also has challenge scripts or anti-bot headers → block
+            if (hasChallengeJs || hasAntiBot)
+                return true;
+            // If body is VERY small (<1KB), indicator match is strong enough alone
+            if (textLength < 1000)
+                return true;
+        }
     }
+    // ── Structural check: title but almost no body text on error status ──
+    if (titleMatch && textLength < 50 && status >= 400)
+        return true;
     return false;
 }
+/**
+ * Detect if a page contains a CAPTCHA challenge.
+ * Separate from isBlocked — a CAPTCHA page CAN be solved, it's not a hard block.
+ */
+export function isCaptchaPage(html) {
+    const lower = html.toLowerCase();
+    return (lower.includes("captcha") ||
+        lower.includes("cf-turnstile") ||
+        lower.includes("g-recaptcha") ||
+        lower.includes("h-captcha") ||
+        lower.includes("verify you are human") ||
+        lower.includes("please complete the security check"));
+}
 /**
  * Detect if a page requires JavaScript rendering to get content.
  */

package/dist/stealth/detector.js.map

@@ -1 +1 @@
-{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../src/stealth/detector.ts"],"names":[],"mappings":"AAAA,MAAM,kBAAkB,GAAG;IACzB,eAAe;IACf,eAAe;IACf,SAAS;IACT,cAAc;IACd,eAAe;IACf,uBAAuB;IACvB,SAAS;IACT,YAAY;IACZ,mBAAmB;IACnB,+BAA+B;IAC/B,sBAAsB;IACtB,oCAAoC;IACpC,2BAA2B;IAC3B,mBAAmB;IACnB,uBAAuB;IACvB,kBAAkB;IAClB,eAAe;IACf,iBAAiB;IACjB,8BAA8B;IAC9B,eAAe;IACf,oBAAoB;CACrB,CAAC;AAEF,MAAM,wBAAwB,GAAG;IAC/B,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,iBAAiB;IACjB,kBAAkB;IAClB,gBAAgB;IAChB,cAAc;IACd,aAAa;CACd,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,uBAAuB;IACvB,sBAAsB;IACtB,yBAAyB;IACzB,eAAe;IACf,iBAAiB;IACjB,QAAQ;IACR,aAAa;CACd,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,OAAgC;IACtF,wBAAwB;IACxB,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAEpE,sBAAsB;IACtB,IAAI,OAAO,IAAI,OAAO,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAEjC,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnF,gCAAgC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC7D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IACvE,CAAC;IAED,+DAA+D;IAC/D,uDAAuD;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC;aAC7B,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;aAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC;aACxC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,IAAI,EAAE,CAAC;QACV,mEAAmE;QACnE,IAAI,UAAU,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,IAAI,MAAM,IAAI,GAAG;YAAE,OAAO,IAAI,CAAC;IAC1E,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACnF,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IAED,mBAAmB;IACnB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACtE,CAAC"}
+{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../src/stealth/detector.ts"],"names":[],"mappings":"AAAA,MAAM,kBAAkB,GAAG;IACzB,eAAe;IACf,SAAS;IACT,cAAc;IACd,eAAe;IACf,uBAAuB;IACvB,YAAY;IACZ,mBAAmB;IACnB,+BAA+B;IAC/B,sBAAsB;IACtB,oCAAoC;IACpC,2BAA2B;IAC3B,mBAAmB;IACnB,uBAAuB;IACvB,kBAAkB;IAClB,eAAe;IACf,iBAAiB;IACjB,8BAA8B;IAC9B,eAAe;IACf,oBAAoB;CACrB,CAAC;AAEF,MAAM,wBAAwB,GAAG;IAC/B,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,iBAAiB;IACjB,kBAAkB;IAClB,gBAAgB;IAChB,cAAc;IACd,aAAa;CACd,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,uBAAuB;IACvB,sBAAsB;IACtB,yBAAyB;IACzB,eAAe;IACf,iBAAiB;IACjB,QAAQ;IACR,aAAa;CACd,CAAC;AAEF,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG;IACvB,QAAQ;IACR,cAAc;IACd,YAAY;IACZ,gBAAgB;IAChB,YAAY;IACZ,aAAa;IACb,aAAa;CACd,CAAC;AAEF;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO;SACjB,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC;SACxC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,IAAI,EAAE,CAAC;IACV,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrC,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAA+B;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACtE,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,CACL,IAAI,CAAC,QAAQ,CAAC,8BAA8B,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,MAAM,yBAAyB,GAAG,IAAI,CAAC,CAAC,sBAAsB;AAC9D,MAAM,oBAAoB,GAAG,IAAI,CAAC,CAAC,+CAA+C;AAClF,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAEpC;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,OAAgC;IACtF,MAAM,eAAe,GAAG,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,iBAAiB,CAAC,eAAe,CAAC,CAAC;IACtD,MAAM,cAAc,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAExE,wDAAwD;IACxD,IAAI,eAAe,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAEjD,iEAAiE;IACjE,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,wEAAwE;QACxE,IAAI,UAAU,IAAI,cAAc;YAAE,OAAO,IAAI,CAAC;QAE9C,sDAAsD;QACtD,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;YACtC,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC;YACvE,CAAC;YAED,iEAAiE;YACjE,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;YACrC,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;QACrF,CAAC;QAED,6FAA6F;QAC7F,OAAO,KAAK,CAAC;IACf,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,GAAG,yBAAyB,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,wBAAwB,EAAE,CAAC;QAC1F,OAAO,KAAK,CAAC;IACf,CAAC;IAED,mDAAmD;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC7D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IACvE,CAAC;IAED,yFAAyF;IACzF,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACtE,8EAA8E;YAC9E,IAAI,cAAc,IAAI,UAAU;gBAAE,OAAO,IAAI,CAAC;YAE9C,uEAAuE;YACvE,IAAI,UAAU,GAAG,IAAI;gBAAE,OAAO,IAAI,CAAC;QACrC,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,IAAI,UAAU,IAAI,UAAU,GAAG,EAAE,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IAEhE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,OAAO,CACL,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;QACzB,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC9B,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;QAC7B,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC3B,KAAK,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QACtC,KAAK,CAAC,QAAQ,CAAC,oCAAoC,CAAC,CACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,2BAA2B;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACnF,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IAED,mBAAmB;IACnB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACtE,CAAC"}

package/dist/stealth/headers.d.ts

@@ -1,3 +1,3 @@
-export declare function generateHeaders(overrides?: Record<string, string>): Record<string, string>;
+export declare function generateHeaders(overrides?: Record<string, string>, url?: string): Record<string, string>;
 export declare function getRandomUserAgent(): string;
 //# sourceMappingURL=headers.d.ts.map

package/dist/stealth/headers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/stealth/headers.ts"],"names":[],"mappings":"AAYA,wBAAgB,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAG1F;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAG3C"}
+{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/stealth/headers.ts"],"names":[],"mappings":"AAuCA,wBAAgB,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CA4ExG;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAG3C"}

package/dist/stealth/headers.js

@@ -2,14 +2,106 @@ import { HeaderGenerator } from "header-generator";
 const generator = new HeaderGenerator({
     browsers: [
         { name: "chrome", minVersion: 120 },
-        { name: "firefox", minVersion: 120 },
+        { name: "firefox", minVersion: 121 },
         { name: "edge", minVersion: 120 },
+        { name: "safari", minVersion: 17 },
     ],
     devices: ["desktop"],
     operatingSystems: ["windows", "macos", "linux"],
 });
-export function generateHeaders(overrides) {
+/**
+ * Parse Chrome major version from User-Agent string.
+ */
+function parseChromeVersion(ua) {
+    const match = ua.match(/Chrome\/(\d+)/);
+    return match ? match[1] : null;
+}
+/**
+ * Detect platform from User-Agent for sec-ch-ua-platform.
+ */
+function detectPlatform(ua) {
+    if (ua.includes("Windows"))
+        return '"Windows"';
+    if (ua.includes("Macintosh") || ua.includes("Mac OS"))
+        return '"macOS"';
+    if (ua.includes("Linux"))
+        return '"Linux"';
+    return '"Windows"';
+}
+/**
+ * Build sec-ch-ua hint from Chrome version.
+ * Matches real Chrome's format with Chromium brand + version.
+ */
+function buildSecChUa(majorVersion) {
+    return `"Chromium";v="${majorVersion}", "Google Chrome";v="${majorVersion}", "Not-A.Brand";v="99"`;
+}
+export function generateHeaders(overrides, url) {
     const headers = generator.getHeaders();
+    // ── Add Client Hints if Chrome/Edge UA detected ──
+    const ua = headers["user-agent"] || "";
+    const chromeVersion = parseChromeVersion(ua);
+    if (chromeVersion) {
+        // sec-ch-ua hints — must be consistent with User-Agent
+        headers["sec-ch-ua"] = buildSecChUa(chromeVersion);
+        headers["sec-ch-ua-mobile"] = "?0";
+        headers["sec-ch-ua-platform"] = detectPlatform(ua);
+    }
+    // ── URL-aware header enrichment ──
+    if (url) {
+        try {
+            const parsed = new URL(url);
+            const hostname = parsed.hostname;
+            // Add origin-based Referer for API endpoints and .gov sites
+            if (hostname.endsWith(".gov") || hostname.endsWith(".gov.com") ||
+                parsed.pathname.includes(".ashx") || parsed.pathname.includes("/api/") ||
+                parsed.pathname.includes("/proxy") || parsed.pathname.includes("Proxy")) {
+                headers["referer"] = `${parsed.origin}/`;
+                headers["origin"] = parsed.origin;
+            }
+            // .gov sites: add DNT and cache-control for less aggressive fingerprint
+            if (hostname.endsWith(".gov")) {
+                headers["dnt"] = "1";
+                headers["cache-control"] = "no-cache";
+            }
+        }
+        catch {
+            // Invalid URL, skip enrichment
+        }
+    }
+    // ── Standard Accept headers ──
+    if (!headers["accept"]) {
+        headers["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8";
+    }
+    if (!headers["accept-language"]) {
+        headers["accept-language"] = "en-US,en;q=0.9";
+    }
+    if (!headers["accept-encoding"]) {
+        headers["accept-encoding"] = "gzip, deflate, br";
+    }
+    // ── Navigation headers ──
+    headers["sec-fetch-dest"] = "document";
+    headers["sec-fetch-mode"] = "navigate";
+    headers["sec-fetch-site"] = headers["referer"] ? "same-origin" : "none";
+    headers["sec-fetch-user"] = "?1";
+    headers["upgrade-insecure-requests"] = "1";
+    headers["priority"] = "u=0, i";
+    // ── Additional realistic browser headers ──
+    // sec-gpc: Global Privacy Control — sent by Chrome/Firefox for GDPR compliance
+    headers["sec-gpc"] = "1";
+    // te: trailers — real Chrome sends this on HTTP/2+ requests
+    headers["te"] = "trailers";
+    // pragma for HTTP/1.1 backwards compatibility with older proxies/caches
+    if (!headers["pragma"]) {
+        headers["pragma"] = "no-cache";
+    }
+    // cache-control — real browsers send this on navigations
+    if (!headers["cache-control"]) {
+        headers["cache-control"] = "max-age=0";
+    }
+    // x-requested-with is sent by some sites to identify non-AJAX vs AJAX
+    delete headers["x-requested-with"];
+    // Remove any non-standard headers that could fingerprint the scraper
+    delete headers["x-devtools-emulate-network-conditions-client-id"];
     return { ...headers, ...overrides };
 }
 export function getRandomUserAgent() {

package/dist/stealth/headers.js.map

@@ -1 +1 @@
-{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../src/stealth/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC;IACpC,QAAQ,EAAE;QACR,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE;QACnC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE;QACpC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE;KAClC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC;CAChD,CAAC,CAAC;AAEH,MAAM,UAAU,eAAe,CAAC,SAAkC;IAChE,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;IACvC,OAAO,EAAE,GAAG,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,YAAY,CAAC,IAAI,8DAA8D,CAAC;AACjG,CAAC"}
+{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../src/stealth/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC;IACpC,QAAQ,EAAE;QACR,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE;QACnC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE;QACpC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE;QACjC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KACnC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC;CAChD,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,kBAAkB,CAAC,EAAU;IACpC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/C,IAAI,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,YAAoB;IACxC,OAAO,iBAAiB,YAAY,yBAAyB,YAAY,yBAAyB,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAkC,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;IAEvC,oDAAoD;IACpD,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,aAAa,GAAG,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE7C,IAAI,aAAa,EAAE,CAAC;QAClB,uDAAuD;QACvD,OAAO,CAAC,WAAW,CAAC,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QACnD,OAAO,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC;QACnC,OAAO,CAAC,oBAAoB,CAAC,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,oCAAoC;IACpC,IAAI,GAAG,EAAE,CAAC;QACR,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YAEjC,4DAA4D;YAC5D,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAC1D,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACtE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5E,OAAO,CAAC,SAAS,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;gBACzC,OAAO,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YACpC,CAAC;YAED,wEAAwE;YACxE,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;gBACrB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,CAAC;YACxC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,CAAC,GAAG,kGAAkG,CAAC;IACzH,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,iBAAiB,CAAC,GAAG,gBAAgB,CAAC;IAChD,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,iBAAiB,CAAC,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAED,2BAA2B;IAC3B,OAAO,CAAC,gBAAgB,CAAC,GAAG,UAAU,CAAC;IACvC,OAAO,CAAC,gBAAgB,CAAC,GAAG,UAAU,CAAC;IACvC,OAAO,CAAC,gBAAgB,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACjC,OAAO,CAAC,2BAA2B,CAAC,GAAG,GAAG,CAAC;IAC3C,OAAO,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;IAE/B,6CAA6C;IAC7C,+EAA+E;IAC/E,OAAO,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;IACzB,4DAA4D;IAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;IAC3B,wEAAwE;IACxE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;IACjC,CAAC;IACD,yDAAyD;IACzD,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,eAAe,CAAC,GAAG,WAAW,CAAC;IACzC,CAAC;IACD,sEAAsE;IACtE,OAAO,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACnC,qEAAqE;IACrE,OAAO,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAElE,OAAO,EAAE,GAAG,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,YAAY,CAAC,IAAI,8DAA8D,CAAC;AACjG,CAAC"}

package/dist/stealth/index.d.ts

@@ -10,11 +10,15 @@ export interface FetchResult {
     proxyUsed?: string;
     chromeProfile?: string;
     antiBotSystem?: string;
+    /** Indicates the result may be incomplete due to partial failure */
+    degraded?: boolean;
 }
 export declare class StealthError extends Error {
     readonly lastLevel: StealthLevel;
     readonly httpStatus: number;
     readonly antiBotSystem: string | null;
+    /** Best partial result collected during escalation, if any */
+    partialResult?: FetchResult;
     constructor(message: string, lastLevel: StealthLevel, httpStatus: number, antiBotSystem: string | null);
 }
 export interface StealthOptions {
@@ -24,6 +28,7 @@ export interface StealthOptions {
     screenshot?: boolean;
     proxy?: string;
     chromeProfile?: string;
+    sessionId?: string;
 }
 export declare function smartFetch(url: string, options?: StealthOptions): Promise<FetchResult>;
 export { isPlaywrightAvailable } from "./browser.js";

package/dist/stealth/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stealth/index.ts"],"names":[],"mappings":"AAUA,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAErC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,YAAa,SAAQ,KAAK;aAGnB,SAAS,EAAE,YAAY;aACvB,UAAU,EAAE,MAAM;aAClB,aAAa,EAAE,MAAM,GAAG,IAAI;gBAH5C,OAAO,EAAE,MAAM,EACC,SAAS,EAAE,YAAY,EACvB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,GAAG,IAAI;CAK/C;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAuFD,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAwK5F;AAmBD,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stealth/index.ts"],"names":[],"mappings":"AAsIA,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAErC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,qBAAa,YAAa,SAAQ,KAAK;aAMnB,SAAS,EAAE,YAAY;aACvB,UAAU,EAAE,MAAM;aAClB,aAAa,EAAE,MAAM,GAAG,IAAI;IAP9C,8DAA8D;IACvD,aAAa,CAAC,EAAE,WAAW,CAAC;gBAGjC,OAAO,EAAE,MAAM,EACC,SAAS,EAAE,YAAY,EACvB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,GAAG,IAAI;CAK/C;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0ID,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAmQ5F;AAoBD,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC"}