@js-recon/js-recon 1.4.3 → 2.0.1-alpha.1

package/build/utility/makeReq.js

@@ -23,14 +23,18 @@ import { ProxyAgent, Socks5ProxyAgent } from "undici";
 import puppeteer from "./puppeteerInstance.js";
 import { getChromiumPath } from "./getChromiumPath.js";
 import * as globals from "./globals.js";
-import { get } from "../proxy/genReq.js";
+import { getWithMetadata } from "../proxy/genReq.js";
 import checkFireWallBlocking from "../proxy/checkFireWallBlocking.js";
 import { parseProxyUrl } from "../proxy/genericProxy.js";
 import { buildOxylabsProxyUrl, composeOxylabsUsername } from "../proxy/oxylabsProxy.js";
 import fs from "fs";
 import { EventEmitter } from "events";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createHash, randomUUID } from "node:crypto";
 import { progressError, progressLog } from "./progressLog.js";
 import { isSigintHandlerActive } from "../run/interruptHandler.js";
+import detectBlockedResponse from "../proxy/detectBlockedResponse.js";
+import { claimOxylabsFallback, getOxylabsFallbackConfiguration } from "../proxy/oxylabsFallback.js";
 /**
  * Proxy-wiring layer. This is the single place a resolved proxy config (see
  * `../proxy/resolveProxyConfig.ts`) is turned into an actual dispatcher/launch args — every
@@ -45,11 +49,6 @@ export const getResolvedProxyConfigFromGlobals = () => {
         oxylabs: globals.getOxylabsConfig(),
     };
 };
-/** Merges a proxy dispatcher (derived from the current globals) into fetch options, if a proxy is configured. */
-export const withProxyDispatcher = (opts = {}) => {
-    const dispatcher = buildUndiciDispatcher(getResolvedProxyConfigFromGlobals());
-    return dispatcher ? Object.assign(Object.assign({}, opts), { dispatcher }) : opts;
-};
 /**
  * Builds an undici dispatcher for the resolved proxy config, to pass as `fetch(url, { dispatcher })`.
  * Returns null for the `aws` method (routed separately via genReq.ts) or no proxy configured.
@@ -74,6 +73,34 @@ export const buildUndiciDispatcher = (resolved) => {
     }
     return null;
 };
+/**
+ * Runs request work with an owned dispatcher and always releases its sockets.
+ * The callback must consume response bodies before returning.
+ */
+export const withManagedDispatcher = (dispatcher, work) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!dispatcher)
+        return yield work({});
+    let completed = false;
+    try {
+        const result = yield work({ dispatcher });
+        completed = true;
+        return result;
+    }
+    finally {
+        const cleanup = completed ? dispatcher.close() : dispatcher.destroy();
+        yield cleanup.catch(() => undefined);
+    }
+});
+/** Runs request work through the globally active proxy, when proxy use is enabled. */
+export const withActiveProxyDispatcher = (work) => __awaiter(void 0, void 0, void 0, function* () {
+    const dispatcher = globals.getUseProxy() ? buildUndiciDispatcher(getResolvedProxyConfigFromGlobals()) : null;
+    return yield withManagedDispatcher(dispatcher, work);
+});
+const requestSignalContext = new AsyncLocalStorage();
+/** Supplies cancellation to every makeRequest call in the async work scope. */
+export const withRequestSignal = (signal, work) => requestSignalContext.run(signal, work);
+/** True when the current async request scope has been cancelled. */
+export const isRequestCancelled = () => { var _a; return ((_a = requestSignalContext.getStore()) === null || _a === void 0 ? void 0 : _a.aborted) === true; };
 /**
  * Builds the Puppeteer `--proxy-server=` launch arg plus optional `page.authenticate()` credentials.
  * `aws` never applies here (Puppeteer never went through API Gateway).
@@ -102,11 +129,9 @@ export const buildPuppeteerProxyArgs = (resolved) => {
     }
     return { arg: null };
 };
-const reportedFailures = new Set();
+/** Returns browser proxy arguments only when proxy routing is currently enabled. */
+export const getActivePuppeteerProxyArgs = () => globals.getUseProxy() ? buildPuppeteerProxyArgs(getResolvedProxyConfigFromGlobals()) : { arg: null };
 const reportFailure = (url, err) => {
-    if (reportedFailures.has(url))
-        return;
-    reportedFailures.add(url);
     if (globals.getCacheOnly()) {
         progressError(chalk.dim(`[!] Cache miss (cache-only mode): ${url}`));
         return;
@@ -116,10 +141,26 @@ const reportFailure = (url, err) => {
         progressLog(chalk.dim("[i] Often, using -k flag (ignore SSL errors) fixes the problem"));
     }
 };
+const reportTimeout = (url, requestTimeout) => {
+    progressError(chalk.red(`[!] Request to ${url} timed out after ${requestTimeout}ms`));
+};
+const reportInvalidUrl = (url) => {
+    progressError(chalk.red(`[!] Invalid request URL: ${url}`));
+};
 // random user agents
 const UAs = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
 ];
+const getCacheIdentityDigest = (url, headers) => {
+    const normalizedHeaders = [...new Headers(headers).entries()].sort(([left], [right]) => left.localeCompare(right));
+    return createHash("sha256")
+        .update(JSON.stringify([url, normalizedHeaders]))
+        .digest("hex");
+};
+const getCacheEntryPath = (url, headers) => {
+    const digest = getCacheIdentityDigest(url, headers);
+    return `${globals.getRespCacheFile()}.entries/${digest}.json`;
+};
 /**
  * Reads response data from cache for future requests.
  *
@@ -135,6 +176,23 @@ const UAs = [
  * @returns A Promise that resolves to a Response object if the cache is found, or null if not
  */
 const readCache = (url, headers) => __awaiter(void 0, void 0, void 0, function* () {
+    const identityDigest = getCacheIdentityDigest(url, headers);
+    const entryPath = getCacheEntryPath(url, headers);
+    if (fs.existsSync(entryPath)) {
+        try {
+            const entry = JSON.parse(fs.readFileSync(entryPath, "utf-8"));
+            if (entry.identityDigest === identityDigest) {
+                return new Response(Buffer.from(entry.bodyBase64, "base64"), {
+                    status: entry.status,
+                    statusText: entry.statusText,
+                    headers: entry.responseHeaders,
+                });
+            }
+        }
+        catch (_a) {
+            // Ignore corrupt sidecar entries and fall back to the legacy cache.
+        }
+    }
     // first, check if the file exists or not
     if (!fs.existsSync(globals.getRespCacheFile())) {
         return null;
@@ -145,13 +203,12 @@ const readCache = (url, headers) => __awaiter(void 0, void 0, void 0, function*
     try {
         cache = JSON.parse(fs.readFileSync(globals.getRespCacheFile(), "utf-8"));
     }
-    catch (_a) {
+    catch (_b) {
         return null;
     }
     if (cache[url]) {
         // check if the response contains the specific request headers
         // iterate through cache[url] and build a Response
-        let headersMatch = true;
         // first check if the essential headers match
         const rscEnabled = headers["RSC"] ? true : false;
         if (rscEnabled) {
@@ -175,139 +232,140 @@ const readCache = (url, headers) => __awaiter(void 0, void 0, void 0, function*
 /**
  * Writes response data to cache for future requests.
  *
- * Stores response body, status, and headers in cache file, handling special
- * cases like RSC (React Server Components) headers separately.
+ * Stores one atomic sidecar entry per URL/request-header identity. The legacy
+ * JSON cache remains readable, but new writes avoid whole-cache rewrites.
  *
  * @param url - The URL to cache the response for
  * @param headers - Request headers that were used
  * @param response - The Response object to cache
  * @returns Promise that resolves when caching is complete
  */
-const writeCache = (url, headers, response) => __awaiter(void 0, void 0, void 0, function* () {
+const writeCache = (url_1, headers_1, response_1, ...args_1) => __awaiter(void 0, [url_1, headers_1, response_1, ...args_1], void 0, function* (url, headers, response, reportErrors = true, signal) {
+    if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+        return;
     try {
-        yield writeCacheUnsafe(url, headers, response);
+        yield writeCacheUnsafe(url, headers, response, signal);
     }
     catch (err) {
         // Caching must never break a request. Log and move on.
-        progressError(chalk.yellow(`[!] Failed to write response cache for ${url}: ${(err === null || err === void 0 ? void 0 : err.message) || err}`));
+        if (reportErrors) {
+            progressError(chalk.yellow(`[!] Failed to write response cache for ${url}: ${(err === null || err === void 0 ? void 0 : err.message) || err}`));
+        }
     }
 });
-const writeCacheUnsafe = (url, headers, response) => __awaiter(void 0, void 0, void 0, function* () {
-    // clone the response
-    const clonedResponse = response.clone();
-    // if cache exists, return
-    if ((yield readCache(url, headers)) !== null) {
-        // console.log("cache already exists for ", url);
+const writeCacheUnsafe = (url, headers, response, signal) => __awaiter(void 0, void 0, void 0, function* () {
+    if (signal === null || signal === void 0 ? void 0 : signal.aborted)
         return;
-    }
-    // open the cache file, and write the response based on the special headers
-    let cache;
-    try {
-        cache = JSON.parse(fs.readFileSync(globals.getRespCacheFile(), "utf-8"));
-    }
-    catch (_a) {
-        cache = {};
-    }
-    if (!cache[url]) {
-        cache[url] = {};
-    }
+    const identityDigest = getCacheIdentityDigest(url, headers);
+    const entryPath = getCacheEntryPath(url, headers);
+    if (fs.existsSync(entryPath))
+        return;
+    const clonedResponse = response.clone();
     const bodyBuffer = Buffer.from(yield clonedResponse.arrayBuffer());
-    const body = bodyBuffer.toString("base64");
-    const status = clonedResponse.status;
-    const resp_headers = clonedResponse.headers;
-    if (headers["RSC"]) {
-        cache[url].rsc = {
-            req_headers: headers,
-            status: status,
-            body_b64: body,
-            resp_headers: resp_headers,
-        };
-        // console.log("rsc", url);
-    }
-    else {
-        cache[url].normal = {
-            req_headers: headers,
-            status: status,
-            body_b64: body,
-            resp_headers: resp_headers,
-        };
-        // console.log("normal", url);
-    }
-    let serialized;
+    if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+        return;
+    const entry = Object.freeze({
+        identityDigest,
+        status: clonedResponse.status,
+        statusText: clonedResponse.statusText,
+        bodyBase64: bodyBuffer.toString("base64"),
+        responseHeaders: Object.freeze(Object.fromEntries(clonedResponse.headers.entries())),
+    });
+    const entryDirectory = `${globals.getRespCacheFile()}.entries`;
+    fs.mkdirSync(entryDirectory, { recursive: true });
+    const temporaryPath = `${entryPath}.${process.pid}.${randomUUID()}.tmp`;
     try {
-        // so, the cache is the response body for the URL
-        serialized = JSON.stringify(cache);
+        fs.writeFileSync(temporaryPath, JSON.stringify(entry));
+        if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+            return;
+        fs.renameSync(temporaryPath, entryPath);
     }
-    catch (err) {
-        if (err instanceof RangeError) {
-            // console.log(
-            //     chalk.yellow(
-            //         `[!] Response cache too large to serialize; dropping entry for ${url} and skipping cache write`
-            //     )
-            // );
-            // // Roll back the entry we just added so future writes don't keep retrying.
-            // if (headers["RSC"]) {
-            //     delete cache[url].rsc;
-            // } else {
-            //     delete cache[url].normal;
-            // }
-            // if (!cache[url].rsc && !cache[url].normal) {
-            //     delete cache[url];
-            // }
-            // return;
-            progressError(chalk.yellow(`[!] Cache too big... Emptying cache`));
-            serialized = JSON.stringify({}); // empty cache
-        }
-        // throw err;
+    finally {
+        if (fs.existsSync(temporaryPath))
+            fs.unlinkSync(temporaryPath);
     }
-    fs.writeFileSync(globals.getRespCacheFile(), serialized);
-    // console.log("wrote cache for ", url);
 });
-/**
- * Performs a single fetch request with retry logic.
- *
- * @param url - The URL to request
- * @param requestOptions - Request options including headers
- * @param requestTimeout - Timeout in milliseconds
- * @returns A Promise that resolves to a Response, or null if all retries fail
- */
-const singleFetch = (url, requestOptions, requestTimeout) => __awaiter(void 0, void 0, void 0, function* () {
-    let res;
+const singleFetch = (url_1, requestOptions_1, requestTimeout_1, ...args_1) => __awaiter(void 0, [url_1, requestOptions_1, requestTimeout_1, ...args_1], void 0, function* (url, requestOptions, requestTimeout, reportErrors = true, dispatcherOverride, maxAttempts = 11) {
+    var _a, _b;
     let counter = 0;
-    while (true) {
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), requestTimeout);
-        const currentRequestOptions = withProxyDispatcher(Object.assign(Object.assign({}, requestOptions), { signal: controller.signal }));
-        try {
-            EventEmitter.defaultMaxListeners = 20;
-            res = yield fetch(url, currentRequestOptions);
-            clearTimeout(timeoutId);
-            if (res) {
-                break;
+    let shouldDestroyOwnedDispatcher = false;
+    const ownedDispatcher = dispatcherOverride || !globals.getUseProxy()
+        ? null
+        : buildUndiciDispatcher(getResolvedProxyConfigFromGlobals());
+    const dispatcher = (_a = dispatcherOverride !== null && dispatcherOverride !== void 0 ? dispatcherOverride : ownedDispatcher) !== null && _a !== void 0 ? _a : undefined;
+    try {
+        while (true) {
+            const controller = new AbortController();
+            let timedOut = false;
+            const timeoutId = setTimeout(() => {
+                timedOut = true;
+                controller.abort();
+            }, requestTimeout);
+            const upstreamSignal = requestOptions.signal;
+            const abortFromUpstream = () => controller.abort();
+            if (upstreamSignal === null || upstreamSignal === void 0 ? void 0 : upstreamSignal.aborted) {
+                shouldDestroyOwnedDispatcher = true;
+                controller.abort();
+                clearTimeout(timeoutId);
+                return null;
             }
-        }
-        catch (err) {
-            clearTimeout(timeoutId);
-            counter++;
-            // BUG: https://github.com/nodejs/node/issues/47246
-            if (err.cause && err.cause.code === "UND_ERR_HEADERS_OVERFLOW") {
-                progressError(chalk.yellow(`[!] The tool detected a header overflow. Please increase the limit by setting environment variable \`NODE_OPTIONS="--max-http-header-size=99999999"\`. If the error still persists, please try again with a higher limit.`));
-                process.exit(21);
+            else {
+                upstreamSignal === null || upstreamSignal === void 0 ? void 0 : upstreamSignal.addEventListener("abort", abortFromUpstream, { once: true });
             }
-            if (err.name === "AbortError") {
-                progressError(chalk.red(`[!] Request to ${url} timed out after ${requestTimeout}ms`));
-                return null;
+            const currentRequestOptions = Object.assign(Object.assign(Object.assign({}, requestOptions), { signal: controller.signal }), (dispatcher ? { dispatcher } : {}));
+            try {
+                EventEmitter.defaultMaxListeners = 20;
+                const response = yield fetch(url, currentRequestOptions); // lgtm[js/insecure-download]: this is a recon tool's generic HTTP client — it fetches whatever URL/scheme the target itself serves, by design.
+                const arrayBuffer = yield response.arrayBuffer();
+                if (controller.signal.aborted) {
+                    shouldDestroyOwnedDispatcher = true;
+                    return null;
+                }
+                const body = Buffer.alloc(arrayBuffer.byteLength);
+                body.set(new Uint8Array(arrayBuffer));
+                return {
+                    body,
+                    status: response.status,
+                    headers: new Headers(response.headers),
+                    ok: response.ok,
+                    text: body.toString("utf-8"),
+                };
             }
-            if (counter > 10) {
-                reportFailure(url, err);
-                return null;
+            catch (err) {
+                counter++;
+                // BUG: https://github.com/nodejs/node/issues/47246
+                if (err.cause && err.cause.code === "UND_ERR_HEADERS_OVERFLOW") {
+                    progressError(chalk.yellow(`[!] The tool detected a header overflow. Please increase the limit by setting environment variable \`NODE_OPTIONS="--max-http-header-size=99999999"\`. If the error still persists, please try again with a higher limit.`));
+                    process.exit(21);
+                }
+                if (err.name === "AbortError") {
+                    shouldDestroyOwnedDispatcher = true;
+                    if (reportErrors && (timedOut || !(upstreamSignal === null || upstreamSignal === void 0 ? void 0 : upstreamSignal.aborted)))
+                        reportTimeout(url, requestTimeout);
+                    return null;
+                }
+                if (counter >= maxAttempts) {
+                    shouldDestroyOwnedDispatcher = true;
+                    if (reportErrors)
+                        reportFailure(url, err);
+                    return null;
+                }
+                yield new Promise((resolve) => setTimeout(resolve, 500));
+            }
+            finally {
+                clearTimeout(timeoutId);
+                upstreamSignal === null || upstreamSignal === void 0 ? void 0 : upstreamSignal.removeEventListener("abort", abortFromUpstream);
             }
-            // sleep 0.5 s before retrying
-            yield new Promise((resolve) => setTimeout(resolve, 500));
-            continue;
         }
     }
-    return res;
+    finally {
+        if (ownedDispatcher) {
+            const cleanup = shouldDestroyOwnedDispatcher || ((_b = requestOptions.signal) === null || _b === void 0 ? void 0 : _b.aborted)
+                ? ownedDispatcher.destroy()
+                : ownedDispatcher.close();
+            yield cleanup.catch(() => undefined);
+        }
+    }
 });
 /**
  * Handles firewall detection and bypass using headless browser.
@@ -316,58 +374,65 @@ const singleFetch = (url, requestOptions, requestTimeout) => __awaiter(void 0, v
  * @param resp_text - The response text to check for firewall signatures
  * @returns A Promise that resolves to Response content if firewall detected, or null if not
  */
-const handleFirewall = (url, resp_text) => __awaiter(void 0, void 0, void 0, function* () {
-    if (resp_text.includes("/?bm-verify=") || resp_text.includes("<title>Just a moment...</title>")) {
-        progressError(chalk.yellow(`[!] CF Firewall detected. Trying to bypass with headless browser`));
-        const chromiumPath = getChromiumPath();
-        const proxyArgs = buildPuppeteerProxyArgs(getResolvedProxyConfigFromGlobals());
-        const browser = yield puppeteer.launch({
-            headless: true,
-            executablePath: chromiumPath,
-            args: [
-                ...(globals.getDisableSandbox()
-                    ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
-                    : []),
-                ...(proxyArgs.arg ? [proxyArgs.arg] : []),
-            ],
-            handleSIGINT: !isSigintHandlerActive(),
-        });
+const handleFirewall = (url_1, responseText_1, responseMetadata_1, signal_1, ...args_1) => __awaiter(void 0, [url_1, responseText_1, responseMetadata_1, signal_1, ...args_1], void 0, function* (url, responseText, responseMetadata, signal, reportErrors = true) {
+    var _a;
+    const detection = detectBlockedResponse({
+        status: (_a = responseMetadata === null || responseMetadata === void 0 ? void 0 : responseMetadata.status) !== null && _a !== void 0 ? _a : 200,
+        headers: responseMetadata === null || responseMetadata === void 0 ? void 0 : responseMetadata.headers,
+        body: responseText,
+    });
+    if (!detection.blocked || (signal === null || signal === void 0 ? void 0 : signal.aborted))
+        return null;
+    const firewall = detection.provider === "cloudfront" ? "CloudFront" : "Cloudflare/CDN";
+    if (reportErrors) {
+        progressError(chalk.yellow(`[!] ${firewall} Firewall detected. Trying to bypass with headless browser`));
+    }
+    const chromiumPath = getChromiumPath();
+    const proxyArgs = getActivePuppeteerProxyArgs();
+    const browser = yield puppeteer.launch({
+        headless: true,
+        executablePath: chromiumPath,
+        args: [
+            ...(globals.getDisableSandbox()
+                ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
+                : []),
+            ...(proxyArgs.arg ? [proxyArgs.arg] : []),
+        ],
+        handleSIGINT: !isSigintHandlerActive(),
+    });
+    const abortBrowser = () => {
+        void browser.close().catch(() => undefined);
+    };
+    signal === null || signal === void 0 ? void 0 : signal.addEventListener("abort", abortBrowser, { once: true });
+    try {
+        if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+            return null;
         const page = yield browser.newPage();
-        if (proxyArgs.authenticate) {
+        if (proxyArgs.authenticate)
             yield page.authenticate(proxyArgs.authenticate);
-        }
         yield page.goto(url);
-        yield new Promise((resolve) => setTimeout(resolve, 5000));
-        const content = yield page.content();
-        yield browser.close();
-        return content;
-    }
-    else if (resp_text.includes("403 ERROR") && resp_text.includes("Generated by cloudfront")) {
-        progressError(chalk.yellow(`[!] Cloudfront Firewall detected. Trying to bypass with headless browser`));
-        const chromiumPath2 = getChromiumPath();
-        const proxyArgs = buildPuppeteerProxyArgs(getResolvedProxyConfigFromGlobals());
-        const browser = yield puppeteer.launch({
-            headless: true,
-            executablePath: chromiumPath2,
-            args: [
-                ...(globals.getDisableSandbox()
-                    ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
-                    : []),
-                ...(proxyArgs.arg ? [proxyArgs.arg] : []),
-            ],
-            handleSIGINT: !isSigintHandlerActive(),
+        if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+            return null;
+        yield new Promise((resolve) => {
+            const finish = () => {
+                clearTimeout(timeout);
+                signal === null || signal === void 0 ? void 0 : signal.removeEventListener("abort", finish);
+                resolve();
+            };
+            const timeout = setTimeout(finish, 5000);
+            signal === null || signal === void 0 ? void 0 : signal.addEventListener("abort", finish, { once: true });
         });
-        const page = yield browser.newPage();
-        if (proxyArgs.authenticate) {
-            yield page.authenticate(proxyArgs.authenticate);
-        }
-        yield page.goto(url);
-        yield new Promise((resolve) => setTimeout(resolve, 5000));
-        const content = yield page.content();
-        yield browser.close();
-        return content;
+        return (signal === null || signal === void 0 ? void 0 : signal.aborted) ? null : yield page.content();
+    }
+    catch (err) {
+        if (signal === null || signal === void 0 ? void 0 : signal.aborted)
+            return null;
+        throw err;
+    }
+    finally {
+        signal === null || signal === void 0 ? void 0 : signal.removeEventListener("abort", abortBrowser);
+        yield browser.close().catch(() => undefined);
     }
-    return null;
 });
 /**
  * Makes a GET request to the given URL and returns the response.
@@ -387,13 +452,22 @@ const handleFirewall = (url, resp_text) => __awaiter(void 0, void 0, void 0, fun
  * @returns A Promise that resolves to a Response, or null if all retries fail
  */
 const makeRequest = (url, args) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b, _c, _d, _e;
     if (url.startsWith("//")) {
         url = "https:" + url;
     }
-    const _a = args || {}, { timeout } = _a, restArgs = __rest(_a, ["timeout"]);
-    const requestOptions = restArgs;
+    const _f = args || {}, { timeout, reportErrors = true } = _f, restArgs = __rest(_f, ["timeout", "reportErrors"]);
+    const contextualSignal = requestSignalContext.getStore();
+    const explicitSignal = (_a = restArgs.signal) !== null && _a !== void 0 ? _a : undefined;
+    const effectiveSignal = contextualSignal && explicitSignal && contextualSignal !== explicitSignal
+        ? AbortSignal.any([contextualSignal, explicitSignal])
+        : (explicitSignal !== null && explicitSignal !== void 0 ? explicitSignal : contextualSignal);
+    const isCancelled = () => (effectiveSignal === null || effectiveSignal === void 0 ? void 0 : effectiveSignal.aborted) === true;
+    if (isCancelled())
+        return null;
     const requestTimeout = timeout || globals.getRequestTimeout();
-    const usingDefaultHeaders = !requestOptions.headers;
+    const usingDefaultHeaders = !restArgs.headers;
+    const requestStartedWithProxy = globals.getUseProxy();
     // Build default headers if not provided
     const baseHeaders = {
         "User-Agent": UAs[Math.floor(Math.random() * UAs.length)],
@@ -407,63 +481,111 @@ const makeRequest = (url, args) => __awaiter(void 0, void 0, void 0, function* (
     try {
         parsedOrigin = new URL(url).origin;
     }
-    catch (_b) {
+    catch (_g) {
+        if (reportErrors)
+            reportInvalidUrl(url);
         return null;
     }
-    if (!requestOptions.headers) {
-        requestOptions.headers = Object.assign(Object.assign({}, baseHeaders), { Referer: parsedOrigin, Origin: parsedOrigin });
-    }
+    const requestOptions = Object.assign(Object.assign(Object.assign({}, restArgs), (effectiveSignal ? { signal: effectiveSignal } : {})), { headers: (_b = restArgs.headers) !== null && _b !== void 0 ? _b : Object.assign(Object.assign({}, baseHeaders), { Referer: parsedOrigin, Origin: parsedOrigin }) });
+    // A successful default-header request may have used the no-Referer fallback.
+    // Probe both request identities so that response remains reusable on later
+    // runs, including cache-only runs that cannot repeat the network fallback.
+    const cacheHeaderCandidates = usingDefaultHeaders
+        ? [requestOptions.headers || {}, baseHeaders]
+        : [requestOptions.headers || {}];
     // if cache is enabled, read the cache and return if cache is present. else, continue
     if (!globals.getDisableCache()) {
-        const cachedResponse = yield readCache(url, requestOptions.headers || {});
+        let cachedEntry = null;
+        for (const candidateHeaders of cacheHeaderCandidates) {
+            const response = yield readCache(url, candidateHeaders);
+            if (response !== null) {
+                cachedEntry = Object.freeze({ response, headers: candidateHeaders });
+                break;
+            }
+        }
+        if (isCancelled())
+            return null;
         // if the response if cached, then check if contains any signatures of firewall
-        if (cachedResponse !== null) {
-            const cachedText = yield cachedResponse.clone().text();
-            const firewallContent = yield handleFirewall(url, cachedText);
-            if (firewallContent) {
-                yield writeCache(url, requestOptions.headers || {}, new Response(firewallContent));
-                return new Response(firewallContent);
+        if (cachedEntry !== null) {
+            const cachedText = yield cachedEntry.response.clone().text();
+            if (isCancelled())
+                return null;
+            const cachedDetection = detectBlockedResponse({
+                status: cachedEntry.response.status,
+                headers: cachedEntry.response.headers,
+                body: cachedText,
+            });
+            const shouldRefreshBlockedCache = !globals.getCacheOnly() &&
+                !requestStartedWithProxy &&
+                getOxylabsFallbackConfiguration().enabled &&
+                cachedDetection.blocked;
+            if (shouldRefreshBlockedCache) {
+                const sidecarPath = getCacheEntryPath(url, cachedEntry.headers);
+                try {
+                    if (fs.existsSync(sidecarPath))
+                        fs.unlinkSync(sidecarPath);
+                }
+                catch (error) {
+                    if (reportErrors) {
+                        progressError(chalk.yellow(`[!] Could not replace blocked cache entry for ${url}: ${error}`));
+                    }
+                }
+            }
+            else {
+                if (globals.getCacheOnly())
+                    return cachedEntry.response;
+                const firewallContent = yield handleFirewall(url, cachedText, { status: cachedEntry.response.status, headers: cachedEntry.response.headers }, effectiveSignal, reportErrors);
+                if (isCancelled())
+                    return null;
+                if (firewallContent) {
+                    yield writeCache(url, cachedEntry.headers, new Response(firewallContent), reportErrors, effectiveSignal);
+                    if (isCancelled())
+                        return null;
+                    return new Response(firewallContent);
+                }
+                return cachedEntry.response;
             }
-            return cachedResponse;
         }
     }
     if (globals.getCacheOnly()) {
-        reportFailure(url, "cache miss in cache-only mode");
+        if (reportErrors)
+            reportFailure(url, "cache miss in cache-only mode");
         return null;
     }
     if (globals.getUseProxy() && globals.getProxyMethod() === "aws") {
-        const get_headers = requestOptions.headers;
-        const body = yield get(url, get_headers);
+        const getHeaders = Object.fromEntries(new Headers(requestOptions.headers).entries());
+        let awsResponse;
+        try {
+            awsResponse = yield getWithMetadata(url, getHeaders, effectiveSignal);
+        }
+        catch (err) {
+            if (isCancelled())
+                return null;
+            if (reportErrors)
+                reportFailure(url, err);
+            return null;
+        }
+        if (isCancelled())
+            return null;
         // check if any firewall is there in the way
-        if (yield checkFireWallBlocking(body)) {
+        if (yield checkFireWallBlocking(awsResponse.body, { status: awsResponse.status, headers: awsResponse.headers })) {
             progressError(chalk.magenta("[!] Please try again without API Gateway"));
             process.exit(18);
         }
         // craft a Response, and return that
-        const response = new Response(body);
+        const response = new Response(awsResponse.body, {
+            status: (_c = awsResponse.status) !== null && _c !== void 0 ? _c : 200,
+            headers: (_d = awsResponse.headers) !== null && _d !== void 0 ? _d : {},
+        });
         // if cache is enabled, write the response to the cache
         if (!globals.getDisableCache()) {
-            yield writeCache(url, get_headers, response);
+            yield writeCache(url, getHeaders, response, reportErrors, effectiveSignal);
         }
+        if (isCancelled())
+            return null;
         return response;
     }
     else {
-        // Helper to read response once and store data for reuse.
-        // We copy the bytes into a Buffer that owns its own ArrayBuffer so the
-        // stored body cannot be invalidated by undici recycling its internal
-        // buffer after we consume the response.
-        const consumeResponse = (res) => __awaiter(void 0, void 0, void 0, function* () {
-            if (!res)
-                return null;
-            const ab = yield res.arrayBuffer();
-            const body = Buffer.alloc(ab.byteLength);
-            body.set(new Uint8Array(ab));
-            const text = body.toString("utf-8");
-            // Snapshot headers as a plain object so we can rebuild a fresh
-            // Headers instance per Response instead of sharing state.
-            const headers = new Headers(res.headers);
-            return { body, status: res.status, headers, ok: res.ok, text };
-        });
         // Helper to create Response from stored data. Pass the body as an
         // immutable string so each Response gets a fully independent backing
         // store; undici has been known to flag Responses as already-consumed
@@ -480,76 +602,187 @@ const makeRequest = (url, args) => __awaiter(void 0, void 0, void 0, function* (
             headers.delete("transfer-encoding");
             return new Response(bodyCopy, { status: data.status, headers });
         };
+        let oxylabsFallbackAttempted = false;
+        const maybeRetryBlockedResponse = (directData, headers) => __awaiter(void 0, void 0, void 0, function* () {
+            var _a, _b;
+            if (!directData || isCancelled())
+                return null;
+            if (!getOxylabsFallbackConfiguration().enabled)
+                return null;
+            const detection = detectBlockedResponse({
+                status: directData.status,
+                headers: directData.headers,
+                body: directData.text,
+            });
+            if (!detection.blocked)
+                return null;
+            const method = ((_a = requestOptions.method) !== null && _a !== void 0 ? _a : "GET").toUpperCase();
+            if (requestStartedWithProxy || oxylabsFallbackAttempted || !["GET", "HEAD"].includes(method)) {
+                return null;
+            }
+            oxylabsFallbackAttempted = true;
+            const proxy = claimOxylabsFallback(url);
+            if (!proxy || isCancelled())
+                return null;
+            if (reportErrors) {
+                progressLog(chalk.yellow(`[i] ${(_b = detection.provider) !== null && _b !== void 0 ? _b : "CDN/WAF"} block detected; retrying this request through Oxylabs`));
+            }
+            const dispatcher = buildUndiciDispatcher({ method: "oxylabs", oxylabs: proxy });
+            if (!dispatcher)
+                return null;
+            let fallbackData = null;
+            try {
+                fallbackData = yield singleFetch(url, Object.assign(Object.assign({}, requestOptions), { headers }), requestTimeout, false, dispatcher, 1);
+            }
+            finally {
+                const cleanup = isCancelled() || !fallbackData ? dispatcher.destroy() : dispatcher.close();
+                yield cleanup.catch(() => undefined);
+            }
+            if (!fallbackData || isCancelled())
+                return null;
+            const fallbackDetection = detectBlockedResponse({
+                status: fallbackData.status,
+                headers: fallbackData.headers,
+                body: fallbackData.text,
+            });
+            if (fallbackDetection.blocked || !fallbackData.ok) {
+                if (reportErrors) {
+                    progressError(chalk.yellow("[!] Oxylabs fallback did not bypass the CDN/WAF response"));
+                }
+                return null;
+            }
+            return fallbackData;
+        });
+        const finishPlainResponse = (data, headers) => __awaiter(void 0, void 0, void 0, function* () {
+            if (!(data === null || data === void 0 ? void 0 : data.ok))
+                return null;
+            const detection = detectBlockedResponse({
+                status: data.status,
+                headers: data.headers,
+                body: data.text,
+            });
+            if (detection.blocked)
+                return null;
+            if (!globals.getDisableCache()) {
+                yield writeCache(url, headers, createResponse(data), reportErrors, effectiveSignal);
+            }
+            return isCancelled() ? null : createResponse(data);
+        });
+        const tryBrowserRecovery = (blockedData, headers) => __awaiter(void 0, void 0, void 0, function* () {
+            if (!(blockedData === null || blockedData === void 0 ? void 0 : blockedData.ok) || isCancelled())
+                return null;
+            let browserContent;
+            try {
+                browserContent = yield handleFirewall(url, blockedData.text, { status: blockedData.status, headers: blockedData.headers }, effectiveSignal, reportErrors);
+            }
+            catch (error) {
+                if (reportErrors && !isCancelled()) {
+                    progressError(chalk.yellow(`[!] Browser firewall bypass failed for ${url}: ${(error === null || error === void 0 ? void 0 : error.message) || error}`));
+                }
+                return null;
+            }
+            if (!browserContent || isCancelled())
+                return null;
+            const browserDetection = detectBlockedResponse({
+                status: 200,
+                headers: { "content-type": "text/html" },
+                body: browserContent,
+            });
+            if (browserDetection.blocked) {
+                if (reportErrors) {
+                    progressError(chalk.yellow("[!] Browser fallback still returned a CDN/WAF challenge"));
+                }
+                return null;
+            }
+            const response = new Response(browserContent);
+            if (!globals.getDisableCache()) {
+                yield writeCache(url, headers, response, reportErrors, effectiveSignal);
+            }
+            return isCancelled() ? null : new Response(browserContent);
+        });
         // When using default headers, try both with and without Referer/Origin
         // to handle servers that return 404 for one but 200 for the other
         if (usingDefaultHeaders) {
             // First try: with Referer and Origin
             const headersWithReferer = Object.assign(Object.assign({}, baseHeaders), { Referer: new URL(url).origin, Origin: new URL(url).origin });
-            const resWithReferer = yield singleFetch(url, Object.assign(Object.assign({}, requestOptions), { headers: headersWithReferer }), requestTimeout);
-            const dataWithReferer = yield consumeResponse(resWithReferer);
-            if (dataWithReferer && dataWithReferer.ok) {
-                // Check for firewall
-                const firewallContent = yield handleFirewall(url, dataWithReferer.text);
-                if (firewallContent) {
-                    if (!globals.getDisableCache()) {
-                        yield writeCache(url, headersWithReferer, new Response(firewallContent));
-                    }
-                    return new Response(firewallContent);
-                }
-                // Cache and return successful response
-                if (!globals.getDisableCache()) {
-                    yield writeCache(url, headersWithReferer, createResponse(dataWithReferer));
-                }
-                return createResponse(dataWithReferer);
-            }
+            const directDataWithReferer = yield singleFetch(url, Object.assign(Object.assign({}, requestOptions), { headers: headersWithReferer }), requestTimeout, false);
+            if (isCancelled())
+                return null;
+            const withRefererResponse = yield finishPlainResponse(directDataWithReferer, headersWithReferer);
+            if (withRefererResponse)
+                return withRefererResponse;
             // Second try: without Referer and Origin
             const headersWithoutReferer = Object.assign({}, baseHeaders);
-            const resWithoutReferer = yield singleFetch(url, Object.assign(Object.assign({}, requestOptions), { headers: headersWithoutReferer }), requestTimeout);
-            const dataWithoutReferer = yield consumeResponse(resWithoutReferer);
-            if (dataWithoutReferer && dataWithoutReferer.ok) {
-                // Check for firewall
-                const firewallContent = yield handleFirewall(url, dataWithoutReferer.text);
-                if (firewallContent) {
-                    if (!globals.getDisableCache()) {
-                        yield writeCache(url, headersWithoutReferer, new Response(firewallContent));
-                    }
-                    return new Response(firewallContent);
-                }
-                // Cache and return successful response
-                if (!globals.getDisableCache()) {
-                    yield writeCache(url, headersWithoutReferer, createResponse(dataWithoutReferer));
-                }
-                return createResponse(dataWithoutReferer);
+            const directDataWithoutReferer = yield singleFetch(url, Object.assign(Object.assign({}, requestOptions), { headers: headersWithoutReferer }), requestTimeout, reportErrors && !directDataWithReferer);
+            if (isCancelled())
+                return null;
+            const withoutRefererResponse = yield finishPlainResponse(directDataWithoutReferer, headersWithoutReferer);
+            if (withoutRefererResponse)
+                return withoutRefererResponse;
+            const candidates = [
+                { data: directDataWithoutReferer, headers: headersWithoutReferer },
+                { data: directDataWithReferer, headers: headersWithReferer },
+            ];
+            const blockedCandidate = candidates.find(({ data }) => data &&
+                detectBlockedResponse({ status: data.status, headers: data.headers, body: data.text }).blocked);
+            if (blockedCandidate) {
+                const fallbackData = yield maybeRetryBlockedResponse(blockedCandidate.data, blockedCandidate.headers);
+                if (isCancelled())
+                    return null;
+                const fallbackResponse = yield finishPlainResponse(fallbackData, blockedCandidate.headers);
+                if (fallbackResponse)
+                    return fallbackResponse;
+                const browserResponse = yield tryBrowserRecovery(blockedCandidate.data, blockedCandidate.headers);
+                if (browserResponse)
+                    return browserResponse;
             }
-            // Both failed, return whichever response we got (prefer non-null)
-            const finalData = dataWithReferer || dataWithoutReferer;
-            if (finalData) {
-                if (!globals.getDisableCache()) {
-                    yield writeCache(url, headersWithReferer, createResponse(finalData));
+            // Prefer the most recent non-blocked response, then any direct response.
+            const finalCandidate = (_e = candidates.find(({ data }) => data &&
+                !detectBlockedResponse({ status: data.status, headers: data.headers, body: data.text }).blocked)) !== null && _e !== void 0 ? _e : candidates.find(({ data }) => data !== null);
+            if (finalCandidate === null || finalCandidate === void 0 ? void 0 : finalCandidate.data) {
+                const finalData = finalCandidate.data;
+                const finalDetection = detectBlockedResponse({
+                    status: finalData.status,
+                    headers: finalData.headers,
+                    body: finalData.text,
+                });
+                if (!globals.getDisableCache() && !finalDetection.blocked) {
+                    yield writeCache(url, finalCandidate.headers, createResponse(finalData), reportErrors, effectiveSignal);
                 }
+                if (isCancelled())
+                    return null;
                 return createResponse(finalData);
             }
             return null;
         }
         else {
             // Custom headers provided, use them directly
-            const res = yield singleFetch(url, requestOptions, requestTimeout);
-            const data = yield consumeResponse(res);
-            if (!data)
+            const directData = yield singleFetch(url, requestOptions, requestTimeout, reportErrors);
+            if (!directData || isCancelled())
                 return null;
-            // Check for firewall
-            const firewallContent = yield handleFirewall(url, data.text);
-            if (firewallContent) {
-                if (!globals.getDisableCache()) {
-                    yield writeCache(url, requestOptions.headers || {}, new Response(firewallContent));
-                }
-                return new Response(firewallContent);
-            }
-            // Cache and return
-            if (!globals.getDisableCache()) {
-                yield writeCache(url, requestOptions.headers || {}, createResponse(data));
+            const directResponse = yield finishPlainResponse(directData, requestOptions.headers || {});
+            if (directResponse)
+                return directResponse;
+            const fallbackData = yield maybeRetryBlockedResponse(directData, requestOptions.headers || {});
+            if (isCancelled())
+                return null;
+            const fallbackResponse = yield finishPlainResponse(fallbackData, requestOptions.headers || {});
+            if (fallbackResponse)
+                return fallbackResponse;
+            const browserResponse = yield tryBrowserRecovery(directData, requestOptions.headers || {});
+            if (browserResponse)
+                return browserResponse;
+            const directDetection = detectBlockedResponse({
+                status: directData.status,
+                headers: directData.headers,
+                body: directData.text,
+            });
+            if (!globals.getDisableCache() && !directDetection.blocked) {
+                yield writeCache(url, requestOptions.headers || {}, createResponse(directData), reportErrors, effectiveSignal);
             }
-            return createResponse(data);
+            if (isCancelled())
+                return null;
+            return createResponse(directData);
         }
     }
 });