ani-auto 1.4.4 → 2.0.0

package/anime/utils/requestManager.js

@@ -0,0 +1,706 @@
+const { launchBrowser } = require("./browser");
+const cloudscraper = require("cloudscraper");
+const axios = require("axios");
+const { execFile } = require("child_process");
+const Config = require("./config");
+const flaresolverr = require("./flaresolverr");
+const { CustomError } = require("../middleware/errorHandler");
+
+class RequestManager {
+  /**
+   * Domains accessible directly from local machine (no FlareSolverr needed)
+   */
+  static LOCAL_DOMAINS = ["kwik", "uwucdn", "uwu", "owocdn"];
+
+  /**
+   * Check if a URL should be fetched directly (not through FlareSolverr)
+   */
+  static _isLocalDomain(url) {
+    try {
+      const host = new URL(url).hostname.toLowerCase();
+      return this.LOCAL_DOMAINS.some(d => host.includes(d));
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Execute a request via system curl with cf_clearance cookie.
+   * Matches KevCui's proven approach: plain curl + cf_clearance + UA + --compressed.
+   */
+  static _curlGet(url, options = {}) {
+    const { cfClearance, userAgent, referer, timeout = 15000 } = options;
+
+    return new Promise((resolve, reject) => {
+      const args = ['-sS', '-L', '--compressed'];
+
+      if (cfClearance) {
+        args.push('-b', `cf_clearance=${cfClearance}`);
+      }
+      if (userAgent) {
+        args.push('-A', userAgent);
+      }
+      if (referer) {
+        args.push('-e', referer);
+      }
+
+      args.push('-w', '\n%{http_code}');
+      args.push(url);
+
+      execFile('curl', args, { timeout, maxBuffer: 10 * 1024 * 1024 }, (err, stdout) => {
+        if (err) {
+          return reject(new Error(`curl failed: ${err.message}`));
+        }
+
+        const lines = stdout.trim().split('\n');
+        const statusCode = parseInt(lines.pop(), 10);
+        const body = lines.join('\n');
+
+        resolve({ body, status: statusCode });
+      });
+    });
+  }
+
+  /**
+   * Universal cloudscraper method - handles GET, POST, and any HTTP method
+   * @param {Object} options - Request options
+   * @param {string} options.method - HTTP method (GET, POST, etc.)
+   * @param {string} options.url - Target URL
+   * @param {Object} options.headers - Custom headers
+   * @param {Object} options.form - Form data (for POST)
+   * @param {Object} options.json - JSON body (for POST)
+   * @param {boolean} options.followRedirect - Follow redirects (default: true)
+   * @param {number} options.timeout - Request timeout in ms
+   * @param {string} options.referer - Referer header
+   * @returns {Promise<Object>} Response with { statusCode, headers, body }
+   */
+  static async cloudscraperRequest(options = {}) {
+    const {
+      method = "GET",
+      url,
+      headers = {},
+      form = null,
+      json = null,
+      encoding,
+      followRedirect = true,
+      followAllRedirects = false,
+      timeout = 8000,
+      referer = Config.getUrl("home"),
+      resolveWithFullResponse = true,
+      simple = false,
+    } = options;
+
+    if (!url) {
+      throw new CustomError("URL is required for cloudscraper request", 400);
+    }
+
+    if (flaresolverr.isEnabled() && !this._isLocalDomain(url)) {
+      const result =
+        (method || "GET").toUpperCase() === "POST"
+          ? await flaresolverr.post(url, form || json || {})
+          : await flaresolverr.get(url);
+      return {
+        statusCode: result.status,
+        headers: {},
+        body: result.body,
+      };
+    }
+
+    const defaultHeaders = {
+      "User-Agent": Config.userAgent,
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.5",
+      Referer: referer,
+      Connection: "keep-alive",
+      "Upgrade-Insecure-Requests": "1",
+      DNT: "1",
+      "Sec-Fetch-Dest": "document",
+      "Sec-Fetch-Mode": "navigate",
+      "Sec-Fetch-Site": "cross-site",
+    };
+
+    const requestOptions = {
+      method,
+      uri: url,
+      headers: { ...defaultHeaders, ...headers },
+      followRedirect,
+      followAllRedirects,
+      simple,
+      resolveWithFullResponse,
+      timeout,
+    };
+    if (encoding !== undefined) {
+      requestOptions.encoding = encoding;
+    }
+
+    // Add body data if provided
+    if (form) {
+      requestOptions.form = form;
+      requestOptions.headers["Content-Type"] =
+        "application/x-www-form-urlencoded";
+    } else if (json) {
+      requestOptions.json = json;
+      requestOptions.headers["Content-Type"] = "application/json";
+    }
+
+    try {
+      console.log(`[Cloudscraper] ${method} ${url}`);
+      const response = await cloudscraper(requestOptions);
+
+      return {
+        statusCode: response.statusCode,
+        headers: response.headers,
+        body: response.body,
+      };
+    } catch (error) {
+      // Handle redirects as responses (not errors)
+      if (error.statusCode === 301 || error.statusCode === 302) {
+        return {
+          statusCode: error.statusCode,
+          headers: error.response?.headers || {},
+          body: error.response?.body || "",
+          location: error.response?.headers?.location,
+        };
+      }
+
+      console.error(`[Cloudscraper Error] ${method} ${url}:`, error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Simplified GET request with cloudscraper
+   */
+  static async cloudscraperGet(url, options = {}) {
+    return this.cloudscraperRequest({
+      method: "GET",
+      url,
+      ...options,
+    });
+  }
+
+  /**
+   * Simplified POST request with cloudscraper
+   */
+  static async cloudscraperPost(url, data = {}, options = {}) {
+    const isJson = options.json !== false;
+
+    return this.cloudscraperRequest({
+      method: "POST",
+      url,
+      ...(isJson ? { json: data } : { form: data }),
+      ...options,
+    });
+  }
+
+  static async fetch(url, cookieHeader, type = "default") {
+    if (type === "default") {
+      return this.fetchApiData(url, {}, cookieHeader);
+    } else if (type === "heavy") {
+      return this.scrapeWithPlaywright(url);
+    } else {
+      console.trace(
+        'Invalid fetch type specified. Please use "json", "heavy", or "default".',
+      );
+      return null;
+    }
+  }
+
+  /**
+   * Fetch a play page: try cookies + axios first, fall back to FlareSolverr.
+   *
+   * WHY THIS ORDER:
+   * FlareSolverr launches a full Chrome browser per request (~0.5–2s each).
+   * Cloudflare rate-limits after ~5 rapid browser-based requests to
+   * animepahe.pw/play/..., causing subsequent episodes to fail.
+   *
+   * Cookies + axios (~0.5s) don't trigger Turnstile, so we try that FIRST.
+   * Only when cookies fail (expired/missing) do we call FlareSolverr,
+   * then save the fresh cookies + User-Agent for next time.
+   *
+   * @param {string} url - Target URL on animepahe.pw
+   * @returns {Promise<string>} HTML response body
+   */
+  static async fetchWithCookies(url) {
+    if (Config.cfClearance) {
+      try {
+        const { body, status } = await this._curlGet(url, {
+          cfClearance: Config.cfClearance,
+          userAgent: Config.userAgent,
+          referer: Config.getUrl("home"),
+          timeout: 15000,
+        });
+
+        if (
+          body &&
+          body.length > 100 &&
+          !body.toLowerCase().includes("just a moment") &&
+          !body.toLowerCase().includes("checking your browser") &&
+          !body.toLowerCase().includes("ddos protection by cloudflare")
+        ) {
+          return body;
+        }
+
+        console.log(
+          "[fetchWithCookies] cf_clearance returned challenge page, falling back to FlareSolverr",
+        );
+      } catch (err) {
+        console.log(
+          `[fetchWithCookies] curl request: ${err.message}, falling back to FlareSolverr`,
+        );
+      }
+    }
+
+    if (flaresolverr.isEnabled()) {
+      const result = await flaresolverr.get(url);
+      this._saveFlareResult(result);
+      return result.body;
+    }
+
+    throw new CustomError(
+      Config.cfClearance
+        ? "Cookies failed and FlareSolverr not enabled"
+        : "No cf_clearance available",
+      503,
+    );
+  }
+
+  /**
+   * Save cf_clearance and User-Agent from a FlareSolverr result.
+   */
+  static _saveFlareResult(result) {
+    if (result.cookies && result.cookies.length > 0) {
+      const cf = result.cookies.find((c) => c.name === "cf_clearance");
+      if (cf && cf.value) {
+        Config.setCfClearance(cf.value);
+        console.log("[_saveFlareResult] Saved cf_clearance from FlareSolverr");
+      }
+    }
+    if (result.userAgent) {
+      Config.userAgent = result.userAgent;
+    }
+  }
+
+  /**
+   * Legacy method - now uses the universal cloudscraper method
+   */
+  static async scrapeWithCloudScraper(url, options = {}) {
+    console.log(`Fetching HTML from ${url}...`);
+
+    if (flaresolverr.isEnabled()) {
+      const { body } = await flaresolverr.get(url);
+      return body;
+    }
+
+    const response = await this.cloudscraperGet(url, {
+      headers: {
+        Referer: Config.baseUrl,
+        "Accept-Encoding": "gzip, deflate, br",
+        dnt: "1",
+        "sec-ch-ua":
+          '"Not A(Brand";v="99", "Microsoft Edge";v="121", "Chromium";v="121"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-origin",
+        "x-requested-with": "XMLHttpRequest",
+        ...options.headers,
+      },
+      timeout: options.timeout || 20000,
+    });
+
+    return response.body;
+  }
+
+  static async scrapeWithPlaywright(url) {
+    console.log("Fetching content from:", url);
+
+    if (flaresolverr.isEnabled()) {
+      const { body } = await flaresolverr.get(url);
+      return body;
+    }
+
+    const proxy = Config.proxyEnabled ? Config.getRandomProxy() : null;
+    console.log(`Using proxy: ${proxy || "none"}`);
+
+    const GLOBAL_TIMEOUT = 120000; // 120s total for Playwright operations (DDoS challenges can be slow)
+
+    const browser = await launchBrowser();
+
+    try {
+      const contextOptions = {};
+
+      if (proxy) {
+        contextOptions.proxy = { server: proxy };
+      }
+
+      const context = await browser.newContext(contextOptions);
+      const page = await context.newPage();
+
+      // Stealth measures
+      await page.addInitScript(() => {
+        delete navigator.__proto__.webdriver;
+        Object.defineProperty(navigator, "plugins", {
+          get: () => [1, 2, 3],
+        });
+        Object.defineProperty(navigator, "languages", {
+          get: () => ["en-US", "en"],
+        });
+
+        const originalQuery = window.navigator.permissions.query;
+        window.navigator.permissions.query = (parameters) =>
+          parameters.name === "notifications"
+            ? Promise.resolve({ state: Notification.permission })
+            : originalQuery(parameters);
+      });
+
+      // Realistic headers
+      await page.setExtraHTTPHeaders({
+        "User-Agent": Config.userAgent,
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        Referer: "https://www.google.com/",
+        "Cache-Control": "no-cache",
+      });
+
+      console.log("Navigating to URL...");
+
+      // Wrap entire operation with a global timeout
+      const content = await Promise.race([
+        (async () => {
+          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 120000 });
+
+          await page.waitForTimeout(3000); // DDoS challenge buffer (reduced from 10s)
+
+          const isApiRequest = url.includes("/api") || url.endsWith(".json");
+
+          if (!isApiRequest) {
+            try {
+              await page.waitForSelector(".episode-wrap, .episode-list", {
+                timeout: 60000,
+              });
+            } catch (e) {
+              console.log("Selector not found, continuing...");
+            }
+          } else {
+            try {
+              await page.waitForFunction(
+                () => {
+                  const text = document.body.textContent;
+                  return text.includes("{") && text.includes("}");
+                },
+                { timeout: 60000 },
+              );
+            } catch (e) {
+              console.log("API content not found, continuing...");
+            }
+          }
+
+          return await page.content();
+        })(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error(`Playwright operation timed out after ${GLOBAL_TIMEOUT / 1000}s`)), GLOBAL_TIMEOUT)),
+      ]);
+
+      return content;
+    } finally {
+      await browser.close();
+    }
+  }
+
+  static async fetchJson(url) {
+    if (flaresolverr.isEnabled()) {
+      const { body } = await flaresolverr.get(url);
+      return flaresolverr.extractJson(body);
+    }
+
+    const html = await this.fetch(url);
+
+    try {
+      // Try to parse the content as JSON
+      const jsonMatch =
+        html.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i) ||
+        html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+
+      if (jsonMatch) {
+        try {
+          return JSON.parse(jsonMatch[1].trim());
+        } catch (e) {
+          console.log(
+            "Failed to parse JSON from matched content, trying whole page",
+          );
+          return JSON.parse(html);
+        }
+      } else {
+        return JSON.parse(html);
+      }
+    } catch (error) {
+      console.error("Failed to parse JSON:", error.message);
+      throw new Error(`Failed to parse JSON from ${url}: ${error.message}`);
+    }
+  }
+
+  static async rawRequest(url, options = {}) {
+    try {
+      return await axios.get(url, options);
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  static async fetchCloudflareProtected(url, options = {}) {
+    console.log("Fetching Cloudflare-protected content from:", url);
+
+    if (flaresolverr.isEnabled()) {
+      const { body } = await flaresolverr.get(url);
+      return body;
+    }
+
+    const proxy = Config.proxyEnabled ? Config.getRandomProxy() : null;
+    console.log(`Using proxy: ${proxy || "none"}`);
+
+    const browser = await launchBrowser();
+
+    try {
+      const contextOptions = {
+        userAgent: Config.userAgent,
+        viewport: { width: 1920, height: 1080 },
+        extraHTTPHeaders: {
+          Accept:
+            "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+          "Accept-Language": "en-US,en;q=0.9",
+          "Accept-Encoding": "gzip, deflate, br",
+          "Cache-Control": "no-cache",
+          Pragma: "no-cache",
+          "Sec-Fetch-Dest": "document",
+          "Sec-Fetch-Mode": "navigate",
+          "Sec-Fetch-Site": "cross-site",
+          "Sec-Fetch-User": "?1",
+          "Upgrade-Insecure-Requests": "1",
+          Referer: options.referer || Config.getUrl("home"),
+        },
+      };
+
+      if (proxy) {
+        contextOptions.proxy = { server: proxy };
+      }
+
+      const context = await browser.newContext(contextOptions);
+      const page = await context.newPage();
+
+      await page.addInitScript(() => {
+        Object.defineProperty(navigator, "webdriver", { get: () => false });
+        Object.defineProperty(navigator, "plugins", {
+          get: () => [1, 2, 3, 4, 5],
+        });
+        Object.defineProperty(navigator, "languages", {
+          get: () => ["en-US", "en"],
+        });
+
+        // Add chrome object
+        window.chrome = {
+          runtime: {},
+          loadTimes: function () {},
+          csi: function () {},
+          app: {},
+        };
+
+        // Mock permissions
+        const originalQuery = window.navigator.permissions.query;
+        window.navigator.permissions.query = (parameters) =>
+          parameters.name === "notifications"
+            ? Promise.resolve({ state: Notification.permission })
+            : originalQuery(parameters);
+      });
+
+      console.log("Navigating to URL...");
+      await page.goto(url, {
+        waitUntil: "domcontentloaded",
+        timeout: 60000,
+      });
+
+      // Handle Cloudflare challenge
+      await this.handleCloudflareChallenge(page);
+
+      const content = await page.content();
+      await context.close();
+
+      return content;
+    } finally {
+      await browser.close();
+    }
+  }
+
+  static async handleCloudflareChallenge(page) {
+    console.log("Checking for Cloudflare challenge...");
+
+    // for any immediate redirects
+    await page.waitForTimeout(3000);
+
+    // Check for various challenge indicators
+    const challengeSelectors = [
+      "#cf-challenge-running",
+      ".cf-challenge-form",
+      "[data-ray]", // Cloudflare Ray ID
+      'title:has-text("Just a moment")',
+      'h1:has-text("Please wait")',
+      'div:has-text("Checking your browser")',
+      'div:has-text("DDoS protection")',
+    ];
+
+    let challengeFound = false;
+    for (const selector of challengeSelectors) {
+      try {
+        const element = await page.$(selector);
+        if (element) {
+          challengeFound = true;
+          console.log(`Challenge detected with selector: ${selector}`);
+          break;
+        }
+      } catch (e) {
+        // Continue checking other selectors
+      }
+    }
+
+    if (challengeFound || page.url().includes("cdn-cgi/challenge")) {
+      console.log("Cloudflare challenge detected, waiting for resolution...");
+
+      try {
+        await page.waitForFunction(
+          () => {
+            const title = document.title.toLowerCase();
+            const url = window.location.href;
+            const bodyText = document.body.textContent.toLowerCase();
+
+            // Check if we're no longer on a challenge page
+            return (
+              !title.includes("just a moment") &&
+              !title.includes("please wait") &&
+              !url.includes("cdn-cgi/challenge") &&
+              !bodyText.includes("checking your browser") &&
+              !bodyText.includes("ddos protection")
+            );
+          },
+          { timeout: 30000 },
+        );
+
+        console.log("Cloudflare challenge resolved successfully");
+
+        // Additional wait to ensure page is fully loaded
+        await page.waitForTimeout(2000);
+      } catch (timeoutError) {
+        console.warn("Challenge resolution timeout - proceeding anyway");
+        // Don't throw error, let the caller handle the response
+      }
+    } else {
+      console.log("No Cloudflare challenge detected");
+    }
+  }
+
+  static async fetchApiData(url, params = {}, cfClearance) {
+    cfClearance = cfClearance || Config.cfClearance || null;
+
+    const qs = new URLSearchParams(params).toString();
+    const fullUrl = qs ? url + '?' + qs : url;
+
+    if (cfClearance) {
+      try {
+        const { body, status } = await this._curlGet(fullUrl, {
+          cfClearance,
+          userAgent: Config.userAgent,
+          referer: Config.getUrl("home"),
+          timeout: 15000,
+        });
+
+        if (
+          status === 403 ||
+          body.includes("DDoS-GUARD") ||
+          body.includes("checking your browser") ||
+          body.includes("Just a moment")
+        ) {
+          console.log("[fetchApiData] cf_clearance failed (challenge page)");
+        } else {
+          return typeof body === "string" ? body : JSON.stringify(body);
+        }
+      } catch (err) {
+        console.log(`[fetchApiData] curl request: ${err.message}`);
+      }
+    }
+
+    if (flaresolverr.isEnabled()) {
+      const target = new URL(url);
+      Object.keys(params).forEach((key) => {
+        target.searchParams.append(key, params[key]);
+      });
+      const result = await flaresolverr.get(target.toString());
+      this._saveFlareResult(result);
+      if (!url.includes("/api")) return result.body;
+      return flaresolverr.extractJson(result.body);
+    }
+
+    throw new CustomError(
+      cfClearance
+        ? "cf_clearance failed and FlareSolverr not enabled"
+        : "No cf_clearance available",
+      503,
+    );
+  }
+
+  /**
+   * Browser-based API fetch - uses Playwright instead of axios
+   * Used when cookies fail and we need to bypass DDoS-Guard
+   */
+  static async fetchApiDataWithBrowser(url, params) {
+    console.log(`[fetchApiDataWithBrowser] Using Playwright for: ${url}`);
+
+    if (flaresolverr.isEnabled()) {
+      const target = new URL(url);
+      Object.keys(params).forEach((key) => {
+        target.searchParams.append(key, params[key]);
+      });
+      const { body } = await flaresolverr.get(target.toString());
+      return flaresolverr.extractJson(body);
+    }
+    
+    const { launchBrowser } = require("./browser");
+    const browser = await launchBrowser();
+    
+    try {
+      const context = await browser.newContext({
+        userAgent: Config.userAgent,
+      });
+      
+      const page = await context.newPage();
+      
+      // Build full URL with params
+      const fullUrl = new URL(url);
+      Object.keys(params).forEach(key => {
+        fullUrl.searchParams.append(key, params[key]);
+      });
+      
+      await page.goto(fullUrl.toString(), {
+        waitUntil: "domcontentloaded",
+        timeout: 60000,
+      });
+      
+      // Wait for JSON response
+      await page.waitForFunction(() => {
+        const text = document.body.textContent;
+        return text.trim().startsWith('{') || text.trim().startsWith('[');
+      }, { timeout: 15000 });
+      
+      const content = await page.textContent();
+      const data = JSON.parse(content.trim());
+      
+      await context.close();
+      return data;
+      
+    } finally {
+      await browser.close();
+    }
+  }
+}
+
+module.exports = RequestManager;