@shriyanss/js-recon 1.0.0 → 1.1.0-beta.2

package/utility/makeReq.js

@@ -1,179 +0,0 @@
-import chalk from "chalk";
-import puppeteer from "puppeteer";
-import * as globals from "./globals.js";
-import { get } from "../api_gateway/genReq.js";
-
-// random user agents
-const UAs = [
-  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Chrome/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Chrome/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Chrome/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
-  "Chrome/iPhone (request desktop): Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87 Version/11.1.1 Safari/605.1.15",
-  "Chrome/iPad: Mozilla/5.0 (iPad; CPU OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
-  "Chrome/iPod: Mozilla/5.0 (iPod; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
-  "Chrome/Android: Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
-  "Chrome/Android: Mozilla/5.0 (Linux; Android 10; SM-A205U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
-  "Chrome/Android: Mozilla/5.0 (Linux; Android 10; LM-Q720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
-  "Firefox/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
-  "Firefox/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:84.0) Gecko/20100101 Firefox/84.0",
-  "Firefox/Linux: Mozilla/5.0 (X11; Linux i686; rv:84.0) Gecko/20100101 Firefox/84.0",
-  "Firefox/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 11_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
-  "Firefox/iPad: Mozilla/5.0 (iPad; CPU OS 11_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
-  "Firefox/Android: Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/84.0",
-  "Safari/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15",
-  "Safari/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
-  "Safari/iPhone (request desktop): Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
-  "Safari/iPad: Mozilla/5.0 (iPad; CPU OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
-  "IE11/Windows: Mozilla/5.0 (Windows NT 10.0; Trident/7.0; rv:11.0) like Gecko",
-  "Edge/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
-  "Edge/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
-  "Edge/Android: Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36 EdgA/45.12.4.5121",
-  "Edge/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 EdgiOS/45.11.11 Mobile/15E148 Safari/605.1.15",
-  "Opera/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
-  "Opera/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
-  "Opera/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
-  "Opera/Android: Mozilla/5.0 (Linux; Android 10; VOG-L29) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36 OPR/61.1.3076.56625",
-  "Vivaldi/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
-  "Vivaldi/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
-  "Vivaldi/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
-  "Yandex/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 YaBrowser/20.12.0 Yowser/2.5 Safari/537.36",
-  "Yandex/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 YaBrowser/20.12.0 Yowser/2.5 Safari/537.36",
-  "Yandex/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 YaBrowser/20.11.2.199 Mobile/15E148 Safari/604.1",
-  "Yandex/Android: Mozilla/5.0 (Linux; arm_64; Android 11; SM-G965F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 YaBrowser/20.12.29.180 Mobile Safari/537.36",
-  "Chrome/ChromeOS: Mozilla/5.0 (X11; CrOS x86_64 13505.63.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Safari/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15",
-  "Firefox/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:84.0) Gecko/20100101 Firefox/84.0",
-  "Chrome/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Vivaldi/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
-  "Edge/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
-  "Safari/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
-  "Chrome/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
-  "Firefox/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
-  "Edge/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
-  "Internet-Explorer/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
-  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-  "Firefox/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
-  "Vivaldi/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
-  "Chrome/Android: Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
-  "Firefox/Android: Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/84.0",
-];
-
-const makeRequest = async (url, args) => {
-  if (globals.useApiGateway) {
-    let get_headers;
-    if (args && args.headers) {
-      get_headers = args.headers;
-    } else {
-      get_headers = {
-        "User-Agent": UAs[Math.floor(Math.random() * UAs.length)],
-        Accept:
-          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Sec-Fetch-Site": "same-origin",
-        "Sec-Fetch-Mode": "cors",
-        "Sec-Fetch-Dest": "empty",
-        Referer: url,
-        Origin: url,
-      };
-    }
-    
-    const body = await get(url, get_headers);
-
-    // craft a Response, and return that
-    const response = new Response(body);
-    return response;
-  } else {
-    if (args === undefined) {
-      args = {
-        headers: {
-          "User-Agent": UAs[Math.floor(Math.random() * UAs.length)],
-          Accept:
-            "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-          "Accept-Language": "en-US,en;q=0.9",
-          "Sec-Fetch-Site": "same-origin",
-          "Sec-Fetch-Mode": "cors",
-          "Sec-Fetch-Dest": "empty",
-          Referer: url,
-          Origin: url,
-        },
-      };
-    }
-    let res;
-    let counter = 0;
-    while (true) {
-      try {
-        res = await fetch(url, args);
-        if (res) {
-          break;
-        }
-      } catch (err) {
-        counter++;
-        if (counter > 10) {
-          console.log(chalk.red(`[!] Failed to fetch ${url}`));
-          return null;
-        }
-        // sleep 0.5 s before retrying
-        await new Promise((resolve) => setTimeout(resolve, 500));
-        continue;
-      }
-    }
-
-    const preservedRes = res.clone();
-
-    // check if this is a firewall
-    // CF first
-    const resp_text = await res.text();
-    if (resp_text.includes("/?bm-verify=")) {
-      console.log(
-        chalk.yellow(
-          `[!] CF Firewall detected. Trying to bypass with headless browser`,
-        ),
-      );
-      // if it is, load it in a headless browser
-      const browser = await puppeteer.launch({
-        headless: true,
-        args: [
-          "--disable-gpu",
-          "--disable-dev-shm-usage",
-          "--disable-setuid-sandbox",
-          "--no-sandbox",
-        ],
-      });
-      const page = await browser.newPage();
-      await page.goto(url);
-      await new Promise((resolve) => setTimeout(resolve, 5000));
-      const content = await page.content();
-      await browser.close();
-      return new Response(content);
-    } else if (resp_text.includes("<title>Just a moment...</title>")) {
-      console.log(
-        chalk.yellow(
-          `[!] CF Firewall detected. Trying to bypass with headless browser`,
-        ),
-      );
-      // if it is, load it in a headless browser
-      const browser = await puppeteer.launch({
-        headless: true,
-        args: [
-          "--disable-gpu",
-          "--disable-dev-shm-usage",
-          "--disable-setuid-sandbox",
-          "--no-sandbox",
-        ],
-      });
-      const page = await browser.newPage();
-      await page.goto(url);
-      await new Promise((resolve) => setTimeout(resolve, 5000));
-      const content = await page.content();
-      await browser.close();
-      return new Response(content);
-    }
-
-    return preservedRes;
-  }
-};
-
-export default makeRequest;

package/utility/resolvePath.js

@@ -1,43 +0,0 @@
-/**
- * Resolves a given path against a base URL using the URL constructor.
- *
- * The function handles various cases of path resolution:
- * - If the base URL does not end with a '/', its last segment is treated as a "file",
- *   and relative paths are resolved from its "directory".
- * - Examples:
- *   - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
- *     (Base for resolution becomes 'https://site.com/')
- *   - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
- *     (Base for resolution is 'https://site.com/something/')
- *   - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
- *     (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
- *
- * @param {string} url - The base URL to resolve against.
- * @param {string} path - The path to resolve.
- * @returns {Promise<string>} - A promise that resolves to the fully resolved URL as a string.
- * @throws Will throw an error if the resolution fails.
- */
-const resolvePath = async (url, path) => {
-  try {
-    // The URL constructor handles various cases of path resolution.
-    // If 'url' (the base URL) does not end with a '/', its last path segment
-    // is typically treated as a "file", and relative paths are resolved
-    // from the "directory" containing that "file".
-    // This behavior aligns with the provided examples:
-    // - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
-    //   (Base for resolution becomes 'https://site.com/')
-    // - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
-    //   (Base for resolution is 'https://site.com/something/')
-    // - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
-    //   (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
-    const resolvedUrl = new URL(path, url);
-    return resolvedUrl.href;
-  } catch (e) {
-    console.error(`Error resolving path "${path}" with base URL "${url}": ${e.message}`);
-    // Rethrowing the error to signal failure to the caller.
-    // Alternative error handling (e.g., returning null) can be implemented if required.
-    throw e;
-  }
-};
-
-export default resolvePath;