@shriyanss/js-recon 1.0.0

package/research/nuxt_js.md

@@ -0,0 +1,125 @@
+# Nuxt.js Research
+## Tech Detection
+Nuxt.js is a framework based on Vue.js. So, if the Vue.js is detected, then it is essential to detect Nuxt.js for accuracy.
+
+To detect Nuxt.js, following technique(s) can be used: 
+- Search for "/_nuxt" in following attributes
+    - `src`, `href`
+
+## Lazy Loaded Files
+### Analysis of [GitLab's About Site](https://about.gitlab.com/)
+It was found that all of the JS files were loaded from the `/_nuxt-new` directory instead of `/_nuxt` directory. The path to the JS files were located in the `<link>` tags' `href` attribute. These `<link>` tags had the value of `rel` attribute as `modulepreload`, and the values of `as` attribute as `script`.
+
+Additionally, there were some `<script>` tags found with `src` attribute as the path to the JS files, which started with `/_nuxt`.
+
+### Analysis of [Vue Mastery](https://www.vuemastery.com/)
+Similar to Gitlab's About Page, Vue Mastery had `<link>` tags with `rel` attribute as `preload` and `href` attribute as the path to the JS files. These `<link>` tags had the value of `as` attribute as `script`, which is common between two sites.
+
+Additionally, there were some `<script>` tags found with `src` attribute as the path to the JS files, which started with `/_nuxt`.
+
+Upon analysis of JS files that can be downloaded with the above two methods, it was found that some of the JS files contained functions to generate dynamic JS files based on the function. One of such function was found in https://www.vuemastery.com/_nuxt/2551e78.js, which was present in a `<script>` tag on the home page source. The function responsible for generating dynamic JS files is:
+```js
+f.p = "/_nuxt/",
+// --snip--
+script.src = function(e) {
+    return f.p + "" + {
+        0: "dad8928",
+        1: "45a7974",
+        2: "4632233",
+        3: "1b5c8a7",
+        4: "605bec3",
+        5: "b2d6888",
+        6: "6f126a6",
+        7: "0660b70",
+        10: "b2c4841",
+        11: "dd7a220",
+        12: "f9f7852",
+        13: "4204cef",
+        14: "6b95651",
+        15: "d23fd5a",
+        16: "674fd56",
+        17: "e43e7d0",
+        18: "6745258",
+        19: "6ea1659",
+        20: "ebdc343",
+        21: "b82338f",
+        22: "6b6976c",
+        23: "3652e0c",
+        24: "ab7b2c1",
+        25: "4d1b31e",
+        26: "4dadfea",
+        27: "e3dff7a",
+        28: "cc26714",
+        29: "5fda719",
+        30: "cae7d03",
+        31: "23f9ebd",
+        32: "8e6c9c2",
+        33: "fe28953",
+        34: "d31580e",
+        35: "da95621",
+        36: "edc6dfe",
+        37: "5a045a0",
+        38: "50dfb9c",
+        39: "6ef34b4",
+        40: "102f187",
+        41: "c1d8d4f",
+        42: "4a0b48e",
+        43: "635c550",
+        44: "0266231",
+        45: "8fd697f",
+        46: "5107115",
+        47: "608af85",
+        48: "500d25b",
+        49: "da3579b",
+        50: "06ee20b",
+        51: "43e16e2",
+        52: "4c13496",
+        53: "a1eb1a8",
+        54: "141dbb5",
+        55: "c9cbc38",
+        56: "7306da6",
+        57: "415103c",
+        58: "57bbbbd",
+        59: "1963258",
+        62: "4fca0c9",
+        63: "93260d7",
+        64: "55a6cb8",
+        65: "15afb5c"
+    }[e] + ".js"
+}(e);
+```
+
+### Analysis of [Vue.js Amsterdam](https://vuejs.amsterdam/)
+Apart from the methods mentioned above, it was found that a lot of JS paths were present in those which are already loaded. They were simple strings like `./sitemap.d7a8ffd7.js`, so they can be extracted on string analysis.
+
+## Client-Side Paths/URLs
+### Analysis of [GitLab's About Site](https://about.gitlab.com/)
+It was found that the client-side paths/URLs were present in the https://about.gitlab.com/_nuxt-new/builds/meta/2d555104-4dad-4b33-a02f-128a966a0c7b.json file. This file was found in the following tab on the home page's source code:
+```html
+<link rel="preload" as="fetch" fetchpriority="low" crossorigin="anonymous" href="/_nuxt-new/builds/meta/2d555104-4dad-4b33-a02f-128a966a0c7b.json">
+```
+
+Here's a preview of the JSON file:
+```json
+{
+    "id": "2d555104-4dad-4b33-a02f-128a966a0c7b",
+    "timestamp": 1750133987484,
+    "matcher": {
+        "static": {},
+        "wildcard": {},
+        "dynamic": {}
+    },
+    "prerendered": [
+        "/de-de/contact-sales",
+        "/de-de/get-help",
+        --snip--
+        "/contact-sales",
+        "/analysts",
+        --snip
+        "/ja-jp",
+        "/ja-jp/search",
+        "/",
+        "/search"
+    ]
+}
+```

package/research/vue_js.md

@@ -0,0 +1,9 @@
+# Vue.js Research
+## Tech Detection
+To detect Vue.js, following technique(s) can be used: 
+- Load the webpage in the browser, and find the `data-v-*` attribute
+    - Note that this attribute might NOT be present if the webpage is not loaded in the browser, i.e. by directly getting the page source
+- However, it was found that most of the Vue.JS sites were using Nuxt. The research for this can be found at [Nuxt.js Research](./nuxt_js.md)
+
+## Lazy Loaded Files
+### Analysis of [Vue.JS official site](https://vuejs.org/)

package/strings/index.js

@@ -0,0 +1,145 @@
+import chalk from "chalk";
+import fs from "fs";
+import path from "path";
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+import prettier from "prettier";
+
+const traverse = _traverse.default;
+
+/**
+ * Extracts all string literals from all .js files in a given directory and its
+ * subdirectories and writes them to a JSON file.
+ * @param {string} directory - The directory to scan for .js files
+ * @param {string} output_file - The file to write the extracted strings to
+ */
+const strings = async (
+  directory,
+  output_file,
+  extract_urls,
+  extracted_url_path,
+) => {
+  console.log(chalk.cyan("[i] Loading 'Strings' module"));
+
+  // check if the directory exists
+  if (!fs.existsSync(directory)) {
+    console.log(chalk.red("[!] Directory does not exist"));
+    return;
+  }
+
+  console.log(chalk.cyan(`[i] Scanning ${directory} directory`));
+
+  // get all files in the directory and sub-directories
+  const files = fs.readdirSync(directory, { recursive: true });
+
+  // filter out non JS files
+  const jsFiles = files.filter((file) => file.endsWith(".js"));
+
+  // read all JS files
+  let js_files_path = [];
+  for (const file of jsFiles) {
+    const filePath = path.join(directory, file);
+    if (!fs.lstatSync(filePath).isDirectory()) {
+      js_files_path.push(filePath);
+    }
+  }
+
+  console.log(chalk.cyan(`[i] Found ${js_files_path.length} JS files`));
+
+  // read all JS files
+  let all_strings = {};
+  for (const file of js_files_path) {
+    const fileContent = fs.readFileSync(file, "utf-8");
+
+    // parse the file contents with babel
+    const ast = parser.parse(fileContent, {
+      sourceType: "unambiguous",
+      plugins: ["jsx", "typescript"],
+    });
+
+    let strings = [];
+
+    traverse(ast, {
+      StringLiteral(path) {
+        strings.push(path.node.value);
+      },
+    });
+
+    all_strings[file] = strings;
+  }
+
+  let strings_count = 0;
+  for (const file of Object.keys(all_strings)) {
+    strings_count += all_strings[file].length;
+  }
+
+  console.log(chalk.cyan(`[i] Extracted ${strings_count} strings`));
+
+  // write to a JSON file
+  const formatted = await prettier.format(JSON.stringify(all_strings), {
+    parser: "json",
+    printWidth: 80,
+    singleQuote: true,
+  });
+  fs.writeFileSync(output_file, formatted);
+
+  console.log(chalk.green(`[✓] Extracted strings to ${output_file}`));
+
+  if (extract_urls) {
+    console.log(chalk.cyan("[i] Extracting URLs and paths from strings"));
+
+    let urls = [];
+    let paths = [];
+
+    for (const file of Object.keys(all_strings)) {
+      for (const string of all_strings[file]) {
+        if (string.match(/^https?:\/\/[a-zA-Z0-9\.\-_]+\/?.*$/)) {
+          // like https://site.com
+          urls.push(string);
+        }
+        if (string.match(/^\/.+$/)) {
+          // like /path/resource
+          // make sure that the path doesn't start with two special chars except '/_'
+          if (string.match(/^\/[^a-zA-Z0-9]/) && !string.startsWith("/_")) {
+            // ignore the path
+          } else {
+            paths.push(string);
+          }
+        }
+        if (string.match(/^[a-zA-Z0-9_\-]\/[a-zA-Z0-9_\-].*$/)) {
+          // like path/to/resource
+          paths.push(string);
+        }
+        if (string.startsWith("./") || string.startsWith("../")) {
+          // like "./path/to/resource" or "../path/to/resource"
+          paths.push(string);
+        }
+      }
+    }
+
+    // dedupe the two lists
+    urls = [...new Set(urls)];
+    paths = [...new Set(paths)];
+
+    console.log(
+      chalk.cyan(`[i] Found ${urls.length} URLs and ${paths.length} paths`),
+    );
+
+    // write to a JSON file
+    const formatted_urls = await prettier.format(
+      JSON.stringify({ urls, paths }),
+      {
+        parser: "json",
+        printWidth: 80,
+        singleQuote: true,
+      },
+    );
+    fs.writeFileSync(extracted_url_path, formatted_urls);
+
+    console.log(
+      chalk.green(`[✓] Written URLs and paths to ${extracted_url_path}`),
+    );
+  }
+};
+
+export default strings;

package/techDetect/index.js

@@ -0,0 +1,156 @@
+import chalk from "chalk";
+import * as cheerio from "cheerio";
+import makeRequest from "../utility/makeReq.js";
+import puppeteer from "puppeteer";
+
+/**
+ * Detects if a webpage uses Next.js by checking if any HTML tag has a src,
+ * srcset, or imageSrcSet attribute that starts with "/_next/".
+ * @param {CheerioStatic} $ - The Cheerio object containing the parsed HTML.
+ * @returns {Promise<{detected: boolean, evidence: string}>}
+ *   A promise that resolves to an object with two properties:
+ *   - detected: A boolean indicating whether Next.js was detected.
+ *   - evidence: A string with the evidence of the detection, or an empty string
+ *     if Next.js was not detected.
+ */
+const checkNextJS = async ($) => {
+  let detected = false;
+  let evidence = "";
+  // iterate through each HTML tag, and file tag value that starts with `/_next/`
+  $("*").each((_, el) => {
+    const tag = $(el).get(0).tagName;
+
+    // check the value of three attributes
+    const src = $(el).attr("src");
+    const srcSet = $(el).attr("srcset");
+    const imageSrcSet = $(el).attr("imageSrcSet");
+
+    if (src || srcSet || imageSrcSet) {
+      if (src && src.startsWith("/_next/")) {
+        detected = true;
+        evidence = `${tag} :: ${src}`;
+      } else if (srcSet && srcSet.startsWith("/_next/")) {
+        detected = true;
+        evidence = `${tag} :: ${srcSet}`;
+      } else if (imageSrcSet && imageSrcSet.startsWith("/_next/")) {
+        detected = true;
+        evidence = `${tag} :: ${imageSrcSet}`;
+      }
+    }
+  });
+
+  return { detected, evidence };
+};
+
+/**
+ * Detects if a webpage uses Vue.js by checking if any HTML tag has a data-v-* attribute.
+ * @param {CheerioStatic} $ - The Cheerio object containing the parsed HTML.
+ * @returns {Promise<{detected: boolean, evidence: string}>}
+ *   A promise that resolves to an object with two properties:
+ *   - detected: A boolean indicating whether Vue.js was detected.
+ *   - evidence: A string with the evidence of the detection, or an empty string
+ *     if Vue.js was not detected.
+ */
+const checkVueJS = async ($) => {
+  let detected = false;
+  let evidence = "";
+
+  $("*").each((_, el) => {
+    const tag = $(el).get(0).tagName;
+    const attribs = el.attribs;
+    if (attribs) {
+      for (const [attrName, attrValue] of Object.entries(attribs)) {
+        if (attrName.startsWith("data-v-")) {
+          detected = true;
+          evidence = `${tag} :: ${attrName}`;
+        }
+      }
+    }
+  });
+
+  return { detected, evidence };
+};
+
+
+const checkNuxtJS = async ($)=>{
+    let detected = false;
+    let evidence = "";
+
+    // go through the page source, and check for "/_nuxt" in the src or href attribute
+    $("*").each((_, el)=>{
+        const tag = $(el).get(0).tagName;
+        const attribs = el.attribs;
+        if (attribs) {
+            for (const [attrName, attrValue] of Object.entries(attribs)) {
+                if (attrName === "src" || attrName === "href") {
+                    if (attrValue.includes("/_nuxt")) {
+                        detected = true;
+                        evidence = `${attrName} :: ${attrValue}`;
+                    }
+                }
+            }
+        }
+    });
+
+    return { detected, evidence };
+}
+
+/**
+ * Detects the front-end framework used by a webpage.
+ * @param {string} url - The URL of the webpage to be detected.
+ * @returns {Promise<{name: string, evidence: string}> | null}
+ *   A promise that resolves to an object with two properties:
+ *   - name: A string indicating the detected framework, or null if no framework was detected.
+ *   - evidence: A string with the evidence of the detection, or an empty string if no framework was detected.
+ */
+const frameworkDetect = async (url) => {
+  console.log(chalk.cyan("[i] Detecting front-end framework"));
+
+  // get the page source
+  const res = await makeRequest(url);
+
+  // get the page source in the browser
+  const browser = await puppeteer.launch({
+    headless: true,
+    args: [
+      "--disable-gpu",
+      "--disable-dev-shm-usage",
+      "--disable-setuid-sandbox",
+      "--no-sandbox",
+    ],
+  });
+  const page = await browser.newPage();
+  await page.goto(url);
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+  const pageSource = await page.content();
+  await browser.close();
+
+  // if (res === null || res === undefined) {
+  //   return;
+  // }
+
+  // const pageSource = await res.text();
+
+  // cheerio to parse the page source
+  const $ = cheerio.load(pageSource);
+
+  // check all technologies one by one
+  const result_checkNextJS = await checkNextJS($);
+  const result_checkVueJS = await checkVueJS($);
+
+  if (result_checkNextJS.detected === true) {
+    return { name: "next", evidence: result_checkNextJS.evidence };
+  } else if (result_checkVueJS.detected === true) {
+    console.log(chalk.green("[✓] Vue.js detected"));
+    console.log(chalk.cyan(`[i] Checking Nuxt.JS`), chalk.dim("(Nuxt.JS is built on Vue.js)"));
+    const result_checkNuxtJS = await checkNuxtJS($);
+    if (result_checkNuxtJS.detected === true) {
+      return { name: "nuxt", evidence: result_checkNuxtJS.evidence };
+    }
+    return { name: "vue", evidence: result_checkVueJS.evidence };
+  }
+
+  return null;
+};
+
+export default frameworkDetect;

package/utility/globals.js

@@ -0,0 +1,6 @@
+
+export let apiGatewayConfigFile = "";
+export let useApiGateway = false;
+
+export const setApiGatewayConfigFile = (file) => { apiGatewayConfigFile = file; };
+export const setUseApiGateway = (value) => { useApiGateway = value; };

package/utility/makeReq.js

@@ -0,0 +1,179 @@
+import chalk from "chalk";
+import puppeteer from "puppeteer";
+import * as globals from "./globals.js";
+import { get } from "../api_gateway/genReq.js";
+
+// random user agents
+const UAs = [
+  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Chrome/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Chrome/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Chrome/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
+  "Chrome/iPhone (request desktop): Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87 Version/11.1.1 Safari/605.1.15",
+  "Chrome/iPad: Mozilla/5.0 (iPad; CPU OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
+  "Chrome/iPod: Mozilla/5.0 (iPod; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
+  "Chrome/Android: Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
+  "Chrome/Android: Mozilla/5.0 (Linux; Android 10; SM-A205U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
+  "Chrome/Android: Mozilla/5.0 (Linux; Android 10; LM-Q720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
+  "Firefox/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
+  "Firefox/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:84.0) Gecko/20100101 Firefox/84.0",
+  "Firefox/Linux: Mozilla/5.0 (X11; Linux i686; rv:84.0) Gecko/20100101 Firefox/84.0",
+  "Firefox/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 11_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
+  "Firefox/iPad: Mozilla/5.0 (iPad; CPU OS 11_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
+  "Firefox/Android: Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/84.0",
+  "Safari/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15",
+  "Safari/iPhone: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+  "Safari/iPhone (request desktop): Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
+  "Safari/iPad: Mozilla/5.0 (iPad; CPU OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+  "IE11/Windows: Mozilla/5.0 (Windows NT 10.0; Trident/7.0; rv:11.0) like Gecko",
+  "Edge/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+  "Edge/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+  "Edge/Android: Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36 EdgA/45.12.4.5121",
+  "Edge/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 EdgiOS/45.11.11 Mobile/15E148 Safari/605.1.15",
+  "Opera/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
+  "Opera/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
+  "Opera/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 OPR/73.0.3856.329",
+  "Opera/Android: Mozilla/5.0 (Linux; Android 10; VOG-L29) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36 OPR/61.1.3076.56625",
+  "Vivaldi/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
+  "Vivaldi/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
+  "Vivaldi/Linux: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
+  "Yandex/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 YaBrowser/20.12.0 Yowser/2.5 Safari/537.36",
+  "Yandex/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 YaBrowser/20.12.0 Yowser/2.5 Safari/537.36",
+  "Yandex/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 YaBrowser/20.11.2.199 Mobile/15E148 Safari/604.1",
+  "Yandex/Android: Mozilla/5.0 (Linux; arm_64; Android 11; SM-G965F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 YaBrowser/20.12.29.180 Mobile Safari/537.36",
+  "Chrome/ChromeOS: Mozilla/5.0 (X11; CrOS x86_64 13505.63.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Safari/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15",
+  "Firefox/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11.1; rv:84.0) Gecko/20100101 Firefox/84.0",
+  "Chrome/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Vivaldi/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
+  "Edge/macOS: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+  "Safari/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+  "Chrome/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1",
+  "Firefox/iOS: Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/30.0 Mobile/15E148 Safari/605.1.15",
+  "Edge/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+  "Internet-Explorer/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
+  "Chrome/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+  "Firefox/Windows: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
+  "Vivaldi/Windows: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Vivaldi/3.5",
+  "Chrome/Android: Mozilla/5.0 (Linux; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Mobile Safari/537.36",
+  "Firefox/Android: Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/84.0",
+];
+
+const makeRequest = async (url, args) => {
+  if (globals.useApiGateway) {
+    let get_headers;
+    if (args && args.headers) {
+      get_headers = args.headers;
+    } else {
+      get_headers = {
+        "User-Agent": UAs[Math.floor(Math.random() * UAs.length)],
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Sec-Fetch-Site": "same-origin",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Dest": "empty",
+        Referer: url,
+        Origin: url,
+      };
+    }
+    
+    const body = await get(url, get_headers);
+
+    // craft a Response, and return that
+    const response = new Response(body);
+    return response;
+  } else {
+    if (args === undefined) {
+      args = {
+        headers: {
+          "User-Agent": UAs[Math.floor(Math.random() * UAs.length)],
+          Accept:
+            "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+          "Accept-Language": "en-US,en;q=0.9",
+          "Sec-Fetch-Site": "same-origin",
+          "Sec-Fetch-Mode": "cors",
+          "Sec-Fetch-Dest": "empty",
+          Referer: url,
+          Origin: url,
+        },
+      };
+    }
+    let res;
+    let counter = 0;
+    while (true) {
+      try {
+        res = await fetch(url, args);
+        if (res) {
+          break;
+        }
+      } catch (err) {
+        counter++;
+        if (counter > 10) {
+          console.log(chalk.red(`[!] Failed to fetch ${url}`));
+          return null;
+        }
+        // sleep 0.5 s before retrying
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        continue;
+      }
+    }
+
+    const preservedRes = res.clone();
+
+    // check if this is a firewall
+    // CF first
+    const resp_text = await res.text();
+    if (resp_text.includes("/?bm-verify=")) {
+      console.log(
+        chalk.yellow(
+          `[!] CF Firewall detected. Trying to bypass with headless browser`,
+        ),
+      );
+      // if it is, load it in a headless browser
+      const browser = await puppeteer.launch({
+        headless: true,
+        args: [
+          "--disable-gpu",
+          "--disable-dev-shm-usage",
+          "--disable-setuid-sandbox",
+          "--no-sandbox",
+        ],
+      });
+      const page = await browser.newPage();
+      await page.goto(url);
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      const content = await page.content();
+      await browser.close();
+      return new Response(content);
+    } else if (resp_text.includes("<title>Just a moment...</title>")) {
+      console.log(
+        chalk.yellow(
+          `[!] CF Firewall detected. Trying to bypass with headless browser`,
+        ),
+      );
+      // if it is, load it in a headless browser
+      const browser = await puppeteer.launch({
+        headless: true,
+        args: [
+          "--disable-gpu",
+          "--disable-dev-shm-usage",
+          "--disable-setuid-sandbox",
+          "--no-sandbox",
+        ],
+      });
+      const page = await browser.newPage();
+      await page.goto(url);
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      const content = await page.content();
+      await browser.close();
+      return new Response(content);
+    }
+
+    return preservedRes;
+  }
+};
+
+export default makeRequest;

package/utility/resolvePath.js

@@ -0,0 +1,43 @@
+/**
+ * Resolves a given path against a base URL using the URL constructor.
+ *
+ * The function handles various cases of path resolution:
+ * - If the base URL does not end with a '/', its last segment is treated as a "file",
+ *   and relative paths are resolved from its "directory".
+ * - Examples:
+ *   - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
+ *     (Base for resolution becomes 'https://site.com/')
+ *   - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
+ *     (Base for resolution is 'https://site.com/something/')
+ *   - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
+ *     (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
+ *
+ * @param {string} url - The base URL to resolve against.
+ * @param {string} path - The path to resolve.
+ * @returns {Promise<string>} - A promise that resolves to the fully resolved URL as a string.
+ * @throws Will throw an error if the resolution fails.
+ */
+const resolvePath = async (url, path) => {
+  try {
+    // The URL constructor handles various cases of path resolution.
+    // If 'url' (the base URL) does not end with a '/', its last path segment
+    // is typically treated as a "file", and relative paths are resolved
+    // from the "directory" containing that "file".
+    // This behavior aligns with the provided examples:
+    // - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
+    //   (Base for resolution becomes 'https://site.com/')
+    // - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
+    //   (Base for resolution is 'https://site.com/something/')
+    // - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
+    //   (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
+    const resolvedUrl = new URL(path, url);
+    return resolvedUrl.href;
+  } catch (e) {
+    console.error(`Error resolving path "${path}" with base URL "${url}": ${e.message}`);
+    // Rethrowing the error to signal failure to the caller.
+    // Alternative error handling (e.g., returning null) can be implemented if required.
+    throw e;
+  }
+};
+
+export default resolvePath;