@shriyanss/js-recon 1.0.0

package/lazyLoad/index.js

@@ -0,0 +1,167 @@
+import chalk from "chalk";
+import fs from "fs";
+import frameworkDetect from "../techDetect/index.js";
+import CONFIG from "../globalConfig.js";
+import _traverse from "@babel/traverse";
+const traverse = _traverse.default;
+import { URL } from "url";
+
+// Next.js
+import subsequentRequests from "./next_js/next_SubsequentRequests.js";
+import next_getJSScript from "./next_js/next_GetJSScript.js";
+import next_getLazyResources from "./next_js/next_GetLazyResources.js";
+
+// Nuxt.js
+import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
+import nuxt_stringAnalysisJSFiles from "./nuxt_js/nuxt_stringAnalysisJSFiles.js";
+import nuxt_astParse from "./nuxt_js/nuxt_astParse.js";
+
+// generic
+import downloadFiles from "./downloadFilesUtil.js";
+import downloadLoadedJs from "./downloadLoadedJsUtil.js";
+
+// import global vars
+import * as globals from "./globals.js";
+
+/**
+ * Downloads all lazy-loaded JavaScript files from the specified URL or file containing URLs.
+ *
+ * The function detects the JavaScript framework used by the webpage (e.g., Next.js, Nuxt.js)
+ * and utilizes specific techniques to find and download lazy-loaded JS files.
+ * It supports subsequent requests for additional JS files if specified.
+ *
+ * @param {string} url - The URL or path to a file containing a list of URLs to process.
+ * @param {string} output - The directory where downloaded files will be saved.
+ * @param {boolean} strictScope - Whether to restrict downloads to the input URL domain.
+ * @param {string[]} inputScope - Specific domains to download JS files from.
+ * @param {number} threads - The number of threads to use for downloading files.
+ * @param {boolean} subsequentRequestsFlag - Whether to include JS files from subsequent requests.
+ * @param {string} urlsFile - The JSON file containing additional URLs for subsequent requests.
+ * @returns {Promise<void>}
+ */
+const lazyLoad = async (
+  url,
+  output,
+  strictScope,
+  inputScope,
+  threads,
+  subsequentRequestsFlag,
+  urlsFile,
+) => {
+  console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
+
+  let urls;
+
+  // check if the url is file or a URL
+  if (fs.existsSync(url)) {
+    urls = fs.readFileSync(url, "utf-8").split("\n");
+    // remove the empty lines
+    urls = urls.filter((url) => url.trim() !== "");
+  } else if (url.match(/https?:\/\/[a-zA-Z0-9\-_\.:]+/)) {
+    urls = [url];
+  } else {
+    console.log(chalk.red("[!] Invalid URL or file path"));
+    return;
+  }
+
+  for (const url of urls) {
+    console.log(chalk.cyan(`[i] Processing ${url}`));
+
+    if (strictScope) {
+      globals.pushToScope(new URL(url).host);
+    } else {
+      globals.setScope(inputScope);
+    }
+
+    globals.setMaxReqQueue(threads);
+    globals.clearJsUrls(); // Initialize js_urls for each URL processing in the loop
+
+    const tech = await frameworkDetect(url);
+
+    if (tech) {
+      if (tech.name === "next") {
+        console.log(chalk.green("[✓] Next.js detected"));
+        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+
+        // find the JS files from script of the webpage
+        const jsFilesFromScriptTag = await next_getJSScript(url);
+
+        // get lazy resources
+        const lazyResourcesFromWebpack = await next_getLazyResources(url);
+        let lazyResourcesFromSubsequentRequests;
+
+        if (subsequentRequestsFlag) {
+          // get JS files from subsequent requests
+          lazyResourcesFromSubsequentRequests = await subsequentRequests(
+            url,
+            urlsFile,
+            threads,
+            output,
+            globals.getJsUrls(), // Pass the global js_urls
+          );
+        }
+
+        // download the resources
+        // but combine them first
+        let jsFilesToDownload = [
+          ...(jsFilesFromScriptTag || []),
+          ...(lazyResourcesFromWebpack || []),
+          ...(lazyResourcesFromSubsequentRequests || []),
+        ];
+        // Ensure js_urls from globals are included if next_getJSScript or next_getLazyResources populated it.
+        // This is because those functions now push to the global js_urls via setters.
+        // The return values of next_getJSScript and next_getLazyResources might be the same array instance
+        // or a new one depending on their implementation, so explicitly get the global one here.
+        jsFilesToDownload.push(...globals.getJsUrls());
+
+        // dedupe the files
+        jsFilesToDownload = [...new Set(jsFilesToDownload)];
+
+        await downloadFiles(jsFilesToDownload, output);
+      } else if (tech.name === "vue") {
+        console.log(chalk.green("[✓] Vue.js detected"));
+        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+      } else if (tech.name === "nuxt") {
+        console.log(chalk.green("[✓] Nuxt.js detected"));
+        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+
+        let jsFilesToDownload = [];
+
+        // find the files from the page source
+        const jsFilesFromPageSource = await nuxt_getFromPageSource(url);
+        const jsFilesFromStringAnalysis = await nuxt_stringAnalysisJSFiles(url);
+
+        jsFilesToDownload.push(...jsFilesFromPageSource);
+        jsFilesToDownload.push(...jsFilesFromStringAnalysis);
+        // dedupe the files
+        jsFilesToDownload = [...new Set(jsFilesToDownload)];
+
+        let jsFilesFromAST = [];
+        console.log(chalk.cyan("[i] Analyzing functions in the files found"));
+        for (const jsFile of jsFilesToDownload) {
+          jsFilesFromAST.push(...(await nuxt_astParse(jsFile)));
+        }
+
+        jsFilesToDownload.push(...jsFilesFromAST);
+
+        jsFilesToDownload.push(...globals.getJsUrls());
+
+        // dedupe the files
+        jsFilesToDownload = [...new Set(jsFilesToDownload)];
+
+        await downloadFiles(jsFilesToDownload, output);
+      }
+    } else {
+      console.log(chalk.red("[!] Framework not detected :("));
+      console.log(chalk.magenta(CONFIG.notFoundMessage));
+      console.log(chalk.yellow("[i] Trying to download loaded JS files"));
+      const js_urls = await downloadLoadedJs(url);
+      if (js_urls && js_urls.length > 0) {
+        console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
+        await downloadFiles(js_urls, output);
+      }
+    }
+  }
+};
+
+export default lazyLoad;

package/lazyLoad/next_js/next_GetJSScript.js

@@ -0,0 +1,99 @@
+// lazyLoad/nextGetJSScript.js
+import chalk from "chalk";
+import { URL } from "url";
+import * as cheerio from "cheerio";
+import makeRequest from "../../utility/makeReq.js";
+import { getJsUrls, pushToJsUrls } from "../globals.js";
+
+/**
+ * Asynchronously fetches the given URL and extracts JavaScript file URLs
+ * from script tags present in the HTML content.
+ *
+ * @param {string} url - The URL of the webpage to fetch and parse.
+ * @returns {Promise<string[]>} - A promise that resolves to an array of
+ * absolute URLs pointing to JavaScript files found in script tags.
+ */
+const next_getJSScript = async (url) => {
+  // get the page source
+  const res = await makeRequest(url);
+  const pageSource = await res.text();
+
+  // cheerio to parse the page source
+  const $ = cheerio.load(pageSource);
+
+  // find all script tags
+  const scriptTags = $("script");
+
+  // iterate through script tags
+  for (const scriptTag of scriptTags) {
+    // get the src attribute
+    const src = $(scriptTag).attr("src");
+
+    // see if the src is a JS file
+    if (
+      src !== undefined &&
+      src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)) {
+      // if the src starts with /, like `/static/js/a.js` find the absolute URL
+      if (src.startsWith("/")) {
+        const absoluteUrl = new URL(url).origin + src;
+        if (!getJsUrls().includes(absoluteUrl)) {
+          pushToJsUrls(absoluteUrl);
+        }
+      } else if (src.match(/^[^/]/)) {
+        // if the src is a relative URL, like `static/js/a.js` find the absolute URL
+        // Get directory URL (origin + path without filename)
+        const pathParts = new URL(url).pathname.split("/");
+        pathParts.pop(); // remove filename from last
+        const directory = new URL(url).origin + pathParts.join("/") + "/";
+
+        if (!getJsUrls().includes(directory + src)) {
+          pushToJsUrls(directory + src);
+        }
+      } else {
+        if (!getJsUrls().includes(src)) {
+          pushToJsUrls(src);
+        }
+      }
+    } else {
+      // if the script tag is inline, it could contain static JS URL
+      // to get these, simply regex from the JS script
+
+      const js_script = $(scriptTag).html();
+      const matches = js_script.match(/static\/chunks\/[a-zA-Z0-9_\-]+\.js/g);
+
+      if (matches) {
+        const uniqueMatches = [...new Set(matches)];
+        for (const match of uniqueMatches) {
+          // if it is using that static/chunks/ pattern, I can just get the filename
+          const filename = match.replace("static/chunks/", "");
+
+          // go through the already found URLs, coz they will have it (src attribute
+          // is there before inline things)
+
+          let js_path_dir;
+
+          for (const js_url of getJsUrls()) {
+            if (
+              !js_path_dir &&
+              new URL(js_url).host === new URL(url).host &&
+              new URL(js_url).pathname.includes("static/chunks/")
+            ) {
+              js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
+            }
+          }
+          if (js_path_dir) { // Ensure js_path_dir was found
+            pushToJsUrls(js_path_dir + "/" + filename);
+          }
+        }
+      }
+    }
+  }
+
+  console.log(
+    chalk.green(`[✓] Found ${getJsUrls().length} JS files from the script tags`),
+  );
+
+  return getJsUrls();
+};
+
+export default next_getJSScript;

package/lazyLoad/next_js/next_GetLazyResources.js

@@ -0,0 +1,201 @@
+import chalk from "chalk";
+import puppeteer from "puppeteer";
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+const traverse = _traverse.default;
+import inquirer from "inquirer";
+import CONFIG from "../../globalConfig.js";
+import makeRequest from "../../utility/makeReq.js";
+import execFunc from "../../utility/runSandboxed.js";
+import { getJsUrls, pushToJsUrls } from "../globals.js"; // Import js_urls functions
+
+/**
+ * Asynchronously fetches the given URL and extracts JavaScript file URLs
+ * from webpack's require.ensure() function.
+ *
+ * @param {string} url - The URL of the webpage to fetch and parse.
+ * @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
+ * absolute URLs pointing to JavaScript files found in require.ensure()
+ * functions, or undefined if no webpack JS is found.
+ */
+const next_getLazyResources = async (url) => {
+  const browser = await puppeteer.launch({
+    headless: true,
+  });
+
+  const page = await browser.newPage();
+
+  await page.setRequestInterception(true);
+
+  page.on("request", async (request) => {
+    // get the request url
+    const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
+
+    // see if the request is a JS file, and is a get request
+    if (
+      request.method() === "GET" &&
+      req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)
+    ) {
+      if (!getJsUrls().includes(req_url)) {
+        pushToJsUrls(req_url);
+      }
+    }
+
+    await request.continue();
+  });
+
+  await page.goto(url);
+
+  await browser.close();
+
+  let webpack_js;
+
+  // iterate through JS files
+  for (const js_url of getJsUrls()) {
+    // match for webpack js file
+    if (js_url.match(/\/webpack.*\.js/)) {
+      console.log(chalk.green(`[✓] Found webpack JS file at ${js_url}`));
+      webpack_js = js_url;
+    }
+  }
+
+  if (!webpack_js) {
+    console.log(chalk.red("[!] No webpack JS file found"));
+    console.log(chalk.magenta(CONFIG.notFoundMessage));
+    return; // Return undefined as per JSDoc
+  }
+
+  // parse the webpack JS file
+  const res = await makeRequest(webpack_js);
+  const webpack_js_source = await res.text();
+
+  // parse it with @babel/*
+  const ast = parser.parse(webpack_js_source, {
+    sourceType: "unambiguous",
+    plugins: ["jsx", "typescript"],
+  });
+
+  let functions = [];
+
+  traverse(ast, {
+    FunctionDeclaration(path) {
+      functions.push({
+        name: path.node.id?.name || "(anonymous)",
+        type: "FunctionDeclaration",
+        source: webpack_js_source.slice(path.node.start, path.node.end),
+      });
+    },
+    FunctionExpression(path) {
+      functions.push({
+        name: path.parent.id?.name || "(anonymous)",
+        type: "FunctionExpression",
+        source: webpack_js_source.slice(path.node.start, path.node.end),
+      });
+    },
+    ArrowFunctionExpression(path) {
+      functions.push({
+        name: path.parent.id?.name || "(anonymous)",
+        type: "ArrowFunctionExpression",
+        source: webpack_js_source.slice(path.node.start, path.node.end),
+      });
+    },
+    ObjectMethod(path) {
+      functions.push({
+        name: path.node.key.name,
+        type: "ObjectMethod",
+        source: webpack_js_source.slice(path.node.start, path.node.end),
+      });
+    },
+    ClassMethod(path) {
+      functions.push({
+        name: path.node.key.name,
+        type: "ClassMethod",
+        source: webpack_js_source.slice(path.node.start, path.node.end),
+      });
+    },
+  });
+
+  let user_verified = false;
+  // method 1
+  // iterate through the functions, and find out which one ends with `".js"`
+
+  let final_Func;
+  for (const func of functions) {
+    if (func.source.match(/"\.js".{0,15}$/)) {
+      console.log(
+        chalk.green(`[✓] Found JS chunk having the following source`),
+      );
+      console.log(chalk.yellow(func.source));
+      final_Func = func.source;
+    }
+  }
+
+  if (!final_Func) { // Added check if final_Func was not found
+    console.log(chalk.red("[!] No suitable function found in webpack JS for lazy loading."));
+    return [];
+  }
+
+  //   ask through input if this is the right thing
+  const askCorrectFuncConfirmation = async () => {
+    const { confirmed } = await inquirer.prompt([
+      {
+        type: "confirm",
+        name: "confirmed",
+        message: "Is this the correct function?",
+        default: true,
+      },
+    ]);
+    return confirmed;
+  };
+
+  user_verified = await askCorrectFuncConfirmation();
+  if (user_verified === true) {
+    console.log(
+      chalk.cyan("[i] Proceeding with the selected function to fetch files"),
+    );
+  } else {
+    console.log(chalk.red("[!] Not executing function."));
+    return [];
+  }
+
+  const urlBuilderFunc = `(() => (${final_Func}))()`;
+
+  let js_paths = [];
+  try {
+    // rather than fuzzing, grep the integers from the func code
+    const integers = final_Func.match(/\d+/g);
+    if (integers) { // Check if integers were found
+        // iterate through all integers, and get the output
+        for (const i of integers) {
+        const output = execFunc(urlBuilderFunc, parseInt(i));
+        if (output.includes("undefined")) {
+            continue;
+        } else {
+            js_paths.push(output);
+        }
+        }
+    }
+  } catch (err) {
+    console.error("Unsafe or invalid code:", err.message);
+    return [];
+  }
+
+  if (js_paths.length > 0) {
+    console.log(chalk.green(`[✓] Found ${js_paths.length} JS chunks`));
+  }
+
+  // build final URL
+  let final_urls = [];
+  for (let i = 0; i < js_paths.length; i++) {
+    // get the directory of webpack file
+    const webpack_dir = webpack_js.split("/").slice(0, -1).join("/");
+    // replace the filename from the js path
+    const js_path_dir = js_paths[i].replace(/\/[a-zA-Z0-9\.]+\.js.*$/, "");
+    const final_url = webpack_dir.replace(js_path_dir, js_paths[i]);
+    final_urls.push(final_url);
+  }
+
+  return final_urls;
+};
+
+export default next_getLazyResources;

package/lazyLoad/next_js/next_SubsequentRequests.js

@@ -0,0 +1,138 @@
+import chalk from "chalk";
+import fs from "fs";
+import path from "path";
+import { getURLDirectory } from "../../utility/urlUtils.js";
+// custom request module
+import makeRequest from "../../utility/makeReq.js";
+
+let queue = [];
+let max_queue;
+
+/**
+ * Given a string of JS content, it finds all the static files used in the
+ * file, and returns them as an array.
+ *
+ * @param {string} js_content - The string of JS content to search through.
+ *
+ * @returns {string[]} An array of strings, each string being a static file
+ * path.
+ */
+const findStaticFiles = async (js_content) => {
+  // do some regex-ing
+  const matches = [
+    ...js_content.matchAll(/\/?static\/chunks\/[a-zA-Z0-9\._\-\/]+\.js/g),
+  ];
+  // return matches
+
+  let toReturn = [];
+
+  for (const match of matches) {
+    toReturn.push(match[0]);
+  }
+
+  return toReturn;
+};
+
+const getURLDirectoryServer = (urlString) => {
+  const url = new URL(urlString);
+  const pathParts = url.pathname.split("/").filter(Boolean); // ['business', 'api']
+  pathParts.pop(); // Remove 'api'
+
+  const newPath = "/" + pathParts.join("/"); // '/business'
+  return `${url.origin}${newPath}`; // 'http://something.com/business'
+};
+
+const subsequentRequests = async (url, urlsFile, threads, output, js_urls) => {
+  max_queue = threads;
+  let staticJSURLs = [];
+
+  console.log(chalk.cyan(`[i] Fetching JS files from subsequent requests`));
+
+  // open the urls file, and load the paths (JSON)
+  const endpoints = JSON.parse(fs.readFileSync(urlsFile, "utf-8")).paths;
+
+  let js_contents = {};
+
+  // make requests to all of them with the special header
+  const reqPromises = endpoints.map(async (endpoint) => {
+    const reqUrl = url + endpoint;
+    try {
+      // delay in case over the thread count
+      while (queue >= max_queue) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+      queue++;
+
+      const res = await makeRequest(reqUrl, {
+        headers: {
+          RSC: "1",
+        },
+      });
+
+      if (
+        res &&
+        res.status === 200 &&
+        res.headers.get("content-type").includes("text/x-component")
+      ) {
+        const text = await res.text();
+        js_contents[endpoint] = text;
+
+        const { host, directory } = getURLDirectory(reqUrl);
+
+        // save the contents to "___subsequent_requests/"
+        // make the subsequent_requests directory if it doesn't exist
+
+        const output_path = path.join(
+          output,
+          host,
+          "___subsequent_requests",
+          directory,
+        );
+        if (!fs.existsSync(output_path)) {
+          fs.mkdirSync(output_path);
+        }
+        fs.writeFileSync(path.join(output_path, "index.js"), text);
+
+        // find the static ones from the JS resp
+        const staticFiles = await findStaticFiles(text);
+
+        // go through each file and get the absolute path of those
+        const absolutePaths = staticFiles.map((file) => {
+          // go through existing JS URLs found
+          let js_path_dir;
+          for (const js_url of js_urls) {
+            if (
+              !js_path_dir &&
+              new URL(js_url).host === new URL(url).host &&
+              new URL(js_url).pathname.includes("static/chunks/")
+            ) {
+              js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
+            }
+          }
+          return js_path_dir.replace("static/chunks", "") + file;
+        });
+
+        // Filter out paths that are already in js_urls before pushing to staticJSURLs
+        const newPaths = absolutePaths.filter(path => !js_urls.includes(path));
+        if (newPaths.length > 0) {
+          staticJSURLs.push(...newPaths);
+        }
+      }
+
+      queue--;
+    } catch (e) {
+      queue--;
+      console.log(chalk.red(`[!] Error fetching ${reqUrl}: ${e}`));
+    }
+  });
+
+  await Promise.all(reqPromises);
+
+  staticJSURLs = [...new Set(staticJSURLs)];
+
+  console.log(chalk.green(`[✓] Found ${staticJSURLs.length} JS chunks from subsequent requests`));
+
+  return staticJSURLs;
+};
+
+export default subsequentRequests;