@shriyanss/js-recon 1.0.0 → 1.1.0-beta.2

package/lazyLoad/next_js/next_GetLazyResources.js

@@ -1,201 +0,0 @@
-import chalk from "chalk";
-import puppeteer from "puppeteer";
-import parser from "@babel/parser";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-import inquirer from "inquirer";
-import CONFIG from "../../globalConfig.js";
-import makeRequest from "../../utility/makeReq.js";
-import execFunc from "../../utility/runSandboxed.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js"; // Import js_urls functions
-
-/**
- * Asynchronously fetches the given URL and extracts JavaScript file URLs
- * from webpack's require.ensure() function.
- *
- * @param {string} url - The URL of the webpage to fetch and parse.
- * @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
- * absolute URLs pointing to JavaScript files found in require.ensure()
- * functions, or undefined if no webpack JS is found.
- */
-const next_getLazyResources = async (url) => {
-  const browser = await puppeteer.launch({
-    headless: true,
-  });
-
-  const page = await browser.newPage();
-
-  await page.setRequestInterception(true);
-
-  page.on("request", async (request) => {
-    // get the request url
-    const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
-
-    // see if the request is a JS file, and is a get request
-    if (
-      request.method() === "GET" &&
-      req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)
-    ) {
-      if (!getJsUrls().includes(req_url)) {
-        pushToJsUrls(req_url);
-      }
-    }
-
-    await request.continue();
-  });
-
-  await page.goto(url);
-
-  await browser.close();
-
-  let webpack_js;
-
-  // iterate through JS files
-  for (const js_url of getJsUrls()) {
-    // match for webpack js file
-    if (js_url.match(/\/webpack.*\.js/)) {
-      console.log(chalk.green(`[✓] Found webpack JS file at ${js_url}`));
-      webpack_js = js_url;
-    }
-  }
-
-  if (!webpack_js) {
-    console.log(chalk.red("[!] No webpack JS file found"));
-    console.log(chalk.magenta(CONFIG.notFoundMessage));
-    return; // Return undefined as per JSDoc
-  }
-
-  // parse the webpack JS file
-  const res = await makeRequest(webpack_js);
-  const webpack_js_source = await res.text();
-
-  // parse it with @babel/*
-  const ast = parser.parse(webpack_js_source, {
-    sourceType: "unambiguous",
-    plugins: ["jsx", "typescript"],
-  });
-
-  let functions = [];
-
-  traverse(ast, {
-    FunctionDeclaration(path) {
-      functions.push({
-        name: path.node.id?.name || "(anonymous)",
-        type: "FunctionDeclaration",
-        source: webpack_js_source.slice(path.node.start, path.node.end),
-      });
-    },
-    FunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "FunctionExpression",
-        source: webpack_js_source.slice(path.node.start, path.node.end),
-      });
-    },
-    ArrowFunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "ArrowFunctionExpression",
-        source: webpack_js_source.slice(path.node.start, path.node.end),
-      });
-    },
-    ObjectMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ObjectMethod",
-        source: webpack_js_source.slice(path.node.start, path.node.end),
-      });
-    },
-    ClassMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ClassMethod",
-        source: webpack_js_source.slice(path.node.start, path.node.end),
-      });
-    },
-  });
-
-  let user_verified = false;
-  // method 1
-  // iterate through the functions, and find out which one ends with `".js"`
-
-  let final_Func;
-  for (const func of functions) {
-    if (func.source.match(/"\.js".{0,15}$/)) {
-      console.log(
-        chalk.green(`[✓] Found JS chunk having the following source`),
-      );
-      console.log(chalk.yellow(func.source));
-      final_Func = func.source;
-    }
-  }
-
-  if (!final_Func) { // Added check if final_Func was not found
-    console.log(chalk.red("[!] No suitable function found in webpack JS for lazy loading."));
-    return [];
-  }
-
-  //   ask through input if this is the right thing
-  const askCorrectFuncConfirmation = async () => {
-    const { confirmed } = await inquirer.prompt([
-      {
-        type: "confirm",
-        name: "confirmed",
-        message: "Is this the correct function?",
-        default: true,
-      },
-    ]);
-    return confirmed;
-  };
-
-  user_verified = await askCorrectFuncConfirmation();
-  if (user_verified === true) {
-    console.log(
-      chalk.cyan("[i] Proceeding with the selected function to fetch files"),
-    );
-  } else {
-    console.log(chalk.red("[!] Not executing function."));
-    return [];
-  }
-
-  const urlBuilderFunc = `(() => (${final_Func}))()`;
-
-  let js_paths = [];
-  try {
-    // rather than fuzzing, grep the integers from the func code
-    const integers = final_Func.match(/\d+/g);
-    if (integers) { // Check if integers were found
-        // iterate through all integers, and get the output
-        for (const i of integers) {
-        const output = execFunc(urlBuilderFunc, parseInt(i));
-        if (output.includes("undefined")) {
-            continue;
-        } else {
-            js_paths.push(output);
-        }
-        }
-    }
-  } catch (err) {
-    console.error("Unsafe or invalid code:", err.message);
-    return [];
-  }
-
-  if (js_paths.length > 0) {
-    console.log(chalk.green(`[✓] Found ${js_paths.length} JS chunks`));
-  }
-
-  // build final URL
-  let final_urls = [];
-  for (let i = 0; i < js_paths.length; i++) {
-    // get the directory of webpack file
-    const webpack_dir = webpack_js.split("/").slice(0, -1).join("/");
-    // replace the filename from the js path
-    const js_path_dir = js_paths[i].replace(/\/[a-zA-Z0-9\.]+\.js.*$/, "");
-    const final_url = webpack_dir.replace(js_path_dir, js_paths[i]);
-    final_urls.push(final_url);
-  }
-
-  return final_urls;
-};
-
-export default next_getLazyResources;

package/lazyLoad/next_js/next_SubsequentRequests.js

@@ -1,138 +0,0 @@
-import chalk from "chalk";
-import fs from "fs";
-import path from "path";
-import { getURLDirectory } from "../../utility/urlUtils.js";
-// custom request module
-import makeRequest from "../../utility/makeReq.js";
-
-let queue = [];
-let max_queue;
-
-/**
- * Given a string of JS content, it finds all the static files used in the
- * file, and returns them as an array.
- *
- * @param {string} js_content - The string of JS content to search through.
- *
- * @returns {string[]} An array of strings, each string being a static file
- * path.
- */
-const findStaticFiles = async (js_content) => {
-  // do some regex-ing
-  const matches = [
-    ...js_content.matchAll(/\/?static\/chunks\/[a-zA-Z0-9\._\-\/]+\.js/g),
-  ];
-  // return matches
-
-  let toReturn = [];
-
-  for (const match of matches) {
-    toReturn.push(match[0]);
-  }
-
-  return toReturn;
-};
-
-const getURLDirectoryServer = (urlString) => {
-  const url = new URL(urlString);
-  const pathParts = url.pathname.split("/").filter(Boolean); // ['business', 'api']
-  pathParts.pop(); // Remove 'api'
-
-  const newPath = "/" + pathParts.join("/"); // '/business'
-  return `${url.origin}${newPath}`; // 'http://something.com/business'
-};
-
-const subsequentRequests = async (url, urlsFile, threads, output, js_urls) => {
-  max_queue = threads;
-  let staticJSURLs = [];
-
-  console.log(chalk.cyan(`[i] Fetching JS files from subsequent requests`));
-
-  // open the urls file, and load the paths (JSON)
-  const endpoints = JSON.parse(fs.readFileSync(urlsFile, "utf-8")).paths;
-
-  let js_contents = {};
-
-  // make requests to all of them with the special header
-  const reqPromises = endpoints.map(async (endpoint) => {
-    const reqUrl = url + endpoint;
-    try {
-      // delay in case over the thread count
-      while (queue >= max_queue) {
-        await new Promise((resolve) => setTimeout(resolve, 100));
-      }
-      queue++;
-
-      const res = await makeRequest(reqUrl, {
-        headers: {
-          RSC: "1",
-        },
-      });
-
-      if (
-        res &&
-        res.status === 200 &&
-        res.headers.get("content-type").includes("text/x-component")
-      ) {
-        const text = await res.text();
-        js_contents[endpoint] = text;
-
-        const { host, directory } = getURLDirectory(reqUrl);
-
-        // save the contents to "___subsequent_requests/"
-        // make the subsequent_requests directory if it doesn't exist
-
-        const output_path = path.join(
-          output,
-          host,
-          "___subsequent_requests",
-          directory,
-        );
-        if (!fs.existsSync(output_path)) {
-          fs.mkdirSync(output_path);
-        }
-        fs.writeFileSync(path.join(output_path, "index.js"), text);
-
-        // find the static ones from the JS resp
-        const staticFiles = await findStaticFiles(text);
-
-        // go through each file and get the absolute path of those
-        const absolutePaths = staticFiles.map((file) => {
-          // go through existing JS URLs found
-          let js_path_dir;
-          for (const js_url of js_urls) {
-            if (
-              !js_path_dir &&
-              new URL(js_url).host === new URL(url).host &&
-              new URL(js_url).pathname.includes("static/chunks/")
-            ) {
-              js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
-            }
-          }
-          return js_path_dir.replace("static/chunks", "") + file;
-        });
-
-        // Filter out paths that are already in js_urls before pushing to staticJSURLs
-        const newPaths = absolutePaths.filter(path => !js_urls.includes(path));
-        if (newPaths.length > 0) {
-          staticJSURLs.push(...newPaths);
-        }
-      }
-
-      queue--;
-    } catch (e) {
-      queue--;
-      console.log(chalk.red(`[!] Error fetching ${reqUrl}: ${e}`));
-    }
-  });
-
-  await Promise.all(reqPromises);
-
-  staticJSURLs = [...new Set(staticJSURLs)];
-
-  console.log(chalk.green(`[✓] Found ${staticJSURLs.length} JS chunks from subsequent requests`));
-
-  return staticJSURLs;
-};
-
-export default subsequentRequests;

package/lazyLoad/nuxt_js/nuxt_astParse.js

@@ -1,194 +0,0 @@
-import parser from "@babel/parser";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-import execFunc from "../../utility/runSandboxed.js";
-import makeRequest from "../../utility/makeReq.js";
-import chalk from "chalk";
-import inquirer from "inquirer";
-import t from "@babel/types";
-import resolvePath from "../../utility/resolvePath.js";
-
-const nuxt_astParse = async (url) => {
-  let filesFound = [];
-  const resp = await makeRequest(url);
-  const body = await resp.text();
-
-  let ast;
-  try {
-    ast = parser.parse(body, {
-      sourceType: "module",
-      plugins: ["jsx", "typescript"],
-    });
-  } catch (error) {
-    console.log(chalk.red("[!] Error parsing JS file: ", url));
-    return filesFound;
-  }
-
-  let functions = [];
-
-  traverse(ast, {
-    FunctionDeclaration(path) {
-      functions.push({
-        name: path.node.id?.name || "(anonymous)",
-        type: "FunctionDeclaration",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    FunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "FunctionExpression",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ArrowFunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "ArrowFunctionExpression",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ObjectMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ObjectMethod",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ClassMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ClassMethod",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-  });
-
-  // iterate through the functions, and find out the one that ends with ".js"
-
-  for (const func of functions) {
-    if (func.source.match(/"\.js".{0,15}$/)) {
-      console.log(
-        chalk.green(`[✓] Found JS chunk having the following source:`),
-      );
-      console.log(chalk.yellow(func.source));
-
-      // ask for confirmation, then proceed
-      const askCorrectFuncConfirmation = async () => {
-        const { value } = await inquirer.prompt([
-          {
-            type: "confirm",
-            name: "value",
-            message: "Is this the correct function?",
-            default: true,
-          },
-        ]);
-        return value;
-      };
-
-      const user_verified = await askCorrectFuncConfirmation();
-      if (user_verified === true) {
-        console.log(
-          chalk.cyan(
-            "[i] Proceeding with the selected function to fetch files",
-          ),
-        );
-
-        // get the value of the unknown vars
-        // first, get the name of the unknown function
-        const unknownVarAst = parser.parse(`(${func.source})`, {
-          sourceType: "script",
-          plugins: ["jsx", "typescript"],
-        });
-        let memberExpressions = [];
-        traverse(unknownVarAst, {
-          MemberExpression(path) {
-            // Only collect identifiers like f.p (not obj["x"])
-            if (
-              t.isIdentifier(path.node.object) &&
-              t.isIdentifier(path.node.property) &&
-              !path.node.computed // ignore obj["x"]
-            ) {
-              const objName = path.node.object.name;
-              const propName = path.node.property.name;
-              memberExpressions.push(`${objName}.${propName}`);
-            }
-          },
-        });
-
-        const unknownVar = memberExpressions[0].split(".");
-
-        // now, resolve the value of this unknown var
-        let unknownVarValue;
-
-        traverse(ast, {
-          AssignmentExpression(path) {
-            const { left, right } = path.node;
-
-            if (
-              t.isMemberExpression(left) &&
-              t.isIdentifier(left.object, { name: unknownVar[0] }) &&
-              t.isIdentifier(left.property, { name: unknownVar[1] }) &&
-              !left.computed
-            ) {
-              if (t.isStringLiteral(right)) {
-                unknownVarValue = right.value;
-              } else {
-                // fallback to source snippet
-                unknownVarValue = func.source.slice(right.start, right.end);
-              }
-            }
-          },
-        });
-
-        // replace the unknown var with the value
-        const funcSource = func.source.replace(
-          new RegExp(`${unknownVar[0]}.${unknownVar[1]}`),
-          `"${unknownVarValue}"`,
-        );
-
-        // continue to executing the function with all possible numbers
-        const urlBuilderFunc = `(() => (${funcSource}))()`;
-        let js_paths = [];
-
-        try {
-          // rather than fuzzing, grep the integers from the func code
-          const integers = funcSource.match(/\d+/g);
-          if (integers) {
-            // Check if integers were found
-            // iterate through all integers, and get the output
-            for (const i of integers) {
-              const output = execFunc(urlBuilderFunc, parseInt(i));
-              if (output.includes("undefined")) {
-                continue;
-              } else {
-                js_paths.push(output);
-              }
-            }
-          }
-        } catch (error) {
-          console.log(chalk.red("[!] Error executing function: ", error));
-        }
-
-        if (js_paths.length > 0) {
-          // iterate through the files, and resolve them
-          for (const js_path of js_paths) {
-            const resolvedPath = await resolvePath(url, js_path);
-            filesFound.push(resolvedPath);
-          }
-        }
-      } else {
-        console.log(chalk.red("[!] Not executing function."));
-        continue;
-      }
-    }
-  }
-
-  if (filesFound.length > 0) {
-    console.log(chalk.green(`[✓] Found ${filesFound.length} JS chunks`));
-  }
-
-  return filesFound;
-};
-
-export default nuxt_astParse;

package/lazyLoad/nuxt_js/nuxt_getFromPageSource.js

@@ -1,77 +0,0 @@
-import chalk from "chalk";
-import makeRequest from "../../utility/makeReq.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js";
-import * as cheerio from "cheerio";
-
-const nuxt_getFromPageSource = async (url) => {
-  console.log(chalk.cyan("[i] Analyzing page source"));
-
-  // get the page source
-  const res = await makeRequest(url);
-  const pageSource = await res.text();
-
-  // cheerio to parse the page source
-  const $ = cheerio.load(pageSource);
-
-  // find all link tags
-  const linkTags = $("link");
-
-  // go through them, and find the ones which have `as=script` attr
-  for (const linkTag of linkTags) {
-    const asAttr = $(linkTag).attr("as");
-    if (asAttr === "script") {
-      const hrefAttr = $(linkTag).attr("href");
-      if (hrefAttr) {
-        // see if it starts with /_nuxt
-        if (hrefAttr.startsWith("/_nuxt")) {
-          // get the URL root, and append the hrefAttr to it
-          const urlRoot = new URL(url).origin;
-          pushToJsUrls(urlRoot + hrefAttr);
-        }
-      }
-    }
-  }
-
-  // now, search all the script tags
-  const scriptTags = $("script");
-  for (const scriptTag of scriptTags) {
-    const src = $(scriptTag).attr("src");
-    if (
-      src !== undefined &&
-      src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)
-    ) {
-      if (src.startsWith("http")) {
-        if (!getJsUrls().includes(src)) {
-          pushToJsUrls(src);
-        }
-      }
-      // if the src starts with /, like `/static/js/a.js` find the absolute URL
-      else if (src.startsWith("/")) {
-        const absoluteUrl = new URL(url).origin + src;
-        if (!getJsUrls().includes(absoluteUrl)) {
-          pushToJsUrls(absoluteUrl);
-        }
-      } else if (src.match(/^[^/]/)) {
-        // if the src is a relative URL, like `static/js/a.js` find the absolute URL
-        // Get directory URL (origin + path without filename)
-        const pathParts = new URL(url).pathname.split("/");
-        pathParts.pop(); // remove the filename from the path
-        const directory = new URL(url).origin + pathParts.join("/") + "/";
-
-        if (!getJsUrls().includes(directory + src)) {
-          pushToJsUrls(directory + src);
-        }
-      } else {
-        continue;
-      }
-    }
-  }
-
-  console.log(
-    chalk.green(`[✓] Found ${getJsUrls().length} JS files from the page source`),
-  );
-
-  return getJsUrls();
-};
-
-export default nuxt_getFromPageSource;

package/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js

@@ -1,99 +0,0 @@
-import chalk from "chalk";
-import makeRequest from "../../utility/makeReq.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js";
-import resolvePath from "../../utility/resolvePath.js";
-
-// for parsing
-import parser from "@babel/parser";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-
-let analyzedFiles = [];
-let filesFound = [];
-
-const parseJSFileContent = async (content) => {
-  try {
-    const ast = parser.parse(content, {
-      sourceType: "module",
-      plugins: ["jsx", "typescript"],
-    });
-
-    let foundJsFiles = {};
-
-    traverse(ast, {
-      StringLiteral(path) {
-        const value = path.node.value;
-        if (value.startsWith("./") && value.endsWith(".js")) {
-          foundJsFiles[value] = value;
-        } else if (value.startsWith("../") && value.endsWith(".js")) {
-          foundJsFiles[value] = value;
-        }
-      },
-    });
-
-    return foundJsFiles;
-  } catch (error) {
-    return {};
-  }
-};
-
-const nuxt_stringAnalysisJSFiles = async (url) => {
-  console.log(chalk.cyan("[i] Analyzing strings in the files found"));
-
-  while (true) {
-    // get all the JS URLs
-    const js_urls = getJsUrls();
-
-    if (js_urls.length === 0) {
-      console.log(chalk.red("[!] No JS files found for string analysis"));
-      break;
-    }
-
-    // if js_urls have everything that is in analyzedFiles, break
-    let everythingAnalyzed = true;
-    for (const url of js_urls) {
-      //   if the url is not in analyzedFiles, set everythingAnalyzed to false
-      if (!analyzedFiles.includes(url)) {
-        everythingAnalyzed = false;
-      }
-    }
-
-    // break if everything is analyzed
-    if (everythingAnalyzed) {
-      break;
-    }
-
-    // iterate through the JS URLs
-    for (const js_url of js_urls) {
-      if (analyzedFiles.includes(js_url)) {
-        continue;
-      }
-
-      const response = await makeRequest(js_url);
-      const respText = await response.text();
-      const foundJsFiles = await parseJSFileContent(respText);
-
-      // iterate through the foundJsFiles and resolve the paths
-      for (const [key, value] of Object.entries(foundJsFiles)) {
-        const resolvedPath = await resolvePath(js_url, value);
-        if (analyzedFiles.includes(resolvedPath)) {
-          continue;
-        }
-        pushToJsUrls(resolvedPath);
-        filesFound.push(resolvedPath);
-      }
-
-      analyzedFiles.push(js_url);
-    }
-  }
-
-  filesFound = [...new Set(filesFound)];
-
-  console.log(
-    chalk.green(`[✓] Found ${filesFound.length} JS files from string analysis`),
-  );
-
-  return filesFound;
-};
-
-export default nuxt_stringAnalysisJSFiles;