@shriyanss/js-recon 1.0.0 → 1.1.0-beta.2

package/globalConfig.js

@@ -1,12 +0,0 @@
-const githubURL = "https://github.com/shriyanss/js-recon"
-const version = "1.0.0";
-const toolDesc = "JS Recon Tool";   
-
-global.CONFIG = {
-    github: githubURL,
-    notFoundMessage: `If you believe this is an error or is a new technology, please create an issue on ${githubURL} and we'll figure it out for you`,
-    version: version,
-    toolDesc: toolDesc
-};
-
-export default CONFIG;

package/index.js

@@ -1,69 +0,0 @@
-#!/usr/bin/env node
-import { program } from "commander";
-import lazyLoad from "./lazyLoad/index.js";
-import endpoints from "./endpoints/index.js";
-import CONFIG from "./globalConfig.js";
-import strings from "./strings/index.js";
-import apiGateway from "./api_gateway/index.js";
-import * as globals from "./utility/globals.js";
-
-program.version(CONFIG.version).description(CONFIG.toolDesc);
-
-program
-  .command("lazyload")
-  .description("Run lazy load module")
-  .requiredOption("-u, --url <url/file>", "Target URL or a file containing a list of URLs (one per line)")
-  .option("-o, --output <directory>", "Output directory", "output")
-  .option("--strict-scope", "Download JS files from only the input URL domain", false)
-  .option("-s, --scope <scope>", "Download JS files from specific domains (comma-separated)", "*")
-  .option("-t, --threads <threads>", "Number of threads to use", 1)
-  .option("--subsequent-requests", "Download JS files from subsequent requests", false)
-  .option("--urls-file <file>", "Input JSON file containing URLs", "extracted_urls.json")
-  .option("--api-gateway", "Generate requests using API Gateway", false)
-  .option("--api-gateway-config <file>", "API Gateway config file", ".api_gateway_config.json")
-  .action(async (cmd) => {
-    globals.setApiGatewayConfigFile(cmd.apiGatewayConfig);
-    globals.setUseApiGateway(cmd.apiGateway);
-    await lazyLoad(cmd.url, cmd.output, cmd.strictScope, cmd.scope.split(","), cmd.threads, cmd.subsequentRequests, cmd.urlsFile);
-  });
-
-program
-  .command("endpoints")
-  .description("Extract API endpoints")
-  .requiredOption("-u, --url <url>", "Target URL")
-  .option("-o, --output <file>", "Output file")
-  .action((cmd) => {
-    endpoints(cmd.url, cmd.output);
-  });
-
-program
-  .command("strings")
-  .description("Extract strings from JS files")
-  .requiredOption("-d, --directory <directory>", "Directory containing JS files")
-  .option("-o, --output <file>", "JSON file to save the strings", "strings.json")
-  .option("-e, --extract-urls", "Extract URLs from strings", false)
-  .option("--extracted-url-path <file>", "Output JSON file for extracted URLs and paths", "extracted_urls.json")
-  .action((cmd) => {
-    strings(cmd.directory, cmd.output, cmd.extractUrls, cmd.extractedUrlPath);
-  });
-
-program
-  .command("api-gateway")
-  .description("Configure AWS API Gateway to rotate IP addresses")
-  .option("-i, --init", "Initialize the config file (create API)", false)
-  .option("-d, --destroy <id>", "Destroy API with the given ID")
-  .option("--destroy-all", "Destroy all the API created by this tool in all regions", false)
-  .option("-r, --region <region>", "AWS region (default: random region)")
-  .option("-a, --access-key <access-key>", "AWS access key (if not provided, AWS_ACCESS_KEY_ID environment variable will be used)")
-  .option("-s, --secret-key <secret-key>", "AWS secret key (if not provided, AWS_SECRET_ACCESS_KEY environment variable will be used)")
-  .option("-c, --config <config>", "Name of the config file", ".api_gateway_config.json")
-  .option("-l, --list", "List all the API created by this tool", false)
-  .option("--feasibility", "Check feasibility of API Gateway", false)
-  .option("--feasibility-url <url>", "URL to check feasibility of")
-  .action((cmd) => {
-    globals.setApiGatewayConfigFile(cmd.config);
-    globals.setUseApiGateway(true);
-    apiGateway(cmd.init, cmd.destroy, cmd.destroyAll, cmd.list, cmd.region, cmd.accessKey, cmd.secretKey, cmd.config, cmd.feasibility, cmd.feasibilityUrl);
-});
-
-program.parse(process.argv);

package/lazyLoad/downloadFilesUtil.js

@@ -1,122 +0,0 @@
-import chalk from "chalk";
-import path from "path";
-import fs from "fs";
-import prettier from "prettier";
-import makeRequest from "../utility/makeReq.js";
-import { getURLDirectory } from "../utility/urlUtils.js";
-import { getScope, getMaxReqQueue } from "./globals.js"; // Import scope and max_req_queue functions
-
-/**
- * Downloads a list of URLs and saves them as files in the specified output directory.
- * It creates the necessary subdirectories based on the URL's host and path.
- * If the URL does not end with `.js`, it is skipped.
- * The function logs the progress and any errors to the console.
- * @param {string[]} urls - An array of URLs to be downloaded.
- * @param {string} output - The directory where the downloaded files will be saved.
- * @returns {Promise<void>}
- */
-const downloadFiles = async (urls, output) => {
-  console.log(
-    chalk.cyan(`[i] Attempting to download ${urls.length} JS chunks`),
-  );
-  fs.mkdirSync(output, { recursive: true });
-
-  // to store ignored JS domain
-  let ignoredJSFiles = [];
-  let ignoredJSDomains = [];
-
-  let download_count = 0;
-  let queue = 0;
-
-  const downloadPromises = urls.map(async (url) => {
-    try {
-      if (url.match(/\.js/)) {
-        // get the directory of the url
-        const { host, directory } = getURLDirectory(url);
-
-        // check scope of file. Only if in scope, download it
-        if (!getScope().includes("*")) {
-          if (!getScope().includes(host)) {
-            ignoredJSFiles.push(url);
-            if (!ignoredJSDomains.includes(host)) {
-              ignoredJSDomains.push(host);
-            }
-            return;
-          }
-        }
-
-        // make the directory inside the output folder
-        const childDir = path.join(output, host, directory);
-        fs.mkdirSync(childDir, { recursive: true });
-
-        // check if queue is full. If so, then wait for random time between
-        // 50 to 300 ms. Then, check again, and loop the process
-        while (queue >= getMaxReqQueue()) {
-          await new Promise((resolve) =>
-            setTimeout(resolve, Math.random() * 250 + 50),
-          );
-        }
-        queue++;
-        const res = await makeRequest(url);
-        queue--;
-
-        const file = `// JS Source: ${url}\n${await res.text()}`;
-        let filename;
-        try {
-          filename = url
-            .split("/")
-            .pop()
-            .match(/[a-zA-Z0-9\.\-_]+\.js/)[0];
-        } catch (err) {
-          // split the URL into multiple chunks. then iterate
-          // through it, and find whatever matches with JS ext
-          const chunks = url.split("/");
-          for (const chunk of chunks) {
-            if (chunk.match(/\.js$/)) {
-              filename = chunk;
-              break;
-            }
-          }
-        }
-        
-        if (!filename) { // Handle cases where filename might not be found
-            console.warn(chalk.yellow(`[!] Could not determine filename for URL: ${url}. Skipping.`));
-            return;
-        }
-
-        const filePath = path.join(childDir, filename);
-        try {
-            fs.writeFileSync(
-                filePath,
-                await prettier.format(file, { parser: "babel" }),
-            );
-        } catch (err) {
-            console.error(chalk.red(`[!] Failed to write file: ${filePath}`));
-        }
-        download_count++;
-      }
-    } catch (err) {
-      console.error(chalk.red(`[!] Failed to download: ${url}`));
-    }
-  });
-
-  await Promise.all(downloadPromises);
-
-  if (ignoredJSFiles.length > 0) {
-    console.log(
-      chalk.yellow(
-        `[i] Ignored ${ignoredJSFiles.length} JS files across ${ignoredJSDomains.length} domain(s) - ${ignoredJSDomains.join(", ")}`,
-      ),
-    );
-  }
-
-  if (download_count > 0) {
-    console.log(
-      chalk.green(
-        `[✓] Downloaded ${download_count} JS chunks to ${output} directory`,
-      ),
-    );
-  }
-};
-
-export default downloadFiles;

package/lazyLoad/downloadLoadedJsUtil.js

@@ -1,54 +0,0 @@
-import chalk from "chalk";
-import puppeteer from "puppeteer";
-
-/**
- * Downloads all the lazy loaded JS files from a given URL.
- *
- * It opens a headless browser instance, navigates to the given URL, and
- * intercepts all the requests. It checks if the request is a JS file
- * and if it is a GET request. If both conditions are satisfied, the URL
- * is added to the array of URLs. Finally, it closes the browser instance
- * and returns the array of URLs.
- *
- * @param {string} url - The URL of the webpage to fetch and parse.
- * @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
- * absolute URLs pointing to JavaScript files found in the page, or undefined for invalid URL.
- */
-const downloadLoadedJs = async (url) => {
-  if (!url.match(/https?:\/\/[a-zA-Z0-9\._\-]+/)) {
-    console.log(chalk.red("[!] Invalid URL"));
-    return; // Return undefined as per JSDoc
-  }
-
-  const browser = await puppeteer.launch({
-    headless: true,
-  });
-
-  const page = await browser.newPage();
-
-  await page.setRequestInterception(true);
-
-  let js_urls_local = []; // Use a local variable, not the global one
-  page.on("request", async (request) => {
-    // get the request url
-    const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
-
-    // see if the request is a JS file, and is a get request
-    if (
-      request.method() === "GET" &&
-      req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)
-    ) {
-      js_urls_local.push(req_url);
-    }
-
-    await request.continue();
-  });
-
-  await page.goto(url);
-
-  await browser.close();
-
-  return js_urls_local;
-};
-
-export default downloadLoadedJs;

package/lazyLoad/globals.js

@@ -1,15 +0,0 @@
-
-let scope = [];
-let js_urls = [];
-let max_req_queue;
-
-export const getScope = () => scope;
-export const setScope = (newScope) => { scope = newScope; };
-export const pushToScope = (item) => { scope.push(item); };
-
-export const getJsUrls = () => js_urls;
-export const clearJsUrls = () => { js_urls = []; };
-export const pushToJsUrls = (url) => { js_urls.push(url); };
-
-export const getMaxReqQueue = () => max_req_queue;
-export const setMaxReqQueue = (newMax) => { max_req_queue = newMax; };

package/lazyLoad/index.js

@@ -1,167 +0,0 @@
-import chalk from "chalk";
-import fs from "fs";
-import frameworkDetect from "../techDetect/index.js";
-import CONFIG from "../globalConfig.js";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-import { URL } from "url";
-
-// Next.js
-import subsequentRequests from "./next_js/next_SubsequentRequests.js";
-import next_getJSScript from "./next_js/next_GetJSScript.js";
-import next_getLazyResources from "./next_js/next_GetLazyResources.js";
-
-// Nuxt.js
-import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
-import nuxt_stringAnalysisJSFiles from "./nuxt_js/nuxt_stringAnalysisJSFiles.js";
-import nuxt_astParse from "./nuxt_js/nuxt_astParse.js";
-
-// generic
-import downloadFiles from "./downloadFilesUtil.js";
-import downloadLoadedJs from "./downloadLoadedJsUtil.js";
-
-// import global vars
-import * as globals from "./globals.js";
-
-/**
- * Downloads all lazy-loaded JavaScript files from the specified URL or file containing URLs.
- *
- * The function detects the JavaScript framework used by the webpage (e.g., Next.js, Nuxt.js)
- * and utilizes specific techniques to find and download lazy-loaded JS files.
- * It supports subsequent requests for additional JS files if specified.
- *
- * @param {string} url - The URL or path to a file containing a list of URLs to process.
- * @param {string} output - The directory where downloaded files will be saved.
- * @param {boolean} strictScope - Whether to restrict downloads to the input URL domain.
- * @param {string[]} inputScope - Specific domains to download JS files from.
- * @param {number} threads - The number of threads to use for downloading files.
- * @param {boolean} subsequentRequestsFlag - Whether to include JS files from subsequent requests.
- * @param {string} urlsFile - The JSON file containing additional URLs for subsequent requests.
- * @returns {Promise<void>}
- */
-const lazyLoad = async (
-  url,
-  output,
-  strictScope,
-  inputScope,
-  threads,
-  subsequentRequestsFlag,
-  urlsFile,
-) => {
-  console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
-
-  let urls;
-
-  // check if the url is file or a URL
-  if (fs.existsSync(url)) {
-    urls = fs.readFileSync(url, "utf-8").split("\n");
-    // remove the empty lines
-    urls = urls.filter((url) => url.trim() !== "");
-  } else if (url.match(/https?:\/\/[a-zA-Z0-9\-_\.:]+/)) {
-    urls = [url];
-  } else {
-    console.log(chalk.red("[!] Invalid URL or file path"));
-    return;
-  }
-
-  for (const url of urls) {
-    console.log(chalk.cyan(`[i] Processing ${url}`));
-
-    if (strictScope) {
-      globals.pushToScope(new URL(url).host);
-    } else {
-      globals.setScope(inputScope);
-    }
-
-    globals.setMaxReqQueue(threads);
-    globals.clearJsUrls(); // Initialize js_urls for each URL processing in the loop
-
-    const tech = await frameworkDetect(url);
-
-    if (tech) {
-      if (tech.name === "next") {
-        console.log(chalk.green("[✓] Next.js detected"));
-        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-
-        // find the JS files from script of the webpage
-        const jsFilesFromScriptTag = await next_getJSScript(url);
-
-        // get lazy resources
-        const lazyResourcesFromWebpack = await next_getLazyResources(url);
-        let lazyResourcesFromSubsequentRequests;
-
-        if (subsequentRequestsFlag) {
-          // get JS files from subsequent requests
-          lazyResourcesFromSubsequentRequests = await subsequentRequests(
-            url,
-            urlsFile,
-            threads,
-            output,
-            globals.getJsUrls(), // Pass the global js_urls
-          );
-        }
-
-        // download the resources
-        // but combine them first
-        let jsFilesToDownload = [
-          ...(jsFilesFromScriptTag || []),
-          ...(lazyResourcesFromWebpack || []),
-          ...(lazyResourcesFromSubsequentRequests || []),
-        ];
-        // Ensure js_urls from globals are included if next_getJSScript or next_getLazyResources populated it.
-        // This is because those functions now push to the global js_urls via setters.
-        // The return values of next_getJSScript and next_getLazyResources might be the same array instance
-        // or a new one depending on their implementation, so explicitly get the global one here.
-        jsFilesToDownload.push(...globals.getJsUrls());
-
-        // dedupe the files
-        jsFilesToDownload = [...new Set(jsFilesToDownload)];
-
-        await downloadFiles(jsFilesToDownload, output);
-      } else if (tech.name === "vue") {
-        console.log(chalk.green("[✓] Vue.js detected"));
-        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-      } else if (tech.name === "nuxt") {
-        console.log(chalk.green("[✓] Nuxt.js detected"));
-        console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-
-        let jsFilesToDownload = [];
-
-        // find the files from the page source
-        const jsFilesFromPageSource = await nuxt_getFromPageSource(url);
-        const jsFilesFromStringAnalysis = await nuxt_stringAnalysisJSFiles(url);
-
-        jsFilesToDownload.push(...jsFilesFromPageSource);
-        jsFilesToDownload.push(...jsFilesFromStringAnalysis);
-        // dedupe the files
-        jsFilesToDownload = [...new Set(jsFilesToDownload)];
-
-        let jsFilesFromAST = [];
-        console.log(chalk.cyan("[i] Analyzing functions in the files found"));
-        for (const jsFile of jsFilesToDownload) {
-          jsFilesFromAST.push(...(await nuxt_astParse(jsFile)));
-        }
-
-        jsFilesToDownload.push(...jsFilesFromAST);
-
-        jsFilesToDownload.push(...globals.getJsUrls());
-
-        // dedupe the files
-        jsFilesToDownload = [...new Set(jsFilesToDownload)];
-
-        await downloadFiles(jsFilesToDownload, output);
-      }
-    } else {
-      console.log(chalk.red("[!] Framework not detected :("));
-      console.log(chalk.magenta(CONFIG.notFoundMessage));
-      console.log(chalk.yellow("[i] Trying to download loaded JS files"));
-      const js_urls = await downloadLoadedJs(url);
-      if (js_urls && js_urls.length > 0) {
-        console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
-        await downloadFiles(js_urls, output);
-      }
-    }
-  }
-};
-
-export default lazyLoad;

package/lazyLoad/next_js/next_GetJSScript.js

@@ -1,99 +0,0 @@
-// lazyLoad/nextGetJSScript.js
-import chalk from "chalk";
-import { URL } from "url";
-import * as cheerio from "cheerio";
-import makeRequest from "../../utility/makeReq.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js";
-
-/**
- * Asynchronously fetches the given URL and extracts JavaScript file URLs
- * from script tags present in the HTML content.
- *
- * @param {string} url - The URL of the webpage to fetch and parse.
- * @returns {Promise<string[]>} - A promise that resolves to an array of
- * absolute URLs pointing to JavaScript files found in script tags.
- */
-const next_getJSScript = async (url) => {
-  // get the page source
-  const res = await makeRequest(url);
-  const pageSource = await res.text();
-
-  // cheerio to parse the page source
-  const $ = cheerio.load(pageSource);
-
-  // find all script tags
-  const scriptTags = $("script");
-
-  // iterate through script tags
-  for (const scriptTag of scriptTags) {
-    // get the src attribute
-    const src = $(scriptTag).attr("src");
-
-    // see if the src is a JS file
-    if (
-      src !== undefined &&
-      src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)) {
-      // if the src starts with /, like `/static/js/a.js` find the absolute URL
-      if (src.startsWith("/")) {
-        const absoluteUrl = new URL(url).origin + src;
-        if (!getJsUrls().includes(absoluteUrl)) {
-          pushToJsUrls(absoluteUrl);
-        }
-      } else if (src.match(/^[^/]/)) {
-        // if the src is a relative URL, like `static/js/a.js` find the absolute URL
-        // Get directory URL (origin + path without filename)
-        const pathParts = new URL(url).pathname.split("/");
-        pathParts.pop(); // remove filename from last
-        const directory = new URL(url).origin + pathParts.join("/") + "/";
-
-        if (!getJsUrls().includes(directory + src)) {
-          pushToJsUrls(directory + src);
-        }
-      } else {
-        if (!getJsUrls().includes(src)) {
-          pushToJsUrls(src);
-        }
-      }
-    } else {
-      // if the script tag is inline, it could contain static JS URL
-      // to get these, simply regex from the JS script
-
-      const js_script = $(scriptTag).html();
-      const matches = js_script.match(/static\/chunks\/[a-zA-Z0-9_\-]+\.js/g);
-
-      if (matches) {
-        const uniqueMatches = [...new Set(matches)];
-        for (const match of uniqueMatches) {
-          // if it is using that static/chunks/ pattern, I can just get the filename
-          const filename = match.replace("static/chunks/", "");
-
-          // go through the already found URLs, coz they will have it (src attribute
-          // is there before inline things)
-
-          let js_path_dir;
-
-          for (const js_url of getJsUrls()) {
-            if (
-              !js_path_dir &&
-              new URL(js_url).host === new URL(url).host &&
-              new URL(js_url).pathname.includes("static/chunks/")
-            ) {
-              js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
-            }
-          }
-          if (js_path_dir) { // Ensure js_path_dir was found
-            pushToJsUrls(js_path_dir + "/" + filename);
-          }
-        }
-      }
-    }
-  }
-
-  console.log(
-    chalk.green(`[✓] Found ${getJsUrls().length} JS files from the script tags`),
-  );
-
-  return getJsUrls();
-};
-
-export default next_getJSScript;