@shriyanss/js-recon 1.0.0

package/api_gateway/index.js

@@ -0,0 +1,325 @@
+import chalk from "chalk";
+import {
+  APIGatewayClient,
+  CreateRestApiCommand,
+  DeleteRestApiCommand,
+} from "@aws-sdk/client-api-gateway";
+import fs from "fs";
+import checkFeasibility from "./checkFeasibility.js";
+
+// read the docs for all the methods for api gateway at https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/api-gateway/
+// for the rate limits, refer to https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+
+const randomRegion = () => {
+  const apiGatewayRegions = [
+    "us-east-2", // US East (Ohio)
+    "us-east-1", // US East (N. Virginia)
+    "us-west-1", // US West (N. California)
+    "us-west-2", // US West (Oregon)
+    "af-south-1", // Africa (Cape Town)
+    "ap-east-1", // Asia Pacific (Hong Kong)
+    "ap-south-2", // Asia Pacific (Hyderabad)
+    "ap-southeast-3", // Asia Pacific (Jakarta)
+    "ap-southeast-5", // Asia Pacific (Malaysia)
+    "ap-southeast-4", // Asia Pacific (Melbourne)
+    "ap-south-1", // Asia Pacific (Mumbai)
+    "ap-northeast-3", // Asia Pacific (Osaka)
+    "ap-northeast-2", // Asia Pacific (Seoul)
+    "ap-southeast-1", // Asia Pacific (Singapore)
+    "ap-southeast-2", // Asia Pacific (Sydney)
+    "ap-east-2", // Asia Pacific (Taipei)
+    "ap-southeast-7", // Asia Pacific (Thailand)
+    "ap-northeast-1", // Asia Pacific (Tokyo)
+    "ca-central-1", // Canada (Central)
+    "ca-west-1", // Canada West (Calgary)
+    "eu-central-1", // Europe (Frankfurt)
+    "eu-west-1", // Europe (Ireland)
+    "eu-west-2", // Europe (London)
+    "eu-south-1", // Europe (Milan)
+    "eu-west-3", // Europe (Paris)
+    "eu-south-2", // Europe (Spain)
+    "eu-north-1", // Europe (Stockholm)
+    "eu-central-2", // Europe (Zurich)
+    "il-central-1", // Israel (Tel Aviv)
+    "mx-central-1", // Mexico (Central)
+    "me-south-1", // Middle East (Bahrain)
+    "me-central-1", // Middle East (UAE)
+    "sa-east-1", // South America (São Paulo)
+  ];
+  return apiGatewayRegions[
+    Math.floor(Math.random() * apiGatewayRegions.length)
+  ];
+};
+
+let aws_access_key;
+let aws_secret_key;
+let region;
+let configFile;
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+  /**
+   * Create a new API Gateway.
+   *
+   * @async
+   * @returns {Promise<void>}
+   */
+const createGateway = async () => {
+  console.log(chalk.cyan("[i] Creating API Gateway"));
+  const client = new APIGatewayClient({
+    region,
+    credentials: {
+      accessKeyId: aws_access_key,
+      secretAccessKey: aws_secret_key,
+    },
+  });
+
+  const apigw_created_at = Date.now();
+  const apigw_name = `js_recon-${apigw_created_at}-${Math.floor(Math.random() * 1000)}`;
+  const command = new CreateRestApiCommand({
+    name: apigw_name,
+    description: `API Gateway for JS Recon created at ${new Intl.DateTimeFormat(
+      "en-US",
+      {
+        year: "numeric",
+        month: "long",
+        day: "2-digit",
+        hour: "2-digit",
+        minute: "2-digit",
+        second: "2-digit",
+        timeZoneName: "short",
+      },
+    ).format(apigw_created_at)}`,
+    endpointConfiguration: {
+      ipAddressType: "dualstack",
+      types: ["REGIONAL"],
+    },
+  });
+  const response = await client.send(command);
+  await sleep(3000);
+  console.log(chalk.green(`[✓] Created API Gateway`));
+  console.log(chalk.bgGreen("ID:"), chalk.green(response.id));
+  console.log(chalk.bgGreen("Name:"), chalk.green(apigw_name));
+  console.log(chalk.bgGreen("Region:"), chalk.green(region));
+
+  // load the config file if any. Else, create a new one
+  let config = {};
+  try {
+    config = JSON.parse(fs.readFileSync(configFile));
+  } catch (e) {
+    config = {};
+  }
+
+  config[apigw_name] = {
+    id: response.id,
+    name: apigw_name,
+    description: response.description,
+    created_at: apigw_created_at,
+    region: region,
+    access_key: aws_access_key,
+    secret_key: aws_secret_key,
+  };
+
+  fs.writeFileSync(configFile, JSON.stringify(config, null, 2));
+  console.log(chalk.green(`[✓] Config saved to ${configFile}`));
+};
+
+/**
+ * Destroy an API Gateway.
+ *
+ * @async
+ * @param {string} id - The ID of the API Gateway to destroy.
+ * @returns {Promise<void>}
+ */
+const destroyGateway = async (id) => {
+  console.log(chalk.cyan("[i] Destroying API Gateway"));
+  if (!id) {
+    console.log(chalk.red("[!] Please provide an API Gateway ID"));
+    return;
+  }
+  //   read the config file
+  let config = JSON.parse(fs.readFileSync(configFile));
+  //   get the name of the api gateway
+  let name = Object.keys(config).find((key) => config[key].id === id);
+
+  console.log(chalk.bgGreen("Name:"), chalk.green(name));
+  console.log(chalk.bgGreen("ID:"), chalk.green(id));
+  console.log(chalk.bgGreen("Region:"), chalk.green(config[name].region));
+  region = config[name].region;
+
+  const client = new APIGatewayClient({
+    region,
+    credentials: {
+      accessKeyId: aws_access_key,
+      secretAccessKey: aws_secret_key,
+    },
+  });
+
+  const command = new DeleteRestApiCommand({
+    restApiId: id,
+  });
+  await client.send(command);
+
+  // remove from the config file
+  delete config[name];
+  fs.writeFileSync(configFile, JSON.stringify(config, null, 2));
+
+  await sleep(30000);
+
+  console.log(chalk.green(`[✓] Destroyed API Gateway: ${id}`));
+};
+
+/**
+ * Destroy all API Gateways.
+ *
+ * @async
+ * @returns {Promise<void>}
+ */
+const destroyAllGateways = async () => {
+  console.log(chalk.cyan("[i] Destroying all API Gateways"));
+  //   read the config file
+  let config = JSON.parse(fs.readFileSync(configFile));
+
+  //   destroy all the gateways
+  for (const [key, value] of Object.entries(config)) {
+    const client = new APIGatewayClient({
+      region: value.region,
+      credentials: {
+        accessKeyId: aws_access_key,
+        secretAccessKey: aws_secret_key,
+      },
+    });
+    console.log(
+      chalk.cyan(
+        `[i] Destroying API Gateway: ${key} : ${value.id} : ${value.region}`,
+      ),
+    );
+
+    const command = new DeleteRestApiCommand({
+      restApiId: value.id,
+    });
+    await sleep(30000);
+    await client.send(command);
+    console.log(
+      chalk.green(
+        `[✓] Destroyed API Gateway: ${key} : ${value.id} : ${value.region}`,
+      ),
+    );
+  }
+
+  // nullify the config file
+  fs.writeFileSync(configFile, JSON.stringify({}, null, 2));
+  console.log(chalk.green("[✓] Destroyed all API Gateways"));
+};
+
+/**
+ * List all API Gateways.
+ *
+ * @async
+ * @returns {Promise<void>}
+ */
+const listGateways = async () => {
+  console.log(chalk.cyan("[i] Listing all API Gateways"));
+
+  // read the config file, and list these
+
+  // check if the config file exists
+  if (!fs.existsSync(configFile)) {
+    console.log(chalk.red("[!] Config file does not exist"));
+    return;
+  }
+
+  const config = JSON.parse(fs.readFileSync(configFile));
+
+  //   if list is empty
+  if (Object.keys(config).length === 0) {
+    console.log(chalk.red("[!] No API Gateways found"));
+    return;
+  }
+
+  console.log(chalk.green("[✓] List of API Gateways"));
+
+  for (const [key, value] of Object.entries(config)) {
+    console.log(chalk.bgGreen("Name:"), chalk.green(key));
+    console.log(chalk.bgGreen("ID:"), chalk.green(value.id));
+    console.log(chalk.bgGreen("Region:"), chalk.green(value.region));
+    console.log("\n");
+  }
+};
+
+/**
+ * Main function for API Gateway.
+ *
+ * @async
+ * @param {boolean} initInput - Whether to initialize the API Gateway.
+ * @param {string} destroyInput - The ID of the API Gateway to destroy.
+ * @param {boolean} destroyAllInput - Whether to destroy all API Gateways.
+ * @param {boolean} listInput - Whether to list all API Gateways.
+ * @param {string} regionInput - The region to use.
+ * @param {string} accessKey - The access key to use.
+ * @param {string} secretKey - The secret key to use.
+ * @param {string} configInput - The config file to use.
+ * @returns {Promise<void>}
+ */
+const apiGateway = async (
+  initInput,
+  destroyInput,
+  destroyAllInput,
+  listInput,
+  regionInput,
+  accessKey,
+  secretKey,
+  configInput,
+  feasibilityInput,
+  feasibilityUrlInput,
+) => {
+  console.log(chalk.cyan("[i] Loading 'API Gateway' module"));
+
+  // if feasibility is true, check feasibility
+  if (feasibilityInput) {
+    if (!feasibilityUrlInput) {
+      console.log(chalk.red("[!] Please provide a URL to check feasibility of"));
+      return;
+    }
+    await checkFeasibility(feasibilityUrlInput);
+    return;
+  }
+
+  // configure the access and secret key
+  aws_access_key = accessKey || process.env.AWS_ACCESS_KEY_ID || undefined;
+  aws_secret_key = secretKey || process.env.AWS_SECRET_ACCESS_KEY || undefined;
+  region = regionInput || randomRegion();
+  configFile = configInput || "config.json";
+
+  if (!aws_access_key || !aws_secret_key) {
+    console.log(chalk.red("[!] AWS Access Key or Secret Key not found"));
+    return;
+  }
+
+  console.log(chalk.cyan(`[i] Using region: ${region}`));
+
+  const keyMask = (key) => {
+    if (key.length < 6) return key;
+    return key.slice(0, 4) + "..." + key.slice(-4);
+  };
+  console.log(chalk.cyan(`[i] Using access key: ${keyMask(aws_access_key)}`));
+
+  // create a new API gateway
+  if (initInput) {
+    await createGateway();
+  } else if (destroyInput) {
+    await destroyGateway(destroyInput);
+  } else if (destroyAllInput) {
+    await destroyAllGateways();
+  } else if (listInput) {
+    await listGateways();
+  } else {
+    console.log(
+      chalk.red(
+        "[!] Please provide a valid action (-i/--init or -d/--destroy or --destroy-all)",
+      ),
+    );
+  }
+};
+
+export default apiGateway;

package/endpoints/index.js

@@ -0,0 +1,7 @@
+import chalk from "chalk";
+
+const endpoints = () => {
+    console.log(chalk.green("Feature under development. Check back later :)"));
+};
+
+export default endpoints;

package/globalConfig.js

@@ -0,0 +1,12 @@
+const githubURL = "https://github.com/shriyanss/js-recon"
+const version = "1.0.0";
+const toolDesc = "JS Recon Tool";   
+
+global.CONFIG = {
+    github: githubURL,
+    notFoundMessage: `If you believe this is an error or is a new technology, please create an issue on ${githubURL} and we'll figure it out for you`,
+    version: version,
+    toolDesc: toolDesc
+};
+
+export default CONFIG;

package/index.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+import { program } from "commander";
+import lazyLoad from "./lazyLoad/index.js";
+import endpoints from "./endpoints/index.js";
+import CONFIG from "./globalConfig.js";
+import strings from "./strings/index.js";
+import apiGateway from "./api_gateway/index.js";
+import * as globals from "./utility/globals.js";
+
+program.version(CONFIG.version).description(CONFIG.toolDesc);
+
+program
+  .command("lazyload")
+  .description("Run lazy load module")
+  .requiredOption("-u, --url <url/file>", "Target URL or a file containing a list of URLs (one per line)")
+  .option("-o, --output <directory>", "Output directory", "output")
+  .option("--strict-scope", "Download JS files from only the input URL domain", false)
+  .option("-s, --scope <scope>", "Download JS files from specific domains (comma-separated)", "*")
+  .option("-t, --threads <threads>", "Number of threads to use", 1)
+  .option("--subsequent-requests", "Download JS files from subsequent requests", false)
+  .option("--urls-file <file>", "Input JSON file containing URLs", "extracted_urls.json")
+  .option("--api-gateway", "Generate requests using API Gateway", false)
+  .option("--api-gateway-config <file>", "API Gateway config file", ".api_gateway_config.json")
+  .action(async (cmd) => {
+    globals.setApiGatewayConfigFile(cmd.apiGatewayConfig);
+    globals.setUseApiGateway(cmd.apiGateway);
+    await lazyLoad(cmd.url, cmd.output, cmd.strictScope, cmd.scope.split(","), cmd.threads, cmd.subsequentRequests, cmd.urlsFile);
+  });
+
+program
+  .command("endpoints")
+  .description("Extract API endpoints")
+  .requiredOption("-u, --url <url>", "Target URL")
+  .option("-o, --output <file>", "Output file")
+  .action((cmd) => {
+    endpoints(cmd.url, cmd.output);
+  });
+
+program
+  .command("strings")
+  .description("Extract strings from JS files")
+  .requiredOption("-d, --directory <directory>", "Directory containing JS files")
+  .option("-o, --output <file>", "JSON file to save the strings", "strings.json")
+  .option("-e, --extract-urls", "Extract URLs from strings", false)
+  .option("--extracted-url-path <file>", "Output JSON file for extracted URLs and paths", "extracted_urls.json")
+  .action((cmd) => {
+    strings(cmd.directory, cmd.output, cmd.extractUrls, cmd.extractedUrlPath);
+  });
+
+program
+  .command("api-gateway")
+  .description("Configure AWS API Gateway to rotate IP addresses")
+  .option("-i, --init", "Initialize the config file (create API)", false)
+  .option("-d, --destroy <id>", "Destroy API with the given ID")
+  .option("--destroy-all", "Destroy all the API created by this tool in all regions", false)
+  .option("-r, --region <region>", "AWS region (default: random region)")
+  .option("-a, --access-key <access-key>", "AWS access key (if not provided, AWS_ACCESS_KEY_ID environment variable will be used)")
+  .option("-s, --secret-key <secret-key>", "AWS secret key (if not provided, AWS_SECRET_ACCESS_KEY environment variable will be used)")
+  .option("-c, --config <config>", "Name of the config file", ".api_gateway_config.json")
+  .option("-l, --list", "List all the API created by this tool", false)
+  .option("--feasibility", "Check feasibility of API Gateway", false)
+  .option("--feasibility-url <url>", "URL to check feasibility of")
+  .action((cmd) => {
+    globals.setApiGatewayConfigFile(cmd.config);
+    globals.setUseApiGateway(true);
+    apiGateway(cmd.init, cmd.destroy, cmd.destroyAll, cmd.list, cmd.region, cmd.accessKey, cmd.secretKey, cmd.config, cmd.feasibility, cmd.feasibilityUrl);
+});
+
+program.parse(process.argv);

package/lazyLoad/downloadFilesUtil.js

@@ -0,0 +1,122 @@
+import chalk from "chalk";
+import path from "path";
+import fs from "fs";
+import prettier from "prettier";
+import makeRequest from "../utility/makeReq.js";
+import { getURLDirectory } from "../utility/urlUtils.js";
+import { getScope, getMaxReqQueue } from "./globals.js"; // Import scope and max_req_queue functions
+
+/**
+ * Downloads a list of URLs and saves them as files in the specified output directory.
+ * It creates the necessary subdirectories based on the URL's host and path.
+ * If the URL does not end with `.js`, it is skipped.
+ * The function logs the progress and any errors to the console.
+ * @param {string[]} urls - An array of URLs to be downloaded.
+ * @param {string} output - The directory where the downloaded files will be saved.
+ * @returns {Promise<void>}
+ */
+const downloadFiles = async (urls, output) => {
+  console.log(
+    chalk.cyan(`[i] Attempting to download ${urls.length} JS chunks`),
+  );
+  fs.mkdirSync(output, { recursive: true });
+
+  // to store ignored JS domain
+  let ignoredJSFiles = [];
+  let ignoredJSDomains = [];
+
+  let download_count = 0;
+  let queue = 0;
+
+  const downloadPromises = urls.map(async (url) => {
+    try {
+      if (url.match(/\.js/)) {
+        // get the directory of the url
+        const { host, directory } = getURLDirectory(url);
+
+        // check scope of file. Only if in scope, download it
+        if (!getScope().includes("*")) {
+          if (!getScope().includes(host)) {
+            ignoredJSFiles.push(url);
+            if (!ignoredJSDomains.includes(host)) {
+              ignoredJSDomains.push(host);
+            }
+            return;
+          }
+        }
+
+        // make the directory inside the output folder
+        const childDir = path.join(output, host, directory);
+        fs.mkdirSync(childDir, { recursive: true });
+
+        // check if queue is full. If so, then wait for random time between
+        // 50 to 300 ms. Then, check again, and loop the process
+        while (queue >= getMaxReqQueue()) {
+          await new Promise((resolve) =>
+            setTimeout(resolve, Math.random() * 250 + 50),
+          );
+        }
+        queue++;
+        const res = await makeRequest(url);
+        queue--;
+
+        const file = `// JS Source: ${url}\n${await res.text()}`;
+        let filename;
+        try {
+          filename = url
+            .split("/")
+            .pop()
+            .match(/[a-zA-Z0-9\.\-_]+\.js/)[0];
+        } catch (err) {
+          // split the URL into multiple chunks. then iterate
+          // through it, and find whatever matches with JS ext
+          const chunks = url.split("/");
+          for (const chunk of chunks) {
+            if (chunk.match(/\.js$/)) {
+              filename = chunk;
+              break;
+            }
+          }
+        }
+        
+        if (!filename) { // Handle cases where filename might not be found
+            console.warn(chalk.yellow(`[!] Could not determine filename for URL: ${url}. Skipping.`));
+            return;
+        }
+
+        const filePath = path.join(childDir, filename);
+        try {
+            fs.writeFileSync(
+                filePath,
+                await prettier.format(file, { parser: "babel" }),
+            );
+        } catch (err) {
+            console.error(chalk.red(`[!] Failed to write file: ${filePath}`));
+        }
+        download_count++;
+      }
+    } catch (err) {
+      console.error(chalk.red(`[!] Failed to download: ${url}`));
+    }
+  });
+
+  await Promise.all(downloadPromises);
+
+  if (ignoredJSFiles.length > 0) {
+    console.log(
+      chalk.yellow(
+        `[i] Ignored ${ignoredJSFiles.length} JS files across ${ignoredJSDomains.length} domain(s) - ${ignoredJSDomains.join(", ")}`,
+      ),
+    );
+  }
+
+  if (download_count > 0) {
+    console.log(
+      chalk.green(
+        `[✓] Downloaded ${download_count} JS chunks to ${output} directory`,
+      ),
+    );
+  }
+};
+
+export default downloadFiles;

package/lazyLoad/downloadLoadedJsUtil.js

@@ -0,0 +1,54 @@
+import chalk from "chalk";
+import puppeteer from "puppeteer";
+
+/**
+ * Downloads all the lazy loaded JS files from a given URL.
+ *
+ * It opens a headless browser instance, navigates to the given URL, and
+ * intercepts all the requests. It checks if the request is a JS file
+ * and if it is a GET request. If both conditions are satisfied, the URL
+ * is added to the array of URLs. Finally, it closes the browser instance
+ * and returns the array of URLs.
+ *
+ * @param {string} url - The URL of the webpage to fetch and parse.
+ * @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
+ * absolute URLs pointing to JavaScript files found in the page, or undefined for invalid URL.
+ */
+const downloadLoadedJs = async (url) => {
+  if (!url.match(/https?:\/\/[a-zA-Z0-9\._\-]+/)) {
+    console.log(chalk.red("[!] Invalid URL"));
+    return; // Return undefined as per JSDoc
+  }
+
+  const browser = await puppeteer.launch({
+    headless: true,
+  });
+
+  const page = await browser.newPage();
+
+  await page.setRequestInterception(true);
+
+  let js_urls_local = []; // Use a local variable, not the global one
+  page.on("request", async (request) => {
+    // get the request url
+    const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
+
+    // see if the request is a JS file, and is a get request
+    if (
+      request.method() === "GET" &&
+      req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)
+    ) {
+      js_urls_local.push(req_url);
+    }
+
+    await request.continue();
+  });
+
+  await page.goto(url);
+
+  await browser.close();
+
+  return js_urls_local;
+};
+
+export default downloadLoadedJs;

package/lazyLoad/globals.js

@@ -0,0 +1,15 @@
+
+let scope = [];
+let js_urls = [];
+let max_req_queue;
+
+export const getScope = () => scope;
+export const setScope = (newScope) => { scope = newScope; };
+export const pushToScope = (item) => { scope.push(item); };
+
+export const getJsUrls = () => js_urls;
+export const clearJsUrls = () => { js_urls = []; };
+export const pushToJsUrls = (url) => { js_urls.push(url); };
+
+export const getMaxReqQueue = () => max_req_queue;
+export const setMaxReqQueue = (newMax) => { max_req_queue = newMax; };