@shriyanss/js-recon 1.0.0 → 1.1.0-beta.1

package/lazyLoad/next_js/next_SubsequentRequests.js

@@ -1,138 +0,0 @@
-import chalk from "chalk";
-import fs from "fs";
-import path from "path";
-import { getURLDirectory } from "../../utility/urlUtils.js";
-// custom request module
-import makeRequest from "../../utility/makeReq.js";
-
-let queue = [];
-let max_queue;
-
-/**
- * Given a string of JS content, it finds all the static files used in the
- * file, and returns them as an array.
- *
- * @param {string} js_content - The string of JS content to search through.
- *
- * @returns {string[]} An array of strings, each string being a static file
- * path.
- */
-const findStaticFiles = async (js_content) => {
-  // do some regex-ing
-  const matches = [
-    ...js_content.matchAll(/\/?static\/chunks\/[a-zA-Z0-9\._\-\/]+\.js/g),
-  ];
-  // return matches
-
-  let toReturn = [];
-
-  for (const match of matches) {
-    toReturn.push(match[0]);
-  }
-
-  return toReturn;
-};
-
-const getURLDirectoryServer = (urlString) => {
-  const url = new URL(urlString);
-  const pathParts = url.pathname.split("/").filter(Boolean); // ['business', 'api']
-  pathParts.pop(); // Remove 'api'
-
-  const newPath = "/" + pathParts.join("/"); // '/business'
-  return `${url.origin}${newPath}`; // 'http://something.com/business'
-};
-
-const subsequentRequests = async (url, urlsFile, threads, output, js_urls) => {
-  max_queue = threads;
-  let staticJSURLs = [];
-
-  console.log(chalk.cyan(`[i] Fetching JS files from subsequent requests`));
-
-  // open the urls file, and load the paths (JSON)
-  const endpoints = JSON.parse(fs.readFileSync(urlsFile, "utf-8")).paths;
-
-  let js_contents = {};
-
-  // make requests to all of them with the special header
-  const reqPromises = endpoints.map(async (endpoint) => {
-    const reqUrl = url + endpoint;
-    try {
-      // delay in case over the thread count
-      while (queue >= max_queue) {
-        await new Promise((resolve) => setTimeout(resolve, 100));
-      }
-      queue++;
-
-      const res = await makeRequest(reqUrl, {
-        headers: {
-          RSC: "1",
-        },
-      });
-
-      if (
-        res &&
-        res.status === 200 &&
-        res.headers.get("content-type").includes("text/x-component")
-      ) {
-        const text = await res.text();
-        js_contents[endpoint] = text;
-
-        const { host, directory } = getURLDirectory(reqUrl);
-
-        // save the contents to "___subsequent_requests/"
-        // make the subsequent_requests directory if it doesn't exist
-
-        const output_path = path.join(
-          output,
-          host,
-          "___subsequent_requests",
-          directory,
-        );
-        if (!fs.existsSync(output_path)) {
-          fs.mkdirSync(output_path);
-        }
-        fs.writeFileSync(path.join(output_path, "index.js"), text);
-
-        // find the static ones from the JS resp
-        const staticFiles = await findStaticFiles(text);
-
-        // go through each file and get the absolute path of those
-        const absolutePaths = staticFiles.map((file) => {
-          // go through existing JS URLs found
-          let js_path_dir;
-          for (const js_url of js_urls) {
-            if (
-              !js_path_dir &&
-              new URL(js_url).host === new URL(url).host &&
-              new URL(js_url).pathname.includes("static/chunks/")
-            ) {
-              js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
-            }
-          }
-          return js_path_dir.replace("static/chunks", "") + file;
-        });
-
-        // Filter out paths that are already in js_urls before pushing to staticJSURLs
-        const newPaths = absolutePaths.filter(path => !js_urls.includes(path));
-        if (newPaths.length > 0) {
-          staticJSURLs.push(...newPaths);
-        }
-      }
-
-      queue--;
-    } catch (e) {
-      queue--;
-      console.log(chalk.red(`[!] Error fetching ${reqUrl}: ${e}`));
-    }
-  });
-
-  await Promise.all(reqPromises);
-
-  staticJSURLs = [...new Set(staticJSURLs)];
-
-  console.log(chalk.green(`[✓] Found ${staticJSURLs.length} JS chunks from subsequent requests`));
-
-  return staticJSURLs;
-};
-
-export default subsequentRequests;

package/lazyLoad/nuxt_js/nuxt_astParse.js

@@ -1,194 +0,0 @@
-import parser from "@babel/parser";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-import execFunc from "../../utility/runSandboxed.js";
-import makeRequest from "../../utility/makeReq.js";
-import chalk from "chalk";
-import inquirer from "inquirer";
-import t from "@babel/types";
-import resolvePath from "../../utility/resolvePath.js";
-
-const nuxt_astParse = async (url) => {
-  let filesFound = [];
-  const resp = await makeRequest(url);
-  const body = await resp.text();
-
-  let ast;
-  try {
-    ast = parser.parse(body, {
-      sourceType: "module",
-      plugins: ["jsx", "typescript"],
-    });
-  } catch (error) {
-    console.log(chalk.red("[!] Error parsing JS file: ", url));
-    return filesFound;
-  }
-
-  let functions = [];
-
-  traverse(ast, {
-    FunctionDeclaration(path) {
-      functions.push({
-        name: path.node.id?.name || "(anonymous)",
-        type: "FunctionDeclaration",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    FunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "FunctionExpression",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ArrowFunctionExpression(path) {
-      functions.push({
-        name: path.parent.id?.name || "(anonymous)",
-        type: "ArrowFunctionExpression",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ObjectMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ObjectMethod",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-    ClassMethod(path) {
-      functions.push({
-        name: path.node.key.name,
-        type: "ClassMethod",
-        source: body.slice(path.node.start, path.node.end),
-      });
-    },
-  });
-
-  // iterate through the functions, and find out the one that ends with ".js"
-
-  for (const func of functions) {
-    if (func.source.match(/"\.js".{0,15}$/)) {
-      console.log(
-        chalk.green(`[✓] Found JS chunk having the following source:`),
-      );
-      console.log(chalk.yellow(func.source));
-
-      // ask for confirmation, then proceed
-      const askCorrectFuncConfirmation = async () => {
-        const { value } = await inquirer.prompt([
-          {
-            type: "confirm",
-            name: "value",
-            message: "Is this the correct function?",
-            default: true,
-          },
-        ]);
-        return value;
-      };
-
-      const user_verified = await askCorrectFuncConfirmation();
-      if (user_verified === true) {
-        console.log(
-          chalk.cyan(
-            "[i] Proceeding with the selected function to fetch files",
-          ),
-        );
-
-        // get the value of the unknown vars
-        // first, get the name of the unknown function
-        const unknownVarAst = parser.parse(`(${func.source})`, {
-          sourceType: "script",
-          plugins: ["jsx", "typescript"],
-        });
-        let memberExpressions = [];
-        traverse(unknownVarAst, {
-          MemberExpression(path) {
-            // Only collect identifiers like f.p (not obj["x"])
-            if (
-              t.isIdentifier(path.node.object) &&
-              t.isIdentifier(path.node.property) &&
-              !path.node.computed // ignore obj["x"]
-            ) {
-              const objName = path.node.object.name;
-              const propName = path.node.property.name;
-              memberExpressions.push(`${objName}.${propName}`);
-            }
-          },
-        });
-
-        const unknownVar = memberExpressions[0].split(".");
-
-        // now, resolve the value of this unknown var
-        let unknownVarValue;
-
-        traverse(ast, {
-          AssignmentExpression(path) {
-            const { left, right } = path.node;
-
-            if (
-              t.isMemberExpression(left) &&
-              t.isIdentifier(left.object, { name: unknownVar[0] }) &&
-              t.isIdentifier(left.property, { name: unknownVar[1] }) &&
-              !left.computed
-            ) {
-              if (t.isStringLiteral(right)) {
-                unknownVarValue = right.value;
-              } else {
-                // fallback to source snippet
-                unknownVarValue = func.source.slice(right.start, right.end);
-              }
-            }
-          },
-        });
-
-        // replace the unknown var with the value
-        const funcSource = func.source.replace(
-          new RegExp(`${unknownVar[0]}.${unknownVar[1]}`),
-          `"${unknownVarValue}"`,
-        );
-
-        // continue to executing the function with all possible numbers
-        const urlBuilderFunc = `(() => (${funcSource}))()`;
-        let js_paths = [];
-
-        try {
-          // rather than fuzzing, grep the integers from the func code
-          const integers = funcSource.match(/\d+/g);
-          if (integers) {
-            // Check if integers were found
-            // iterate through all integers, and get the output
-            for (const i of integers) {
-              const output = execFunc(urlBuilderFunc, parseInt(i));
-              if (output.includes("undefined")) {
-                continue;
-              } else {
-                js_paths.push(output);
-              }
-            }
-          }
-        } catch (error) {
-          console.log(chalk.red("[!] Error executing function: ", error));
-        }
-
-        if (js_paths.length > 0) {
-          // iterate through the files, and resolve them
-          for (const js_path of js_paths) {
-            const resolvedPath = await resolvePath(url, js_path);
-            filesFound.push(resolvedPath);
-          }
-        }
-      } else {
-        console.log(chalk.red("[!] Not executing function."));
-        continue;
-      }
-    }
-  }
-
-  if (filesFound.length > 0) {
-    console.log(chalk.green(`[✓] Found ${filesFound.length} JS chunks`));
-  }
-
-  return filesFound;
-};
-
-export default nuxt_astParse;

package/lazyLoad/nuxt_js/nuxt_getFromPageSource.js

@@ -1,77 +0,0 @@
-import chalk from "chalk";
-import makeRequest from "../../utility/makeReq.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js";
-import * as cheerio from "cheerio";
-
-const nuxt_getFromPageSource = async (url) => {
-  console.log(chalk.cyan("[i] Analyzing page source"));
-
-  // get the page source
-  const res = await makeRequest(url);
-  const pageSource = await res.text();
-
-  // cheerio to parse the page source
-  const $ = cheerio.load(pageSource);
-
-  // find all link tags
-  const linkTags = $("link");
-
-  // go through them, and find the ones which have `as=script` attr
-  for (const linkTag of linkTags) {
-    const asAttr = $(linkTag).attr("as");
-    if (asAttr === "script") {
-      const hrefAttr = $(linkTag).attr("href");
-      if (hrefAttr) {
-        // see if it starts with /_nuxt
-        if (hrefAttr.startsWith("/_nuxt")) {
-          // get the URL root, and append the hrefAttr to it
-          const urlRoot = new URL(url).origin;
-          pushToJsUrls(urlRoot + hrefAttr);
-        }
-      }
-    }
-  }
-
-  // now, search all the script tags
-  const scriptTags = $("script");
-  for (const scriptTag of scriptTags) {
-    const src = $(scriptTag).attr("src");
-    if (
-      src !== undefined &&
-      src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)
-    ) {
-      if (src.startsWith("http")) {
-        if (!getJsUrls().includes(src)) {
-          pushToJsUrls(src);
-        }
-      }
-      // if the src starts with /, like `/static/js/a.js` find the absolute URL
-      else if (src.startsWith("/")) {
-        const absoluteUrl = new URL(url).origin + src;
-        if (!getJsUrls().includes(absoluteUrl)) {
-          pushToJsUrls(absoluteUrl);
-        }
-      } else if (src.match(/^[^/]/)) {
-        // if the src is a relative URL, like `static/js/a.js` find the absolute URL
-        // Get directory URL (origin + path without filename)
-        const pathParts = new URL(url).pathname.split("/");
-        pathParts.pop(); // remove the filename from the path
-        const directory = new URL(url).origin + pathParts.join("/") + "/";
-
-        if (!getJsUrls().includes(directory + src)) {
-          pushToJsUrls(directory + src);
-        }
-      } else {
-        continue;
-      }
-    }
-  }
-
-  console.log(
-    chalk.green(`[✓] Found ${getJsUrls().length} JS files from the page source`),
-  );
-
-  return getJsUrls();
-};
-
-export default nuxt_getFromPageSource;

package/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js

@@ -1,99 +0,0 @@
-import chalk from "chalk";
-import makeRequest from "../../utility/makeReq.js";
-import { getJsUrls, pushToJsUrls } from "../globals.js";
-import resolvePath from "../../utility/resolvePath.js";
-
-// for parsing
-import parser from "@babel/parser";
-import _traverse from "@babel/traverse";
-const traverse = _traverse.default;
-
-let analyzedFiles = [];
-let filesFound = [];
-
-const parseJSFileContent = async (content) => {
-  try {
-    const ast = parser.parse(content, {
-      sourceType: "module",
-      plugins: ["jsx", "typescript"],
-    });
-
-    let foundJsFiles = {};
-
-    traverse(ast, {
-      StringLiteral(path) {
-        const value = path.node.value;
-        if (value.startsWith("./") && value.endsWith(".js")) {
-          foundJsFiles[value] = value;
-        } else if (value.startsWith("../") && value.endsWith(".js")) {
-          foundJsFiles[value] = value;
-        }
-      },
-    });
-
-    return foundJsFiles;
-  } catch (error) {
-    return {};
-  }
-};
-
-const nuxt_stringAnalysisJSFiles = async (url) => {
-  console.log(chalk.cyan("[i] Analyzing strings in the files found"));
-
-  while (true) {
-    // get all the JS URLs
-    const js_urls = getJsUrls();
-
-    if (js_urls.length === 0) {
-      console.log(chalk.red("[!] No JS files found for string analysis"));
-      break;
-    }
-
-    // if js_urls have everything that is in analyzedFiles, break
-    let everythingAnalyzed = true;
-    for (const url of js_urls) {
-      //   if the url is not in analyzedFiles, set everythingAnalyzed to false
-      if (!analyzedFiles.includes(url)) {
-        everythingAnalyzed = false;
-      }
-    }
-
-    // break if everything is analyzed
-    if (everythingAnalyzed) {
-      break;
-    }
-
-    // iterate through the JS URLs
-    for (const js_url of js_urls) {
-      if (analyzedFiles.includes(js_url)) {
-        continue;
-      }
-
-      const response = await makeRequest(js_url);
-      const respText = await response.text();
-      const foundJsFiles = await parseJSFileContent(respText);
-
-      // iterate through the foundJsFiles and resolve the paths
-      for (const [key, value] of Object.entries(foundJsFiles)) {
-        const resolvedPath = await resolvePath(js_url, value);
-        if (analyzedFiles.includes(resolvedPath)) {
-          continue;
-        }
-        pushToJsUrls(resolvedPath);
-        filesFound.push(resolvedPath);
-      }
-
-      analyzedFiles.push(js_url);
-    }
-  }
-
-  filesFound = [...new Set(filesFound)];
-
-  console.log(
-    chalk.green(`[✓] Found ${filesFound.length} JS files from string analysis`),
-  );
-
-  return filesFound;
-};
-
-export default nuxt_stringAnalysisJSFiles;

package/research/firewall_bypass.md

@@ -1,38 +0,0 @@
-# Firewall Bypass
-While the development of this tool, the tester came across several websites using firewall, majorly Cloudflare. A technique has to be developed to bypass this blockage, and to download the JS files.
-
-## Cloudflare
-### Detection
-Whenever the client is restricted by Cloudflare, it returns a HTML page with a JS challenge to be solved. However, the JS is can't be executed by Node.JS. It would need a browser environment. Sample HTML page is shown below:
-
-```html
-<meta http-equiv="refresh"
-        content="5; URL='/?bm-verify=--snip--'" />
---snip--
-<script> var i = 1749580415; var j = i + Number("7341" + "47171"); </script>
---snip--
-<script> var xhr = new XMLHttpRequest(); xhr.withCredentials = true; xhr.addEventListener("loadend", function () { try { var data = JSON.parse(xhr.responseText); if (data.hasOwnProperty('reload')) { if (data["reload"] == true) { window.location.replace(window.location.href.replace(/[&?]bm-verify=[^#]*/, "")); if (window.location.hash) { window.location.reload(); } } } else if (data.hasOwnProperty(--snip </script>
-```
-
-### Bypass
-#### #1
-The most common way to bypass this is to use a browser environment, such as a headless browser.
-
-To achive this task, a detection mechanism is added in the custom function to make HTTP requests. If CF is detected, it will load the page in a headless browser, and then return the content.
-
-#### #2
-Utilizing a browser plugin to fetch the contents is a more advanced way to bypass this. The user can install the plugin in their testing browser/proxy, and as they navigate the site, the plugin will get the contents and send then to the server for processing (it would be like a caching system).
-
-#### #3
-The `makeRequest()` can be modified to send headers that a browser would send. These include:
-- `User-Agent` of a browser
-- `Accept`
-- `Accept-Language`
-- `Sec-Fetch-Site: same-origin`
-- `Sec-Fetch-Mode: cors`
-- `Sec-Fetch-Dest: empty`
-- `Referer`
-- `Origin`
-
-#### #4
-Utilizing AWS API Gateway or similar services to rotate IP addresses can help in bypassing the firewall. This, however, could be blocked by the by CF if it is configured to block requests from AWS or similar ISPs.

package/research/next_js.md

@@ -1,116 +0,0 @@
-# Next.js Research
-## Tech Detection
-To detect Next.js, following technique(s) can be used: 
-- Iterate through all the HTML tags, and find `src`, `srcSet` or `imageSrcSet` attribute with value starting with`/_next/` in the page source
-
-## Embedded JS files
-A lot of JS files were identified from the page source by inspecting the value of `src` attribute of `script` tags. These included files like `main.js`, `polyfills.js`, `webpack.js` etc.
-
-## Lazy Loaded Files
-### Analysis of [Vercel Docs](https://vercel.com/docs)
-Upon analysis of page source and HTTP requests, it was found that the webpack filename had a pattern of `webpack-<hash>.js`. So, webpack JS files were identified and fetched.
-
-Upon inspection of the webpack JS file, it was found that code was distributed into several functions. It was observed that the function responsible for returning the JS path ended with `".js"`. This observation was seen multiple times in the Next.js apps by the developer (aka web researcher), hence it bacame a standard method to get the path of the JS files.
-
-### Analysis of [X.ai](https://x.ai)
-It was found that most of the things were done in a similar way, however, some extra chars were present after `".js"` in the function responsible for returning JS path in the webpack JS file.
-
-To handle this, the regex was modified to also include some additional characters at the end.
-```js
-.match(/"\.js".{0,15}$/)
-```
-
-Also, it was found that very less URLs were present in the `webpack.js` file (4 at the time of analysis). Upon further inspection, it was found that the URLs are present in the inline `<script>` tags returned in the page source. For example:
-```
-self.__next_f.push([1,"14:I[41498,[\"3867\",\"static/chunks/b6d67c9f-618ea7c61a79562d.js\",\"5017\",\"static/chunks/622eaf3d-350d2142e11f9e13.js\",\"1889\",\"static/chunks/0fd4459a-4e25f772815e6f19.js\",\"4406\",\"static/chunks/4406-fdbbb31c90e98725.js\",\"8627\",\"static/chunks/8627-54829c23d6b1a53f.js\",\"6520\",\"static/chunks/6520-f51ddcfa4e30ebc8.js\",\"1296\",\"static/chunks/1296-b8c2dd03773a40db.js\",\"8922\",\"static/chunks/8922-f01acc4fc5fecfad.js\",\"6929\",\"static/chunks/6929-a636a59ffdb51b17.js\",\"2601\"......snip....../chunks/b6d67c9f-"])
-```
-
-To handle this, a feature was implemented to get the JS files from the inline `<script>` tags as well.
-
-Upon researching further, it was found that additional JS files were being returned in the subsequent HTTP requests. For example, the file at `https://x.ai/_next/static/chunks/app/careers/page-9e04dce6fed05790.js` was loaded from a request to `https://x.ai/careers?_rsc=jwl50`.
-
-Request to `https://x.ai/careers?_rsc=jwl50` should be made with a special header called `RSC: 1` in order to get access to additional JS file paths. This request can be also sent to `/` with the same header to get more files.
-
-Possible solution: To handle this, the strings can be extracted from the page sources, and then requests to them can be made along with this request header. If it returns a `200 OK`, then it can be also added to the list of JS files. 
-
-### Analysis of [OpenAI](https://openai.com)
-Upon inspection of HTTP requests, a similar behavior of sending the requests with `RSC: 1` header to get additional JS files was found.
-
-However, a slight difference was found. When sending the request to `/business` with the same header, the server returned a different (`308 Permanent Redirect` to `/business/`) response. This can be handled by sending the request to `/business/` with the same header.
-
-Additionally, it was noticed that when the requests were sent without the `RSC: 1` header, the request got blocked by Cloudflare, and it resulted in a `403 Forbidden` response with a message `Just a moment...`. This could indicate a potential firewall bypass
-
-## Client-Side Paths/URLs
-Client-side paths/URLs are web addresses handled by the browser using JavaScript, usually without reloading the page. They are used for navigation, API requests, and loading resources dynamically within the client environment.
-
-### Analysis of [X.ai](https://x.ai)
-Upon inspection of the client side paths, it was found that they are present in `href` across JS chunks.
-
-These are a part of a list, which contains objects with keys like `href` (string), `label` (string), `active` (boolean) and `children` (array of objects of the same type).
-
-For instance, here's a example of such a list:
-```js
-let L = [
-    {
-      href: "/grok",
-      label: "Grok",
-      active: e.startsWith("/grok"),
-      children: [
-        { href: "/grok", label: "For Everyone", active: "/grok" == e },
-        {
-          href: "/grok/business",
-          label: "For Business",
-          active: "/grok/business" == e,
-        },
-      ],
-    },
-    {
-      href: "/api",
-      label: "API",
-      active: e.startsWith("/api"),
-      children: [
-        { href: "/api#capabilities", label: "Overview" },
-        {
-          href: "https://docs.x.ai/docs/models?cluster=us-east-1#detailed-pricing-for-all-grok-models",
-          label: "Pricing",
-          external: !0,
-        },
-        {
-          href: "https://console.x.ai",
-          label: "API Console Login",
-          external: !0,
-        },
-        {
-          href: "https://docs.x.ai",
-          label: "Documentation",
-          external: !0,
-        },
-      ],
-    },
-    { href: "/company", label: "Company", active: "/company" == e },
-    { href: "/colossus", label: "Colossus", active: "/colossus" == e },
-    {
-      href: "/careers",
-      label: "Careers",
-      active: e.startsWith("/careers"),
-    },
-    { href: "/news", label: "News", active: e.startsWith("/news") },
-]
-```
-
-Possible methodology: The tool can iterate over all the JS chunks, and find the list of objects with keys like `href` (string), `label` (string), `active` (boolean) and `children` (array of objects of the same type). Then, it can organize them in a report.
-
-### Analyis of [1Password](https://1password.com)
-It was found that the client-side paths were stored in mostly stored in a way like:
-```js
-let s = JSON.parse(
-          '["/state-of-enterprise-security-report/thank-you/",......"/webinars/1p-quarterly-security-spotlight-and-roadmap-review/thank-you/"]',
-)
-```
-
-Some similar pattern was also observed in [OpenAI](https://openai.com), however, the full analysis of OpenAI's client-side paths is not done at the time of writing this.
-
-It was also found that some paths were stored directly as a list. For example:
-```js
-let n = ["/pricing/xam", "/pricing/password-manager"];
-```