@shriyanss/js-recon 1.0.0

package/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Shriyans Sudhi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,166 @@
+# js-recon
+## Installation
+To install the tool, run:
+```bash
+npm i -g @shriyanss/js-recon
+```
+
+## Usage
+```
+$ js-recon -h
+Usage: js-recon [options] [command]
+
+JS Recon Tool
+
+Options:
+  -V, --version          output the version number
+  -h, --help             display help for command
+
+Commands:
+  lazyload [options]     Run lazy load module
+  endpoints [options]    Extract API endpoints
+  strings [options]      Extract strings from JS files
+  api-gateway [options]  Configure AWS API Gateway to rotate IP addresses
+  help [command]         display help for command
+```
+
+### Lazy load
+```
+$ js-recon lazyload -h
+Usage: js-recon lazyload [options]
+
+Run lazy load module
+
+Options:
+  -u, --url <url/file>         Target URL or a file containing a list of URLs (one per line)
+  -o, --output <directory>     Output directory (default: "output")
+  --strict-scope               Download JS files from only the input URL domain (default: false)
+  -s, --scope <scope>          Download JS files from specific domains (comma-separated) (default: "*")
+  -t, --threads <threads>      Number of threads to use (default: 1)
+  --subsequent-requests        Download JS files from subsequent requests (default: false)
+  --urls-file <file>           Input JSON file containing URLs (default: "extracted_urls.json")
+  --api-gateway                Generate requests using API Gateway (default: false)
+  --api-gateway-config <file>  API Gateway config file (default: ".api_gateway_config.json")
+  -h, --help                   display help for command
+```
+
+### Strings
+```
+$ js-recon strings -h
+Usage: js-recon strings [options]
+
+Extract strings from JS files
+
+Options:
+  -d, --directory <directory>  Directory containing JS files
+  -o, --output <file>          JSON file to save the strings (default: "strings.json")
+  -e, --extract-urls           Extract URLs from strings (default: false)
+  --extracted-url-path <file>  Output JSON file for extracted URLs and paths (default: "extracted_urls.json")
+  -h, --help                   display help for command
+```
+
+### API Gateway
+```
+$ js-recon api-gateway -h
+Usage: js-recon api-gateway [options]
+
+Configure AWS API Gateway to rotate IP addresses
+
+Options:
+  -i, --init                     Initialize the config file (create API) (default: false)
+  -d, --destroy <id>             Destroy API with the given ID
+  --destroy-all                  Destroy all the API created by this tool in all regions (default: false)
+  -r, --region <region>          AWS region (default: random region)
+  -a, --access-key <access-key>  AWS access key (if not provided, AWS_ACCESS_KEY_ID environment variable will be used)
+  -s, --secret-key <secret-key>  AWS secret key (if not provided, AWS_SECRET_ACCESS_KEY environment variable will be used)
+  -c, --config <config>          Name of the config file (default: ".api_gateway_config.json")
+  -l, --list                     List all the API created by this tool (default: false)
+  --feasibility                  Check feasibility of API Gateway (default: false)
+  --feasibility-url <url>        URL to check feasibility of
+  -h, --help                     display help for command
+```
+
+## Features
+### Download lazy loaded JS files
+You can download the lazy loaded files. To do so, you can run the following command
+```bash
+js-recon lazyload -u <url> -o <output>
+```
+
+For example, you can try this with [Vercel Docs](https://vercel.com/docs):
+```bash
+js-recon lazyload -u https://vercel.com/docs
+```
+
+Currently, the following JS frameworks are supported:
+- Next.js (read research [here](research/next_js.md))
+
+### Extract strings from JS files
+You can extract strings from JS files. To do so, you can run the following command
+```bash
+js-recon strings -d <directory> -o <output>
+```
+
+For example, you can try this with [1Password](https://1password.com):
+```bash
+js-recon strings -d output/1password.com -o strings.json
+```
+
+## Examples
+### Get all possible JS files for a Next.js app
+*You can read the full research for the same [here](research/next_js.md#lazy-loaded-files)*
+
+First of all, run the lazy load module (strict scope and 1 thread for accurate results) [research1](research/next_js.md#analysis-of-vercel-docs):
+```bash
+js-recon lazyload -u <url> -o <output> --strict-scope -t 1
+```
+
+Then, get all the strings from the JS files found. Also, extract URLs and paths found in those JS files.:
+```bash
+js-recon strings -d <directory> -o <output> -e
+```
+
+Finally, parse those URLs and paths to get more JS files (note the `--subsequent-requests` flag apart from `--strict-scope` and `--threads`) [research](research/next_js.md#analysis-of-xai):
+```bash
+js-recon endpoints -u <url> -o <output> --strict-scope -t 1 --subsequent-requests
+```
+
+### Use AWS API Gateway to rotate IP address on each request
+First of all, the user has to configure the API keys for AWS with right permissions to use the API Gateway module of the tool. The access key and the secret key can be stored in the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` respectively. Alternatively, these can be provided as command line argument to the tool.
+
+Once this is done, the configuration file can be generated. The configuration file is a JSON file that contains the API Gateway information, including the API Gateway ID, region, access key, and secret key. This file is used by the tool in the runtime. The configuration file can be generated by running the following command:
+```bash
+js-recon api-gateway -i -r <region> -a <access-key> -s <secret-key>
+```
+If the region is not provided, the tool will select a random region from a pre-defined list. If the `-a` and `-s` flags aren't provided, it will default to the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` respectively.
+
+This command will generate a new API gateway in the specified region, which can be inspected on the AWS console. The API gateway ID will be stored in the configuration file. The configuration file can be found at `./.api_gateway_config.json` by default.
+
+The user can list the API gateways generated by running the following command:
+```bash
+js-recon api-gateway -l
+```
+
+The user can generate as many API gateways as they want. The tool will automatically select a random API gateway from the configuration file to make requests to. Creating multiple gateways in different region can help in changing the region of the IP address on each request, however, **creating multiple gateways in the same region is not recommended** as AG will rotate the IP address on each request.
+
+It is recommended to check if the firewall is blocking the requests from the API gateway. The user can do so by running the following command:
+```bash
+js-recon api-gateway --feasibility --feasibility-url <url>
+```
+
+Next, the API gateway can be utilized for rotating the IP address on each request. To do so, the user can add the `--api-gateway` flag to the lazy load module. The user can also provide the API gateway config file using the `--api-gateway-config` flag if they have changed the defaults.
+
+For example,
+```bash
+js-recon lazyload -u <url> -o <output> --api-gateway
+```
+
+Now that the user has completed their task, they can delete the API gateway using the following command:
+```bash
+js-recon api-gateway -d <api-gateway-id>
+```
+
+Alternatively, they can delete all the API gateways using the following command:
+```bash
+js-recon api-gateway --destroy-all
+```

package/api_gateway/checkFeasibility.js

@@ -0,0 +1,25 @@
+import { get } from "./genReq.js";
+import chalk from "chalk";
+import checkFireWallBlocking from "./checkFireWallBlocking.js";
+
+const checkFeasibility = async (url) => {
+  console.log(chalk.cyan(`[i] Checking feasibility of API Gateway with ${url}`));
+  try {
+    // send 10 requests, and check if any of those contain any signs of blocking
+    for (let i = 0; i < 10; i++) {
+      const response = await get(url);
+      const isFireWallBlocking = await checkFireWallBlocking(response);
+      if (isFireWallBlocking) {
+        console.log(chalk.magenta("[!] Please try again without API Gateway"));
+        return;
+      }
+    }
+    console.log(chalk.green("[✓] Feasibility check passed."), chalk.dim("However, this doesn't represent the true nature of the firewall used."));
+  } catch (error) {
+    console.log(
+      chalk.red(`[!] An error occured in feasibility check: ${error}`),
+    );
+  }
+};
+
+export default checkFeasibility;

package/api_gateway/checkFireWallBlocking.js

@@ -0,0 +1,17 @@
+import chalk from "chalk";
+
+const checkFireWallBlocking = async (body) => {
+
+    // check common signs of CF first
+    if (body.includes("<title>Just a moment...</title>")) {
+        console.log(chalk.red("[!] Cloudflare detected"));
+        return true;
+    } else if (body.includes("<title>Attention Required! | Cloudflare</title>")) {
+        console.log(chalk.red("[!] Cloudflare detected"));
+        return true;
+    }
+    
+    return false;
+};
+
+export default checkFireWallBlocking;

package/api_gateway/genReq.js

@@ -0,0 +1,214 @@
+import {
+  APIGatewayClient,
+  CreateResourceCommand,
+  GetResourcesCommand,
+  PutMethodCommand,
+  PutIntegrationCommand,
+  //   CreateDeploymentCommand,
+  //   CreateStageCommand,
+  PutIntegrationResponseCommand,
+  PutMethodResponseCommand,
+  TestInvokeMethodCommand,
+  DeleteResourceCommand,
+} from "@aws-sdk/client-api-gateway";
+import fs from "fs";
+import md5 from "md5";
+import chalk from "chalk";
+import * as globals from "../utility/globals.js";
+import checkFireWallBlocking from "./checkFireWallBlocking.js";
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Given a URL, generates a new API Gateway for it and returns the response of the URL.
+ * @param {string} url The URL to generate an API Gateway for.
+ * @param {object} [headers] The headers to include in the request.
+ * @returns {Promise<string>} The response of the URL.
+ */
+const get = async (url, headers) => {
+  // read the config file
+  let config = JSON.parse(fs.readFileSync(globals.apiGatewayConfigFile));
+  // select a random api gateway
+  let apiGateway =
+    Object.keys(config)[Math.floor(Math.random() * Object.keys(config).length)];
+
+  const client = new APIGatewayClient({
+    region: config[apiGateway].region,
+    credentials: {
+      accessKeyId: config[apiGateway].access_key,
+      secretAccessKey: config[apiGateway].secret_key,
+    },
+  });
+
+  // get the root resource id
+  const getResourceCommand = new GetResourcesCommand({
+    restApiId: config[apiGateway].id,
+    limit: 999999999,
+  });
+  const getResourceResponse = await client.send(getResourceCommand);
+  await sleep(200);
+
+  // before creating a resource, check if the resource already exists
+  const resourceExists = getResourceResponse.items.find(
+    (item) => item.pathPart === md5(url),
+  );
+
+  let newResourceResponse;
+  if (resourceExists) {
+    // console.log(chalk.yellow("[!] Resource already exists"));
+    newResourceResponse = {
+      id: resourceExists.id,
+    };
+  } else {
+    // create a new resource
+    let rootId;
+    if (getResourceResponse.items.find((item) => item.path === "/")) {
+      rootId = getResourceResponse.items.find((item) => item.path === "/").id;
+    } else {
+      rootId = getResourceResponse.items[0].parentId;
+    }
+    const newResourceCommand = new CreateResourceCommand({
+      restApiId: config[apiGateway].id,
+      parentId: rootId,
+      pathPart: md5(url), // md5 of the url
+    });
+    newResourceResponse = await client.send(newResourceCommand);
+    await sleep(200);
+
+    // add a new method
+    const newMethodCommand = new PutMethodCommand({
+      restApiId: config[apiGateway].id,
+      resourceId: newResourceResponse.id,
+      httpMethod: "GET",
+      authorizationType: "NONE",
+      requestParameters: {
+        "method.request.header.RSC": false,
+        "method.request.header.User-Agent": false,
+        "method.request.header.Referer": false,
+        "method.request.header.Accept": false,
+        "method.request.header.Accept-Language": false,
+        "method.request.header.Accept-Encoding": false,
+        "method.request.header.Content-Type": false,
+        "method.request.header.Content-Length": false,
+        "method.request.header.Origin": false,
+        "method.request.header.X-Forwarded-For": false,
+        "method.request.header.X-Forwarded-Host": false,
+        "method.request.header.X-IP": false,
+        "method.request.header.X-Forwarded-Proto": false,
+        "method.request.header.X-Forwarded-Port": false,
+        "method.request.header.Sec-Fetch-Site": false,
+        "method.request.header.Sec-Fetch-Mode": false,
+        "method.request.header.Sec-Fetch-Dest": false,
+      },
+      integrationHttpMethod: "GET",
+      type: "HTTP",
+      timeoutInMillis: 29000,
+    });
+    const newMethodResponse = await client.send(newMethodCommand);
+    await sleep(100);
+
+    // create new integration
+    const newIntegrationCommand = new PutIntegrationCommand({
+      restApiId: config[apiGateway].id,
+      resourceId: newResourceResponse.id,
+      httpMethod: "GET",
+      integrationHttpMethod: "GET",
+      type: "HTTP",
+      timeoutInMillis: 29000,
+      uri: url,
+    });
+    const newIntegrationResponse = await client.send(newIntegrationCommand);
+    await sleep(100);
+
+    // create a new method response
+    const newMethodResponseCommand = new PutMethodResponseCommand({
+      httpMethod: "GET",
+      resourceId: newResourceResponse.id,
+      restApiId: config[apiGateway].id,
+      statusCode: "200",
+    });
+    const newMethodResponseResponse = await client.send(
+      newMethodResponseCommand,
+    );
+    await sleep(100);
+
+    // put integration response
+    const putIntegrationResponseCommand = new PutIntegrationResponseCommand({
+      httpMethod: "GET",
+      resourceId: newResourceResponse.id,
+      restApiId: config[apiGateway].id,
+      statusCode: "200",
+    });
+    const putIntegrationResponseResponse = await client.send(
+      putIntegrationResponseCommand,
+    );
+    await sleep(100);
+  }
+
+  // Generate dynamic stage name
+  //   const dynamicStageName = `prod-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
+  //   console.log(chalk.blue(`[*] Using dynamic stage name: ${dynamicStageName}`));
+
+  // Create a deployment
+  //   const createDeploymentCommand = new CreateDeploymentCommand({
+  //     restApiId: config[apiGateway].id,
+  //     stageName: dynamicStageName,
+  //     description: `Deployment for ${url} at ${new Date().toISOString()} to stage ${dynamicStageName}`,
+  //   });
+  //   const deploymentResponse = await client.send(createDeploymentCommand);
+  //   console.log(chalk.green("[+] Deployment created:"), deploymentResponse.id);
+  //   await sleep(100);
+
+  const testInvokeMethodQuery = new TestInvokeMethodCommand({
+    httpMethod: "GET",
+    resourceId: newResourceResponse.id,
+    restApiId: config[apiGateway].id,
+    headers: headers || {},
+  });
+  const testInvokeMethodResponse = await client.send(testInvokeMethodQuery);
+  await sleep(100);
+
+  const body = await testInvokeMethodResponse.body;
+
+  // check if any firewall is there in the way
+  const isFireWallBlocking = await checkFireWallBlocking(body);
+
+  // delete the resource
+  const deleteResourceCommand = new DeleteResourceCommand({
+    restApiId: config[apiGateway].id,
+    resourceId: newResourceResponse.id,
+  });
+  try {
+    await client.send(deleteResourceCommand);
+  } catch {}
+
+  if (isFireWallBlocking) {
+    console.log(chalk.magenta("[!] Please try again without API Gateway"));
+    process.exit(1);
+  }
+
+  return body;
+
+  // create a new stage
+  //   dynamicStageName = `prod-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`;
+  //   const newStageCommand = new CreateStageCommand({
+  //     restApiId: config[apiGateway].id,
+  //     stageName: dynamicStageName, // Use dynamic stage name
+  //     deploymentId: deploymentResponse.id, // Use the ID from the deployment
+  //     cacheClusterEnabled: false,
+  //     // cacheClusterSize: "0", // Removed as cacheClusterEnabled is false
+  //     methodSettings: [
+  //       {
+  //         httpMethod: "*",
+  //         // resourceId: "*", // This might need to be more specific if you don't want it for all resources
+  //         throttlingBurstLimit: 5,
+  //         throttlingRateLimit: 10,
+  //       },
+  //     ],
+  //   });
+  //   const newStageResponse = await client.send(newStageCommand);
+  //   await sleep(100);
+  //   console.log(newStageResponse);
+};
+
+export { get };