@js-recon/js-recon 1.3.1-beta.2

package/build/utility/resolvePath.js

@@ -0,0 +1,44 @@
+/**
+ * Resolves a relative path with a given base URL.
+ *
+ * The behavior of the URL constructor is used to handle various cases of path resolution.
+ * If the base URL does not end with a '/', its last path segment is typically treated as a "file",
+ * and relative paths are resolved from the "directory" containing that "file".
+ * Examples:
+ * - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
+ *   (Base for resolution becomes 'https://site.com/')
+ * - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
+ *   (Base for resolution is 'https://site.com/something/')
+ * - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
+ *   (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
+ *
+ * @param {string} url - The base URL for resolving the relative path
+ * @param {string} path - The relative path to be resolved
+ * @returns {string} The resolved absolute URL
+ * @throws {Error} If an error occurs while resolving the path
+ */
+const resolvePath = (url, path) => {
+    try {
+        // The URL constructor handles various cases of path resolution.
+        // If 'url' (the base URL) does not end with a '/', its last path segment
+        // is typically treated as a "file", and relative paths are resolved
+        // from the "directory" containing that "file".
+        // This behavior aligns with the provided examples:
+        // - url='https://site.com/something', path='./main.js' => 'https://site.com/main.js'
+        //   (Base for resolution becomes 'https://site.com/')
+        // - url='https://site.com/something/', path='./main.js' => 'https://site.com/something/main.js'
+        //   (Base for resolution is 'https://site.com/something/')
+        // - url='https://site.com/something/other', path='../main.js' => 'https://site.com/main.js'
+        //   (Base for resolution becomes 'https://site.com/something/', then '../' navigates up)
+        const resolvedUrl = new URL(path, url);
+        return resolvedUrl.href;
+    }
+    catch (e) {
+        console.error(`Error resolving path "${path}" with base URL "${url}": ${e.message}`);
+        // Rethrowing the error to signal failure to the caller.
+        // Alternative error handling (e.g., returning null) can be implemented if required.
+        throw e;
+    }
+};
+export default resolvePath;
+//# sourceMappingURL=resolvePath.js.map

package/build/utility/resolvePath.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolvePath.js","sourceRoot":"","sources":["../../src/utility/resolvePath.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC9C,IAAI,CAAC;QACD,gEAAgE;QAChE,yEAAyE;QACzE,oEAAoE;QACpE,+CAA+C;QAC/C,mDAAmD;QACnD,qFAAqF;QACrF,sDAAsD;QACtD,gGAAgG;QAChG,2DAA2D;QAC3D,4FAA4F;QAC5F,yFAAyF;QACzF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvC,OAAO,WAAW,CAAC,IAAI,CAAC;IAC5B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,yBAAyB,IAAI,oBAAoB,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,wDAAwD;QACxD,oFAAoF;QACpF,MAAM,CAAC,CAAC;IACZ,CAAC;AACL,CAAC,CAAC;AAEF,eAAe,WAAW,CAAC"}

package/build/utility/runSandboxed.js

@@ -0,0 +1,42 @@
+import "ses";
+// Lock down the global environment for security
+lockdown();
+/**
+ * Executes a given function in a sandboxed environment.
+ * @param {string} code - The code to execute.
+ * @param {any} param - The parameter to pass to the function.
+ * @returns {any} The result of the function execution.
+ */
+/**
+ * Executes a given function in a sandboxed environment.
+ *
+ * This function takes a string of code that represents a function, and
+ * executes it in a sandboxed environment. It also takes a parameter
+ * that is passed to the function when it is executed.
+ *
+ * The sandboxed environment is created using the vm module's
+ * Compartment class. This class is used to create a new global
+ * environment that is isolated from the current global environment.
+ *
+ * The function that is passed as a string is wrapped in a self-invoking
+ * anonymous function, and then evaluated in the sandboxed environment.
+ * The parameter that is passed to the function is then passed to the evaluated
+ * function, and the result of the function execution is returned.
+ *
+ * @param {string} code - The code to execute. This should represent a function.
+ * @param {any} param - The parameter to pass to the function.
+ * @returns {any} The result of the function execution.
+ */
+const execFunc = (code, param) => {
+    const c = new Compartment({
+        console,
+    });
+    const wrappedCode = `
+    (${code})
+  `;
+    const func = c.evaluate(wrappedCode);
+    const output = func(param);
+    return output;
+};
+export default execFunc;
+//# sourceMappingURL=runSandboxed.js.map

package/build/utility/runSandboxed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runSandboxed.js","sourceRoot":"","sources":["../../src/utility/runSandboxed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC;AAEb,gDAAgD;AAChD,QAAQ,EAAE,CAAC;AAEX;;;;;GAKG;AACH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;IAC7B,MAAM,CAAC,GAAG,IAAI,WAAW,CAAC;QACtB,OAAO;KACV,CAAC,CAAC;IACH,MAAM,WAAW,GAAG;OACjB,IAAI;GACR,CAAC;IACA,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAE3B,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,eAAe,QAAQ,CAAC"}

package/build/utility/urlUtils.js

@@ -0,0 +1,21 @@
+/**
+ * Given a URL, returns an object with the host and directory
+ * of the URL. The directory is the path of the URL after the
+ * host, and the filename is removed if it ends with a file extension.
+ * For example, given "https://vercel.com/static/js/main.js", it will return
+ * an object with host "vercel.com" and directory "/static/js".
+ * @param {string} url - The URL to parse.
+ * @returns {Object} An object with the host and directory of the URL.
+ */
+const getURLDirectory = (url) => {
+    const u = new URL(url);
+    const pathname = u.pathname;
+    // Remove filename (last part after final /) if it ends with .js or any file extension
+    const dir = pathname.replace(/\/[^\/?#]+\.[^\/?#]+$/, "");
+    return {
+        host: u.host.replace(":", "_"), // e.g., "vercel.com" or "localhost_3000"
+        directory: dir, // e.g., "/static/js"
+    };
+};
+export { getURLDirectory };
+//# sourceMappingURL=urlUtils.js.map

package/build/utility/urlUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"urlUtils.js","sourceRoot":"","sources":["../../src/utility/urlUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IAE5B,sFAAsF;IACtF,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAE1D,OAAO;QACH,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,yCAAyC;QACzE,SAAS,EAAE,GAAG,EAAE,qBAAqB;KACxC,CAAC;AACN,CAAC,CAAC;AAEF,OAAO,EAAE,eAAe,EAAE,CAAC"}

package/contributing/nextjs.md

@@ -0,0 +1,410 @@
+# Contributing New Next.js Discovery Methods
+
+This guide explains how to add new JavaScript file discovery techniques for Next.js applications to `js-recon`.
+
+## Table of Contents
+
+- [Architecture Overview](#architecture-overview)
+- [Discovery Method Types](#discovery-method-types)
+- [Adding a New Discovery Method](#adding-a-new-discovery-method)
+- [Integration into NextJsCrawler](#integration-into-nextjscrawler)
+- [Best Practices](#best-practices)
+- [Testing Your Method](#testing-your-method)
+- [Research Mode](#research-mode)
+
+---
+
+## Architecture Overview
+
+The Next.js discovery system is built around the **`NextJsCrawler`** class (`src/lazyLoad/next_js/NextJsCrawler.ts`), which implements a **three-phase crawling strategy**:
+
+### Phase 1: Initial Discovery
+
+Heavyweight methods that run **once** to bootstrap the crawl:
+
+- Script tag parsing (`next_getJSScript`)
+- Webpack runtime analysis (`next_GetLazyResourcesWebpackJs`) – uses Puppeteer
+- `_buildManifest.js` parsing (`next_getLazyResourcesBuildManifestJs`)
+- Subsequent requests with RSC headers (`subsequentRequests`)
+
+### Phase 2: Recursive Discovery
+
+Lightweight methods that run **multiple times** on newly discovered URLs until convergence:
+
+- Promise.all pattern detection (`next_promiseResolve`)
+- Layout.js href extraction (`next_parseLayoutJs`)
+- Script tag re-parsing on new pages (`next_getJSScript`)
+
+### Phase 3: Finalization
+
+Post-processing on the final URL set:
+
+- Source map brute-forcing (`next_bruteForceJsFiles`)
+
+---
+
+## Discovery Method Types
+
+### Initial Discovery Methods
+
+**When to use:**
+
+- The method is **expensive** (e.g., launches a browser, makes many requests)
+- The method doesn't benefit from being run multiple times
+- The method provides a bootstrap set of URLs
+
+**Characteristics:**
+
+- Runs exactly **once** per crawl
+- Added to `initialDiscovery()` in `NextJsCrawler.ts`
+- Examples: Puppeteer-based webpack analysis, manifest parsing
+
+### Recursive Discovery Methods
+
+**When to use:**
+
+- The method analyzes **JavaScript file contents** to find more JS files
+- The method discovers **client-side paths** that need to be visited
+- The method can find new URLs by examining URLs found in previous passes
+
+**Characteristics:**
+
+- Runs in a **loop** until no new URLs are discovered (max 10 iterations by default)
+- Takes an array of URLs as input and returns newly discovered URLs
+- Added to `recursivePass()` in `NextJsCrawler.ts`
+- Examples: Promise.all pattern detection, layout href parsing
+
+---
+
+## Adding a New Discovery Method
+
+### Step 1: Create the Method Module
+
+Create a new file in `src/lazyLoad/next_js/` following the naming convention `next_<methodName>.ts`.
+
+#### Example: Detecting dynamic imports
+
+```typescript
+// src/lazyLoad/next_js/next_dynamicImports.ts
+import chalk from "chalk";
+import makeRequest from "../../utility/makeReq.js";
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+const traverse = _traverse.default;
+
+/**
+ * Finds JS files referenced in dynamic import() statements.
+ *
+ * @param baseUrl - The base URL of the application
+ * @param urls - Array of JS file URLs to analyze
+ * @returns Array of newly discovered JS file URLs
+ */
+const next_dynamicImports = async (baseUrl: string, urls: string[]): Promise<string[]> => {
+    console.log(chalk.cyan("[i] Analyzing dynamic import() statements"));
+
+    const discoveredUrls: string[] = [];
+
+    for (const url of urls) {
+        try {
+            const response = await makeRequest(url);
+            if (!response?.ok) continue;
+
+            const jsContent = await response.text();
+            const ast = parser.parse(jsContent, {
+                sourceType: "unambiguous",
+                plugins: ["jsx", "typescript"],
+                errorRecovery: true,
+            });
+
+            traverse(ast, {
+                Import(path) {
+                    // Detect: import('path/to/file.js')
+                    const parent = path.parent;
+                    if (
+                        parent.type === "CallExpression" &&
+                        parent.arguments.length > 0 &&
+                        parent.arguments[0].type === "StringLiteral"
+                    ) {
+                        const importPath = parent.arguments[0].value;
+                        if (importPath.endsWith(".js")) {
+                            // Resolve relative to base URL
+                            const resolvedUrl = new URL(importPath, baseUrl).href;
+                            discoveredUrls.push(resolvedUrl);
+                        }
+                    }
+                },
+            });
+        } catch (error) {
+            // Skip files that can't be parsed
+            continue;
+        }
+    }
+
+    const uniqueUrls = [...new Set(discoveredUrls)];
+
+    if (uniqueUrls.length > 0) {
+        console.log(chalk.green(`[✓] Found ${uniqueUrls.length} JS files from dynamic imports`));
+    }
+
+    return uniqueUrls;
+};
+
+export default next_dynamicImports;
+```
+
+### Step 2: Method Signature Guidelines
+
+**For initial discovery methods:**
+
+```typescript
+async function next_methodName(url: string): Promise<string[]>;
+```
+
+- Takes the base URL as input
+- Returns array of discovered URLs
+
+**For recursive discovery methods:**
+
+```typescript
+async function next_methodName(baseUrl: string, urls: string[]): Promise<string[]>;
+```
+
+- Takes base URL and array of URLs to analyze
+- Returns array of **newly** discovered URLs
+- Should handle empty input gracefully
+
+### Step 3: Common Patterns
+
+#### Pattern 1: AST-based Analysis
+
+Use Babel parser to analyze JavaScript syntax:
+
+```typescript
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+const traverse = _traverse.default;
+
+const ast = parser.parse(jsContent, {
+    sourceType: "unambiguous",
+    plugins: ["jsx", "typescript"],
+    errorRecovery: true,
+});
+
+traverse(ast, {
+    // Visitor pattern
+    CallExpression(path) {
+        // Analyze specific AST nodes
+    },
+});
+```
+
+#### Pattern 2: String/Regex Matching
+
+Quick pattern detection without parsing:
+
+```typescript
+const matches = jsContent.matchAll(/static\/chunks\/[a-zA-Z0-9_\-]+\.js/g);
+for (const match of matches) {
+    discoveredUrls.push(resolveUrl(match[0]));
+}
+```
+
+#### Pattern 3: Request-Based Discovery
+
+Make HTTP requests to known patterns:
+
+```typescript
+const candidateUrl = `${baseUrl}/_next/static/${hash}/chunk.js`;
+const response = await makeRequest(candidateUrl);
+if (response.status === 200) {
+    discoveredUrls.push(candidateUrl);
+}
+```
+
+---
+
+## Integration into NextJsCrawler
+
+### For Initial Discovery Methods
+
+Add to `initialDiscovery()` in `NextJsCrawler.ts`. Use this phase for methods that only need to run once, like those starting from the application's root URL:
+
+```typescript
+private async initialDiscovery(): Promise<void> {
+    // ... existing methods ...
+
+    // 5. Your new initial discovery method
+    const jsFromPrefetch = await next_prefetchHints(this.url);
+    this.techniqueEfficiencyMapping["next_prefetchHints"] = jsFromPrefetch;
+    this.registerUrls(jsFromPrefetch);
+
+    // ... rest of the method ...
+}
+```
+
+**Don't forget to add the import at the top:**
+
+```typescript
+import next_prefetchHints from "./next_prefetchHints.js";
+```
+
+### For Recursive Discovery Methods
+
+Add to `recursivePass()` in `NextJsCrawler.ts`. Use this phase for methods that analyze JS file contents to find more JS files:
+
+```typescript
+private async recursivePass(jsUrls: string[]): Promise<string[]> {
+    let newInThisPass: string[] = [];
+
+    // ... existing methods ...
+
+    // Your new recursive method
+    const jsFromDynamicImports = await next_dynamicImports(this.url, jsUrls);
+    this.techniqueEfficiencyMapping["next_dynamicImports"] = [
+        ...(this.techniqueEfficiencyMapping["next_dynamicImports"] || []),
+        ...jsFromDynamicImports,
+    ];
+    newInThisPass.push(...this.registerUrls(jsFromDynamicImports));
+
+    return newInThisPass;
+}
+```
+
+**Key points:**
+
+- Append to `techniqueEfficiencyMapping` (not replace) for recursive methods
+- Use `registerUrls()` to deduplicate and track new URLs
+- Only add truly new URLs to `newInThisPass`
+
+---
+
+## Best Practices
+
+### 1. Error Handling
+
+Always handle errors gracefully – don't crash the entire crawl:
+
+```typescript
+try {
+    const response = await makeRequest(url);
+    if (!response?.ok) return [];
+    // ... process response ...
+} catch (error) {
+    // Log if needed, but continue
+    return [];
+}
+```
+
+### 2. URL Resolution
+
+Use `URL` constructor for proper relative URL resolution:
+
+```typescript
+const absoluteUrl = new URL(relativePath, baseUrl).href;
+```
+
+### 3. Deduplication
+
+Always deduplicate before returning:
+
+```typescript
+return [...new Set(discoveredUrls)];
+```
+
+### 4. Performance
+
+- Cache expensive operations (e.g., don't re-fetch the same URL)
+- Use `lazyLoadGlobals.presentInCrawledUrls()` to check if already visited
+- Mark URLs as crawled with `lazyLoadGlobals.addCrawledUrl()`
+
+```typescript
+import { presentInCrawledUrls, addCrawledUrl } from "../globals.js";
+
+for (const url of urls) {
+    if (presentInCrawledUrls(url)) continue;
+
+    // ... analyze URL ...
+
+    addCrawledUrl(url);
+}
+```
+
+### 5. Logging
+
+Use `chalk` for consistent logging:
+
+```typescript
+import chalk from "chalk";
+
+console.log(chalk.cyan("[i] Starting analysis...")); // Info
+console.log(chalk.green("[✓] Found 10 files")); // Success
+console.log(chalk.yellow("[!] Warning message")); // Warning
+console.log(chalk.red("[!] Error occurred")); // Error
+```
+
+---
+
+## Testing Your Method
+
+### 1. Integration Test with Real Sites
+
+Test against known Next.js sites:
+
+```bash
+npm run cleanup && npm run start -- run -u https://nextjs.org -y --research
+```
+
+Check `research.json` to see how many URLs your method discovered:
+
+```json
+{
+    "next_dynamicImports": [
+        "https://nextjs.org/_next/static/chunks/123.js",
+        "https://nextjs.org/_next/static/chunks/456.js"
+    ]
+}
+```
+
+### 2. Verify Convergence
+
+Ensure your recursive method doesn't cause infinite loops:
+
+- Check that "Recursive crawl converged" message appears
+- If it hits max iterations (defaults to 10), investigate why
+
+### 3. Performance Validation
+
+Monitor execution time:
+
+```bash
+time npm run start -- run -u https://example.com -y
+```
+
+---
+
+## Research Mode
+
+Use research mode to validate and compare discovery methods:
+
+```bash
+npm run start -- run -u https://example.com -y --research
+```
+
+This generates `research.json` with per-method efficiency:
+
+```json
+{
+    "next_getJSScript": [
+        "https://example.com/_next/static/chunks/main.js",
+        "https://example.com/_next/static/chunks/webpack.js"
+    ],
+    "next_dynamicImports": ["https://example.com/_next/static/chunks/lazy-component.js"],
+    "next_promiseResolve": ["https://example.com/_next/static/chunks/pages/about.js"]
+}
+```
+
+**Analysis tips:**
+
+- **Unique discoveries**: Methods that find URLs no other method finds
+- **Overlap**: Methods that discover the same URLs (candidates for removal)
+- **Efficiency**: URLs found per method execution

package/package.json

@@ -0,0 +1,75 @@
+{
+    "name": "@js-recon/js-recon",
+    "version": "1.3.1-beta.2",
+    "description": "JS Recon Tool",
+    "main": "build/index.js",
+    "type": "module",
+    "bin": {
+        "js-recon": "build/index.js"
+    },
+    "types": "build/index.d.ts",
+    "scripts": {
+        "build": "rm -rf build/ && tsc",
+        "start": "node --max-old-space-size=8192 build/index.js",
+        "test": "node build/index.js -h",
+        "cleanup": "rm -rf build output .resp_cache.json endpoints.json extracted_urls{.txt,.json,-openapi.json} strings.json mapped{-openapi.json,.json,-openapi.postman_collection.json} analyze.json test{.yaml,.js} shriyanss-js-recon-*.tgz js-recon.db report.{html,md} js_recon_run_output *_test.js extracted/ research.json results* out.* && tsc"
+    },
+    "keywords": [],
+    "author": "Shriyans Sudhi",
+    "license": "MIT",
+    "dependencies": {
+        "@anthropic-ai/sdk": "^0.102.0",
+        "@aws-sdk/client-api-gateway": "^3.958.0",
+        "@babel/parser": "^7.28.5",
+        "@babel/traverse": "^7.28.5",
+        "@babel/types": "^7.27.6",
+        "@modelcontextprotocol/sdk": "^1.29.0",
+        "@types/chalk": "^0.4.31",
+        "@types/cli-progress": "^3.11.6",
+        "better-sqlite3": "^12.5.0",
+        "blessed": "^0.1.81",
+        "chalk": "^5.6.2",
+        "cheerio": "^1.0.0",
+        "cli-highlight": "^2.1.11",
+        "cli-progress": "^3.12.0",
+        "commander": "^14.0.0",
+        "datatables.net": "^2.3.6",
+        "datatables.net-dt": "^2.3.6",
+        "esquery": "^1.6.0",
+        "fs": "^0.0.2",
+        "graphql": "^16.14.1",
+        "highlight.js": "^11.11.1",
+        "inquirer": "^12.6.3",
+        "marked": "^16.1.2",
+        "md5": "^2.3.0",
+        "ollama": "^0.6.3",
+        "openai": "^5.8.2",
+        "path": "^0.12.7",
+        "prettier": "^3.7.4",
+        "puppeteer": "^24.43.1",
+        "puppeteer-extra": "^3.3.6",
+        "puppeteer-extra-plugin-stealth": "^2.11.2",
+        "readline": "^1.3.0",
+        "ses": "^1.13.0",
+        "yaml": "^2.8.0",
+        "zod": "^3.23.8"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/shriyanss/js-recon.git"
+    },
+    "bugs": {
+        "url": "https://github.com/shriyanss/js-recon/issues"
+    },
+    "homepage": "https://github.com/shriyanss/js-recon#readme",
+    "devDependencies": {
+        "@types/babel__traverse": "^7.28.0",
+        "@types/better-sqlite3": "^7.6.13",
+        "@types/blessed": "^0.1.25",
+        "@types/graphql": "^14.2.3",
+        "@types/node": "^24.12.4",
+        "ts-node": "^10.9.2",
+        "tsconfig-paths": "^4.2.0",
+        "typescript": "^5.9.2"
+    }
+}