@js-recon/js-recon 1.4.4 → 2.0.1-alpha.1

package/build/lazyLoad/generic/generic_webpackChunkPaths.js

@@ -0,0 +1,56 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from "fs";
+import path from "path";
+import { extractWebpackChunkUrls } from "../shared/webpackChunkParsers.js";
+import { extractFileSourceUrl } from "./generic_stringsDiscovery.js";
+import { confirmJsContentType } from "./generic_scanAttributesForJs.js";
+/**
+ * Scans every JS file `generic` tech has already downloaded under outputDir for the
+ * same webpack chunk-path-builder patterns the React crawler already resolves (see
+ * `src/lazyLoad/shared/webpackChunkParsers.ts`). Internal#75: without this, a webpack
+ * or module-federation entry chunk downloaded under `generic` tech only ever yields the
+ * chunks that happened to fire live during the crawl — its own async-chunk hash-map is
+ * never statically enumerated the way React's `react_webpackChunkPaths.ts` already does.
+ *
+ * Reads from disk (recovering each file's source URL via the same
+ * `// File Source: <url>` header `generic_stringsDiscovery.ts` relies on) rather than
+ * re-fetching over the network, since the file is already on disk from an earlier
+ * download pass.
+ */
+const generic_webpackChunkPaths = (outputDir_1, ...args_1) => __awaiter(void 0, [outputDir_1, ...args_1], void 0, function* (outputDir, alreadyKnownUrls = new Set(), threads = 1) {
+    if (!fs.existsSync(outputDir))
+        return [];
+    const files = fs.readdirSync(outputDir, { recursive: true, encoding: "utf8" });
+    const jsFiles = files.filter((f) => f.endsWith(".js") || f.endsWith(".mjs"));
+    const candidates = new Set();
+    for (const relPath of jsFiles) {
+        const filePath = path.join(outputDir, relPath);
+        let content;
+        try {
+            content = fs.readFileSync(filePath, "utf-8");
+        }
+        catch (_a) {
+            continue;
+        }
+        const sourceUrl = extractFileSourceUrl(content);
+        if (!sourceUrl)
+            continue;
+        for (const url of extractWebpackChunkUrls(content, sourceUrl)) {
+            if (!alreadyKnownUrls.has(url))
+                candidates.add(url);
+        }
+    }
+    if (candidates.size === 0)
+        return [];
+    return confirmJsContentType([...candidates], threads);
+});
+export default generic_webpackChunkPaths;
+//# sourceMappingURL=generic_webpackChunkPaths.js.map

package/build/lazyLoad/generic/generic_webpackChunkPaths.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_webpackChunkPaths.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_webpackChunkPaths.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AAExE;;;;;;;;;;;;GAYG;AACH,MAAM,yBAAyB,GAAG,yBAIb,EAAE,gEAHnB,SAAiB,EACjB,mBAAgC,IAAI,GAAG,EAAE,EACzC,UAAkB,CAAC;IAEnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAEzC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAa,CAAC;IAC3F,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7E,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;QAAC,WAAM,CAAC;YACL,SAAS;QACb,CAAC;QAED,MAAM,SAAS,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,KAAK,MAAM,GAAG,IAAI,uBAAuB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;IACL,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,oBAAoB,CAAC,CAAC,GAAG,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC,CAAA,CAAC;AAEF,eAAe,yBAAyB,CAAC"}

package/build/lazyLoad/globals.js

@@ -8,6 +8,10 @@ let json_urls = [];
 let max_req_queue;
 /** List of URLs which has been crawled */
 let crawled_urls = [];
+/** Global map of JS file content hash -> count of files seen with that hash (used by generic-tech stagnation detection) */
+let js_file_hash_counts = new Map();
+/** Total number of JS files whose content hash has been recorded */
+let js_file_total_count = 0;
 /**
  * Gets the current scope configuration.
  * @returns Array of domains in scope
@@ -108,4 +112,31 @@ export const addCrawledUrl = (url) => {
 export const clearCrawledUrls = () => {
     crawled_urls = [];
 };
+/**
+ * Records a JS file's content hash, incrementing both its per-hash count and the total count.
+ * Used by generic-tech stagnation detection to track how much discovered content is duplicate.
+ * @param hash - Content hash (e.g. sha256) of a discovered JS file
+ */
+export const recordJsFileHash = (hash) => {
+    var _a;
+    js_file_hash_counts.set(hash, ((_a = js_file_hash_counts.get(hash)) !== null && _a !== void 0 ? _a : 0) + 1);
+    js_file_total_count++;
+};
+/**
+ * Gets the current map of JS file content hash -> count.
+ * @returns Map of content hash to occurrence count
+ */
+export const getJsFileHashCounts = () => js_file_hash_counts;
+/**
+ * Gets the total number of JS files whose content hash has been recorded.
+ * @returns Total recorded JS file count
+ */
+export const getJsFileTotalCount = () => js_file_total_count;
+/**
+ * Clears all recorded JS file content hashes and resets the total count.
+ */
+export const clearJsFileHashCounts = () => {
+    js_file_hash_counts = new Map();
+    js_file_total_count = 0;
+};
 //# sourceMappingURL=globals.js.map

package/build/lazyLoad/globals.js.map

@@ -1 +1 @@
-{"version":3,"file":"globals.js","sourceRoot":"","sources":["../../src/lazyLoad/globals.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,IAAI,KAAK,GAAa,EAAE,CAAC;AACzB,qEAAqE;AACrE,IAAI,OAAO,GAAa,EAAE,CAAC;AAC3B,+DAA+D;AAC/D,IAAI,SAAS,GAAa,EAAE,CAAC;AAC7B,oDAAoD;AACpD,IAAI,aAAqB,CAAC;AAC1B,0CAA0C;AAC1C,IAAI,YAAY,GAAa,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAa,EAAE,CAAC,KAAK,CAAC;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,QAAkB,EAAQ,EAAE;IACjD,KAAK,GAAG,QAAQ,CAAC;AACrB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAQ,EAAE;IAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,GAAa,EAAE,CAAC,OAAO,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAS,EAAE;IAClC,OAAO,GAAG,EAAE,CAAC;AACjB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAsB,EAAQ,EAAE;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,oBAAoB;IACpB,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAa,EAAE,CAAC,SAAS,CAAC;AAErD;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,GAAS,EAAE;IACpC,SAAS,GAAG,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAQ,EAAE;IAChD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAW,EAAE,CAAC,aAAa,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAc,EAAQ,EAAE;IACnD,aAAa,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAW,EAAE;IACzD,OAAO,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAsB,EAAQ,EAAE;IAC1D,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,YAAY,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IAC9B,CAAC;SAAM,CAAC;QACJ,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAS,EAAE;IACvC,YAAY,GAAG,EAAE,CAAC;AACtB,CAAC,CAAC"}
+{"version":3,"file":"globals.js","sourceRoot":"","sources":["../../src/lazyLoad/globals.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,IAAI,KAAK,GAAa,EAAE,CAAC;AACzB,qEAAqE;AACrE,IAAI,OAAO,GAAa,EAAE,CAAC;AAC3B,+DAA+D;AAC/D,IAAI,SAAS,GAAa,EAAE,CAAC;AAC7B,oDAAoD;AACpD,IAAI,aAAqB,CAAC;AAC1B,0CAA0C;AAC1C,IAAI,YAAY,GAAa,EAAE,CAAC;AAChC,2HAA2H;AAC3H,IAAI,mBAAmB,GAAwB,IAAI,GAAG,EAAE,CAAC;AACzD,oEAAoE;AACpE,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAa,EAAE,CAAC,KAAK,CAAC;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,QAAkB,EAAQ,EAAE;IACjD,KAAK,GAAG,QAAQ,CAAC;AACrB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAQ,EAAE;IAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,GAAa,EAAE,CAAC,OAAO,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAS,EAAE;IAClC,OAAO,GAAG,EAAE,CAAC;AACjB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAsB,EAAQ,EAAE;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,oBAAoB;IACpB,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAa,EAAE,CAAC,SAAS,CAAC;AAErD;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,GAAS,EAAE;IACpC,SAAS,GAAG,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAQ,EAAE;IAChD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAW,EAAE,CAAC,aAAa,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAc,EAAQ,EAAE;IACnD,aAAa,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAW,EAAE;IACzD,OAAO,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAsB,EAAQ,EAAE;IAC1D,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,YAAY,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IAC9B,CAAC;SAAM,CAAC;QACJ,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAS,EAAE;IACvC,YAAY,GAAG,EAAE,CAAC;AACtB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAQ,EAAE;;IACnD,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAA,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,mBAAmB,EAAE,CAAC;AAC1B,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAwB,EAAE,CAAC,mBAAmB,CAAC;AAElF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAW,EAAE,CAAC,mBAAmB,CAAC;AAErE;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAS,EAAE;IAC5C,mBAAmB,GAAG,IAAI,GAAG,EAAE,CAAC;IAChC,mBAAmB,GAAG,CAAC,CAAC;AAC5B,CAAC,CAAC"}

package/build/lazyLoad/index.js

@@ -10,13 +10,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 var _a;
 import chalk from "chalk";
 import fs from "fs";
-import frameworkDetect from "./techDetect/index.js";
-import CONFIG from "../globalConfig.js";
+import frameworkDetect, { getLastInterceptedUrls } from "./techDetect/index.js";
 import _traverse from "@babel/traverse";
 const traverse = ((_a = _traverse.default) !== null && _a !== void 0 ? _a : _traverse);
 import { URL } from "url";
 // Next.js
 import NextJsCrawler from "./next_js/NextJsCrawler.js";
+import next_serverActionIdScan from "./next_js/next_serverActionIdScan.js";
 import { next_buildId_RSC } from "./next_js/next_buildId.js";
 // Nuxt.js
 import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
@@ -41,8 +41,9 @@ import react_getScriptTags from "./react/react_getScriptTags.js";
 import react_webpackChunkPaths from "./react/react_webpackChunkPaths.js";
 import react_sourcemapUrls from "./react/react_sourcemapUrls.js";
 import react_followImports from "./react/react_followImports.js";
-import downloadLoadedJs from "./downloadLoadedJsUtil.js";
 import { DownloadQueue } from "./downloadQueue.js";
+import generic_crawlPages from "./generic/generic_crawlPages.js";
+import resolveGenericScope from "./generic/generic_resolveScope.js";
 import path from "path";
 import { join } from "path";
 import { extractSourceMaps } from "../sourcemaps/index.js";
@@ -55,6 +56,8 @@ import { progressLog, progressWarn } from "../utility/progressLog.js";
 import { resolveHostOutputDirectory } from "./outputPath.js";
 import { withRequestSignal } from "../utility/makeReq.js";
 import { resolveTargetInputs } from "../utility/targetInputs.js";
+import { accumulateTechnique, createTechniqueRecorder } from "./researchUtils.js";
+import { printMsg, MSG } from "../utility/printMsg.js";
 /**
  * Downloads the required JavaScript files for a given URL
  * @param {string} url The URL to download the JS files from
@@ -67,7 +70,7 @@ import { resolveTargetInputs } from "../utility/targetInputs.js";
  * @param {boolean} insecure If true, then disable SSL certificate verification
  * @returns {Promise<void>} A Promise that resolves when the download is complete
  */
-const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1) => __awaiter(void 0, [url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1], void 0, function* (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile, insecure, buildId, sourcemapDir, research, researchOutput, maxIterations, maxJsSizeMb = 2, hardTimeoutMs = 30 * 60 * 1000, maxPageVisits = 200, includeMethods = [], excludeMethods = [], detectionTimeoutMs = 30 * 1000, cancellationSignal, isBatch = false) {
+const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1) => __awaiter(void 0, [url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1], void 0, function* (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile, insecure, buildId, sourcemapDir, research, researchOutput, maxIterations, maxJsSizeMb = 2, hardTimeoutMs = 30 * 60 * 1000, maxPageVisits = 200, includeMethods = [], excludeMethods = [], detectionTimeoutMs = 30 * 1000, cancellationSignal, isBatch = false, maxRedirects = 20, stringsEnabled = false, stringsMaxIterations = 5, stagnationTimeinMs = 0, stagnationPercentage = 80, stagnationMonitorMs = 60 * 1000, rscParamBruteforceLimit = 20) {
     const resolvedTargets = resolveTargetInputs(url);
     // Hoisted so the timeout handler can stop discovery and drain downloads.
     let activeCrawler = null;
@@ -99,14 +102,14 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
     if (cancellationSignal === null || cancellationSignal === void 0 ? void 0 : cancellationSignal.aborted)
         requestCancellation();
     const work = () => __awaiter(void 0, void 0, void 0, function* () {
-        console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
+        printMsg(MSG.Header, "[i] Loading 'Lazy Load' module");
         if (globals.getDisableSandbox()) {
-            console.error(chalk.yellow("[!] Browser sandbox disabled"));
+            printMsg(MSG.Warn, "[!] Browser sandbox disabled");
         }
         globals.setInsecure(insecure);
         if (insecure) {
             process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-            console.error(chalk.yellow("[!] Running in insecure mode. SSL certificate verification disabled"));
+            printMsg(MSG.Warn, "[!] Running in insecure mode. SSL certificate verification disabled");
         }
         // if cache enabled, check if the cache file exists or not. If no, then create a new one
         if (!globals.getDisableCache()) {
@@ -117,7 +120,7 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
         for (const url of resolvedTargets.targets) {
             if (hardTimeoutReached)
                 break;
-            console.log(chalk.cyan(`[i] Processing ${url}`));
+            printMsg(MSG.Header, `[i] Processing ${url}`);
             lazyLoadGlobals.clearJsUrls();
             lazyLoadGlobals.clearJsonUrls();
             lazyLoadGlobals.clearCrawledUrls();
@@ -149,15 +152,15 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     activeDetectionController = null;
             }
             if (detectionTimedOut) {
-                console.error(chalk.yellow(`[!] Framework detection timed out after ${detectionTimeoutMs}ms; treating as not detected`));
+                printMsg(MSG.Warn, `[!] Framework detection timed out after ${detectionTimeoutMs}ms; treating as not detected`);
             }
             if (hardTimeoutReached)
                 break;
             globals.setTech(tech ? tech.name : "");
             if (tech) {
                 if (tech.name === "next") {
-                    console.log(chalk.green("[✓] Next.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] Next.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     activeQueue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     const crawler = new NextJsCrawler({
                         url,
@@ -168,6 +171,7 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         research,
                         maxIterations,
                         maxPageVisits,
+                        rscParamBruteforceLimit,
                         onUrlsDiscovered: (urls) => enqueue(activeQueue, urls),
                         includeMethods,
                         excludeMethods,
@@ -187,6 +191,14 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         return;
                     activeQueue.printSummary();
                     activeQueue = null;
+                    if (shouldRunMethod("next_serverActionIdScan", includeMethods, excludeMethods)) {
+                        const scanDir = resolveHostOutputDirectory(output, new URL(url).host, isBatch);
+                        const scanResult = yield next_serverActionIdScan(scanDir);
+                        if (scanResult.allActionIds.length > 0) {
+                            printMsg(MSG.Run, `[✓] Found ${scanResult.allActionIds.length} Server Action ID(s) referenced in downloaded chunks (disclosure only — not invoked)`);
+                            fs.writeFileSync(path.join(scanDir, "server_action_ids.json"), JSON.stringify(scanResult, null, 4));
+                        }
+                    }
                     if (buildId) {
                         // get the buildId
                         // the directory is the output <output>/<host.replace(":", "_")>/___subsequent_requests
@@ -195,7 +207,7 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         if (hardTimeoutReached)
                             return;
                         if (buildId) {
-                            console.log(chalk.cyan("[+] Found buildId: " + buildId));
+                            printMsg(MSG.Header, "[+] Found buildId: " + buildId);
                             // now, write it to a file
                             fs.writeFileSync(path.join(targetOutputDir, "BUILD_ID"), buildId);
                         }
@@ -204,75 +216,97 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (research) {
                         // prettify the JSON and write
                         fs.writeFileSync(researchOutput, JSON.stringify(crawler.techniqueEfficiencyMapping, null, 4));
-                        console.log(chalk.green("[✓] Research mode enabled. Technique efficiency written to " + researchOutput));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
                     }
                     // extract the source maps
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "vue") {
-                    console.log(chalk.green("[✓] Vue.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] Vue.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     activeQueue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     const queue = activeQueue;
                     const onFilesDiscovered = (files) => enqueue(queue, files);
+                    const vueResearchMap = {};
+                    const vueOnTechnique = research ? createTechniqueRecorder(vueResearchMap) : undefined;
                     // run the full discovery pipeline against the entry URL
-                    const { clientSidePaths } = yield vue_discoverJsFiles(url, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, threads);
+                    const { clientSidePaths } = yield vue_discoverJsFiles(url, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, threads, true, true, vueOnTechnique);
                     if (hardTimeoutReached)
                         return;
                     // recurse the same pipeline through every client-side path we found
-                    yield vue_recursiveClientSidePathDownload(clientSidePaths, threads, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods);
+                    yield vue_recursiveClientSidePathDownload(clientSidePaths, threads, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, vueOnTechnique);
                     if (hardTimeoutReached)
                         return;
                     yield queue.drain();
                     if (hardTimeoutReached)
                         return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(vueResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                     // extract the source maps
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "nuxt") {
-                    console.log(chalk.green("[✓] Nuxt.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] Nuxt.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const nuxtResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("nuxt_getFromPageSource", includeMethods, excludeMethods)
                         ? yield nuxt_getFromPageSource(url)
                         : [];
                     enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_getFromPageSource", jsFilesFromPageSource);
                     const jsFilesFromStringAnalysis = shouldRunMethod("nuxt_stringAnalysisJSFiles", includeMethods, excludeMethods)
                         ? yield nuxt_stringAnalysisJSFiles(url, threads)
                         : [];
                     enqueue(queue, jsFilesFromStringAnalysis);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_stringAnalysisJSFiles", jsFilesFromStringAnalysis);
                     const firstBatch = [...new Set([...jsFilesFromPageSource, ...jsFilesFromStringAnalysis])];
-                    let jsFilesFromAST = [];
+                    const jsFilesFromAST = [];
                     if (shouldRunMethod("nuxt_astParse", includeMethods, excludeMethods)) {
-                        console.log(chalk.cyan("[i] Analyzing functions in the files found"));
+                        printMsg(MSG.Header, "[i] Analyzing functions in the files found");
                         for (const jsFile of firstBatch) {
                             jsFilesFromAST.push(...(yield nuxt_astParse(jsFile)));
                         }
                     }
                     enqueue(queue, jsFilesFromAST);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_astParse", jsFilesFromAST);
                     enqueue(queue, lazyLoadGlobals.getJsUrls());
                     const buildsManifestFiles = shouldRunMethod("nuxt_getBuildsManifest", includeMethods, excludeMethods)
                         ? yield nuxt_getBuildsManifest(url)
                         : [];
                     enqueue(queue, buildsManifestFiles);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_getBuildsManifest", buildsManifestFiles);
                     yield queue.drain();
                     if (hardTimeoutReached)
                         return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(nuxtResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                 }
                 else if (tech.name === "svelte") {
-                    console.log(chalk.green("[✓] Svelte detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] Svelte detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const svelteResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("svelte_getFromPageSource", includeMethods, excludeMethods)
                         ? yield svelte_getFromPageSource(url)
                         : [];
                     enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_getFromPageSource", jsFilesFromPageSource);
                     // probe /<appDir>/version.json — SvelteKit serves this for the `updated` store
                     // but never references it from HTML or JS, so it is invisible to all other steps
                     const appDir = (() => {
@@ -292,6 +326,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (versionJsonFiles.length > 0) {
                         enqueue(queue, versionJsonFiles);
                     }
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_getVersionJson", versionJsonFiles);
                     // analyze the strings now
                     let jsFilesFromStringAnalysis = [];
                     let mapFilesFromStringAnalysis = [];
@@ -303,16 +339,23 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         if (mapFilesFromStringAnalysis.length > 0) {
                             enqueue(queue, mapFilesFromStringAnalysis);
                         }
+                        if (research) {
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", jsFilesFromStringAnalysis);
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", mapFilesFromStringAnalysis);
+                        }
                     }
                     // recursively follow ESM static imports (import ... from "./chunk.js")
+                    // svelte_followImports reuses react_followImports — tracked under react_followImports key
                     const visited = new Set();
                     let toFollow = [...new Set([...jsFilesFromPageSource, ...jsFilesFromStringAnalysis])];
                     while (toFollow.length > 0) {
                         const newFiles = yield react_followImports(toFollow, maxJsSizeMb, url, visited, threads);
                         if (newFiles.length === 0)
                             break;
-                        console.log(chalk.green(`[✓] Discovered ${newFiles.length} more JS file(s) via imports`));
+                        printMsg(MSG.Run, `[✓] Discovered ${newFiles.length} more JS file(s) via imports`);
                         enqueue(queue, newFiles);
+                        if (research)
+                            accumulateTechnique(svelteResearchMap, "react_followImports", newFiles);
                         toFollow = newFiles;
                     }
                     // crawl same-origin HTML pages found via <a href> and <link href>,
@@ -320,6 +363,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     const jsFilesFromPageCrawl = shouldRunMethod("svelte_recursivePageCrawl", includeMethods, excludeMethods)
                         ? yield svelte_recursivePageCrawl(url, maxJsSizeMb, (files) => enqueue(queue, files))
                         : [];
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_recursivePageCrawl", jsFilesFromPageCrawl);
                     // Svelte/Astro apps use client-side routing — the home page rarely has
                     // <a href> links in its server-rendered HTML. Scan downloaded JS for
                     // embedded page path strings (e.g. "/admin", "/debug") and visit each
@@ -331,6 +376,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (jsFilesFromPathScan.length > 0) {
                         enqueue(queue, jsFilesFromPathScan);
                     }
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_discoverPagesFromJs", jsFilesFromPathScan);
                     // run string analysis once more to catch JS files discovered during page crawl
                     if ((jsFilesFromPageCrawl.length > 0 || jsFilesFromPathScan.length > 0) &&
                         shouldRunMethod("svelte_stringAnalysisJSFiles", includeMethods, excludeMethods)) {
@@ -339,23 +386,34 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         if (mapFilesFromStringAnalysis2.length > 0) {
                             enqueue(queue, mapFilesFromStringAnalysis2);
                         }
+                        if (research) {
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", jsFilesFromStringAnalysis2);
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", mapFilesFromStringAnalysis2);
+                        }
                     }
                     yield queue.drain();
                     if (hardTimeoutReached)
                         return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(svelteResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "angular") {
-                    console.log(chalk.green("[✓] Angular detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] Angular detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const angularResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("angular_getFromPageSource", includeMethods, excludeMethods)
                         ? yield angular_getFromPageSource(url)
                         : [];
                     enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(angularResearchMap, "angular_getFromPageSource", jsFilesFromPageSource);
                     // files using the main.js
                     if (shouldRunMethod("angular_getFromMainJs", includeMethods, excludeMethods)) {
                         let mainJsUrl;
@@ -368,10 +426,14 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         if (mainJsUrl) {
                             const jsFilesFromMainJs = yield angular_getFromMainJs(mainJsUrl);
                             enqueue(queue, jsFilesFromMainJs);
+                            if (research)
+                                accumulateTechnique(angularResearchMap, "angular_getFromMainJs", jsFilesFromMainJs);
                             // recursively follow chunk-to-chunk imports beyond the first hop
                             if (shouldRunMethod("angular_recursiveChunkImports", includeMethods, excludeMethods)) {
                                 const transitiveChunks = yield angular_recursiveChunkImports(jsFilesFromMainJs, threads);
                                 enqueue(queue, transitiveChunks);
+                                if (research)
+                                    accumulateTechnique(angularResearchMap, "angular_recursiveChunkImports", transitiveChunks);
                             }
                         }
                     }
@@ -379,16 +441,21 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (hardTimeoutReached)
                         return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(angularResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                 }
                 else if (tech.name === "react") {
-                    console.log(chalk.green("[✓] React detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
+                    printMsg(MSG.Run, "[✓] React detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
                     const queue = new DownloadQueue(output, threads, {
                         compactOutput: true,
                         onProgress: (progress) => activeDownloadProgress === null || activeDownloadProgress === void 0 ? void 0 : activeDownloadProgress.update(progress),
                         alreadyBatchTargetRoot: isBatch,
                     });
                     activeQueue = queue;
+                    const reactResearchMap = {};
                     // Seed: <script src> tags + <link rel="modulepreload"> (Vite vendor chunks)
                     const jsFilesFromPageSource = shouldRunMethod("react_getScriptTags", includeMethods, excludeMethods)
                         ? yield react_getScriptTags(url, maxJsSizeMb, output, isBatch)
@@ -396,6 +463,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (hardTimeoutReached)
                         return;
                     enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(reactResearchMap, "react_getScriptTags", jsFilesFromPageSource);
                     // webpack-style chunk path builders (CRA / custom webpack configs)
                     const webpackChunkPaths = shouldRunMethod("react_webpackChunkPaths", includeMethods, excludeMethods)
                         ? yield react_webpackChunkPaths(url, maxJsSizeMb, jsFilesFromPageSource, threads)
@@ -406,6 +475,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     activeDownloadProgress.update(queue.progress);
                     try {
                         enqueue(queue, webpackChunkPaths);
+                        if (research)
+                            accumulateTechnique(reactResearchMap, "react_webpackChunkPaths", webpackChunkPaths);
                         // Recursively follow ESM imports and Vite __vite_mapDeps references.
                         // visited starts empty so the seed files are fetched and parsed in the first round.
                         const visited = new Set();
@@ -419,6 +490,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                                     break;
                                 progressLog(chalk.green(`[✓] Discovered ${newFiles.length} more JS file(s) via imports`));
                                 enqueue(queue, newFiles);
+                                if (research)
+                                    accumulateTechnique(reactResearchMap, "react_followImports", newFiles);
                                 toFollow = newFiles;
                             }
                         }
@@ -427,6 +500,8 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                             shouldRunMethod("react_sourcemapUrls", includeMethods, excludeMethods)) {
                             const sourcemapUrls = yield react_sourcemapUrls([...visited], threads);
                             enqueue(queue, sourcemapUrls);
+                            if (research)
+                                accumulateTechnique(reactResearchMap, "react_sourcemapUrls", sourcemapUrls);
                         }
                         yield queue.drain();
                     }
@@ -437,50 +512,38 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                     if (hardTimeoutReached)
                         return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(reactResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
             }
             else {
-                console.error(chalk.red("[!] Framework not detected :("));
-                console.log(chalk.magenta(CONFIG.notFoundMessage));
-                console.log(chalk.yellow("[i] Trying to download loaded JS files"));
-                const js_urls = yield downloadLoadedJs(url);
+                printMsg(MSG.Warn, "[i] Framework not detected — falling back to generic JS extraction");
+                globals.setTech("generic");
+                // If the caller didn't customize scope (still the "*" default) and isn't
+                // using --strict-scope, default the generic crawl's scope to the origin
+                // reached after following redirects, rather than crawling the whole web.
+                if (!strictScope && inputScope.length === 1 && inputScope[0] === "*") {
+                    const resolvedScope = yield resolveGenericScope(url, maxRedirects);
+                    lazyLoadGlobals.setScope(resolvedScope);
+                }
+                // Downloads JS incrementally as pages are crawled (see generic_crawlPages.ts) —
+                // the returned list is only used for the final summary count.
+                const genericResearchMap = {};
+                const allUrls = yield generic_crawlPages(url, maxJsSizeMb, output, maxPageVisits, threads, research ? genericResearchMap : undefined, stringsEnabled, stringsMaxIterations, getLastInterceptedUrls(), stagnationTimeinMs, stagnationPercentage, stagnationMonitorMs);
                 if (hardTimeoutReached)
                     return;
-                if (js_urls && js_urls.length > 0) {
-                    console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
-                    // Second-chance tech detection: scan downloaded URL paths for
-                    // framework signatures that Puppeteer may have missed on timeout
-                    // (e.g. Next.js served at a non-root basePath).
-                    let secondChanceTech = null;
-                    let secondChanceEvidence = "";
-                    for (const u of js_urls) {
-                        if (u.includes("/_next/")) {
-                            secondChanceTech = "next";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                        if (u.includes("/_nuxt/")) {
-                            secondChanceTech = "nuxt";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                        if (u.includes("/_app/immutable/")) {
-                            secondChanceTech = "svelte";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                    }
-                    if (secondChanceTech) {
-                        console.log(chalk.green(`[✓] Detected ${secondChanceTech} from downloaded file paths (evidence: ${secondChanceEvidence})`));
-                        globals.setTech(secondChanceTech);
-                    }
-                    const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
-                    enqueue(queue, js_urls);
-                    yield queue.drain();
-                    if (hardTimeoutReached)
-                        return;
-                    queue.printSummary();
+                if (allUrls.length > 0) {
+                    printMsg(MSG.Run, `[✓] Found ${allUrls.length} JS file(s)`);
+                }
+                else {
+                    printMsg(MSG.Warn, "[i] No JS files discovered");
+                }
+                if (research) {
+                    fs.writeFileSync(researchOutput, JSON.stringify(genericResearchMap, null, 4));
+                    printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
                 }
             }
         }