@js-recon/js-recon 1.4.3 → 2.0.1-alpha.1

package/build/lazyLoad/index.js

@@ -10,13 +10,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 var _a;
 import chalk from "chalk";
 import fs from "fs";
-import frameworkDetect from "./techDetect/index.js";
-import CONFIG from "../globalConfig.js";
+import frameworkDetect, { getLastInterceptedUrls } from "./techDetect/index.js";
 import _traverse from "@babel/traverse";
 const traverse = ((_a = _traverse.default) !== null && _a !== void 0 ? _a : _traverse);
 import { URL } from "url";
 // Next.js
 import NextJsCrawler from "./next_js/NextJsCrawler.js";
+import next_serverActionIdScan from "./next_js/next_serverActionIdScan.js";
 import { next_buildId_RSC } from "./next_js/next_buildId.js";
 // Nuxt.js
 import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
@@ -41,8 +41,9 @@ import react_getScriptTags from "./react/react_getScriptTags.js";
 import react_webpackChunkPaths from "./react/react_webpackChunkPaths.js";
 import react_sourcemapUrls from "./react/react_sourcemapUrls.js";
 import react_followImports from "./react/react_followImports.js";
-import downloadLoadedJs from "./downloadLoadedJsUtil.js";
 import { DownloadQueue } from "./downloadQueue.js";
+import generic_crawlPages from "./generic/generic_crawlPages.js";
+import resolveGenericScope from "./generic/generic_resolveScope.js";
 import path from "path";
 import { join } from "path";
 import { extractSourceMaps } from "../sourcemaps/index.js";
@@ -50,6 +51,13 @@ import { extractSourceMaps } from "../sourcemaps/index.js";
 import * as lazyLoadGlobals from "./globals.js";
 import * as globals from "../utility/globals.js";
 import { shouldRunMethod } from "./methodFilter.js";
+import DownloadProgress from "../utility/downloadProgress.js";
+import { progressLog, progressWarn } from "../utility/progressLog.js";
+import { resolveHostOutputDirectory } from "./outputPath.js";
+import { withRequestSignal } from "../utility/makeReq.js";
+import { resolveTargetInputs } from "../utility/targetInputs.js";
+import { accumulateTechnique, createTechniqueRecorder } from "./researchUtils.js";
+import { printMsg, MSG } from "../utility/printMsg.js";
 /**
  * Downloads the required JavaScript files for a given URL
  * @param {string} url The URL to download the JS files from
@@ -62,19 +70,46 @@ import { shouldRunMethod } from "./methodFilter.js";
  * @param {boolean} insecure If true, then disable SSL certificate verification
  * @returns {Promise<void>} A Promise that resolves when the download is complete
  */
-const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1) => __awaiter(void 0, [url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1], void 0, function* (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile, insecure, buildId, sourcemapDir, research, researchOutput, maxIterations, maxJsSizeMb = 2, hardTimeoutMs = 30 * 60 * 1000, maxPageVisits = 200, includeMethods = [], excludeMethods = [], detectionTimeoutMs = 30 * 1000) {
+const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1) => __awaiter(void 0, [url_1, output_1, strictScope_1, inputScope_1, threads_1, subsequentRequestsFlag_1, urlsFile_1, insecure_1, buildId_1, sourcemapDir_1, research_1, researchOutput_1, maxIterations_1, ...args_1], void 0, function* (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile, insecure, buildId, sourcemapDir, research, researchOutput, maxIterations, maxJsSizeMb = 2, hardTimeoutMs = 30 * 60 * 1000, maxPageVisits = 200, includeMethods = [], excludeMethods = [], detectionTimeoutMs = 30 * 1000, cancellationSignal, isBatch = false, maxRedirects = 20, stringsEnabled = false, stringsMaxIterations = 5, stagnationTimeinMs = 0, stagnationPercentage = 80, stagnationMonitorMs = 60 * 1000, rscParamBruteforceLimit = 20) {
+    const resolvedTargets = resolveTargetInputs(url);
     // Hoisted so the timeout handler can stop discovery and drain downloads.
     let activeCrawler = null;
     let activeQueue = null;
+    let activeDownloadProgress = null;
+    let activeDetectionController = null;
+    let hardTimeoutReached = false;
+    const workController = new AbortController();
+    const requestCancellation = () => {
+        hardTimeoutReached = true;
+        workController.abort();
+        activeDetectionController === null || activeDetectionController === void 0 ? void 0 : activeDetectionController.abort();
+        activeCrawler === null || activeCrawler === void 0 ? void 0 : activeCrawler.stop();
+        activeDownloadProgress === null || activeDownloadProgress === void 0 ? void 0 : activeDownloadProgress.stop();
+        activeDownloadProgress = null;
+    };
+    const enqueue = (queue, urls) => {
+        if (!hardTimeoutReached)
+            queue.push(urls);
+    };
+    const handleExternalCancellation = () => requestCancellation();
+    const cleanupActiveWork = () => __awaiter(void 0, void 0, void 0, function* () {
+        activeDownloadProgress === null || activeDownloadProgress === void 0 ? void 0 : activeDownloadProgress.stop();
+        activeDownloadProgress = null;
+        if (hardTimeoutReached)
+            yield (activeQueue === null || activeQueue === void 0 ? void 0 : activeQueue.drain().catch(() => undefined));
+    });
+    cancellationSignal === null || cancellationSignal === void 0 ? void 0 : cancellationSignal.addEventListener("abort", handleExternalCancellation, { once: true });
+    if (cancellationSignal === null || cancellationSignal === void 0 ? void 0 : cancellationSignal.aborted)
+        requestCancellation();
     const work = () => __awaiter(void 0, void 0, void 0, function* () {
-        console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
+        printMsg(MSG.Header, "[i] Loading 'Lazy Load' module");
         if (globals.getDisableSandbox()) {
-            console.error(chalk.yellow("[!] Browser sandbox disabled"));
+            printMsg(MSG.Warn, "[!] Browser sandbox disabled");
         }
         globals.setInsecure(insecure);
         if (insecure) {
             process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-            console.error(chalk.yellow("[!] Running in insecure mode. SSL certificate verification disabled"));
+            printMsg(MSG.Warn, "[!] Running in insecure mode. SSL certificate verification disabled");
         }
         // if cache enabled, check if the cache file exists or not. If no, then create a new one
         if (!globals.getDisableCache()) {
@@ -82,52 +117,51 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                 fs.writeFileSync(globals.getRespCacheFile(), "{}");
             }
         }
-        let urls;
-        // check if the url is file or a URL
-        if (fs.existsSync(url)) {
-            urls = fs.readFileSync(url, "utf8").split("\n");
-            // remove the empty lines
-            urls = urls.filter((url) => url.trim() !== "");
-        }
-        else if (url.match(/https?:\/\/[a-zA-Z0-9\-_\.:]+/)) {
-            urls = [url];
-        }
-        else {
-            console.error(chalk.red("[!] Invalid URL or file path"));
-            process.exit(3);
-        }
-        for (const url of urls) {
-            console.log(chalk.cyan(`[i] Processing ${url}`));
+        for (const url of resolvedTargets.targets) {
+            if (hardTimeoutReached)
+                break;
+            printMsg(MSG.Header, `[i] Processing ${url}`);
+            lazyLoadGlobals.clearJsUrls();
+            lazyLoadGlobals.clearJsonUrls();
+            lazyLoadGlobals.clearCrawledUrls();
             if (strictScope) {
-                lazyLoadGlobals.pushToScope(new URL(url).host);
+                lazyLoadGlobals.setScope([new URL(url).host]);
             }
             else {
-                lazyLoadGlobals.setScope(inputScope);
+                lazyLoadGlobals.setScope([...inputScope]);
             }
             lazyLoadGlobals.setMaxReqQueue(threads);
+            globals.setTech("");
             let tech;
-            if (detectionTimeoutMs > 0) {
-                let timedOut = false;
-                tech = yield Promise.race([
-                    frameworkDetect(url),
-                    new Promise((resolve) => setTimeout(() => {
-                        timedOut = true;
-                        resolve(null);
-                    }, detectionTimeoutMs)),
-                ]);
-                if (timedOut) {
-                    console.error(chalk.yellow(`[!] Framework detection timed out after ${detectionTimeoutMs}ms; treating as not detected`));
-                }
+            const detectionController = new AbortController();
+            activeDetectionController = detectionController;
+            let detectionTimedOut = false;
+            const detectionTimeoutHandle = detectionTimeoutMs > 0
+                ? setTimeout(() => {
+                    detectionTimedOut = true;
+                    detectionController.abort();
+                }, detectionTimeoutMs)
+                : undefined;
+            try {
+                tech = yield frameworkDetect(url, detectionController.signal);
             }
-            else {
-                tech = yield frameworkDetect(url);
+            finally {
+                if (detectionTimeoutHandle !== undefined)
+                    clearTimeout(detectionTimeoutHandle);
+                if (activeDetectionController === detectionController)
+                    activeDetectionController = null;
+            }
+            if (detectionTimedOut) {
+                printMsg(MSG.Warn, `[!] Framework detection timed out after ${detectionTimeoutMs}ms; treating as not detected`);
             }
+            if (hardTimeoutReached)
+                break;
             globals.setTech(tech ? tech.name : "");
             if (tech) {
                 if (tech.name === "next") {
-                    console.log(chalk.green("[✓] Next.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    activeQueue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] Next.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    activeQueue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     const crawler = new NextJsCrawler({
                         url,
                         output,
@@ -137,91 +171,142 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         research,
                         maxIterations,
                         maxPageVisits,
-                        onUrlsDiscovered: (urls) => activeQueue.push(urls),
+                        rscParamBruteforceLimit,
+                        onUrlsDiscovered: (urls) => enqueue(activeQueue, urls),
                         includeMethods,
                         excludeMethods,
+                        isBatch,
                     });
                     activeCrawler = crawler;
-                    yield crawler.crawl();
-                    activeCrawler = null; // done — prevent timeout handler from calling stop()
+                    try {
+                        yield crawler.crawl();
+                    }
+                    finally {
+                        activeCrawler = null; // done — prevent timeout handler from calling stop()
+                    }
+                    if (hardTimeoutReached)
+                        return;
                     yield activeQueue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     activeQueue.printSummary();
                     activeQueue = null;
+                    if (shouldRunMethod("next_serverActionIdScan", includeMethods, excludeMethods)) {
+                        const scanDir = resolveHostOutputDirectory(output, new URL(url).host, isBatch);
+                        const scanResult = yield next_serverActionIdScan(scanDir);
+                        if (scanResult.allActionIds.length > 0) {
+                            printMsg(MSG.Run, `[✓] Found ${scanResult.allActionIds.length} Server Action ID(s) referenced in downloaded chunks (disclosure only — not invoked)`);
+                            fs.writeFileSync(path.join(scanDir, "server_action_ids.json"), JSON.stringify(scanResult, null, 4));
+                        }
+                    }
                     if (buildId) {
                         // get the buildId
                         // the directory is the output <output>/<host.replace(":", "_")>/___subsequent_requests
-                        const buildId = yield next_buildId_RSC(output + "/" + new URL(url).host.replace(":", "_") + "/___subsequent_requests");
+                        const targetOutputDir = resolveHostOutputDirectory(output, new URL(url).host, isBatch);
+                        const buildId = yield next_buildId_RSC(path.join(targetOutputDir, "___subsequent_requests"));
+                        if (hardTimeoutReached)
+                            return;
                         if (buildId) {
-                            console.log(chalk.cyan("[+] Found buildId: " + buildId));
+                            printMsg(MSG.Header, "[+] Found buildId: " + buildId);
                             // now, write it to a file
-                            fs.writeFileSync(path.join(output, new URL(url).host.replace(":", "_") + "/BUILD_ID"), buildId);
+                            fs.writeFileSync(path.join(targetOutputDir, "BUILD_ID"), buildId);
                         }
                     }
                     // if the research mode is enabled, then write the technique efficiency to a file
                     if (research) {
                         // prettify the JSON and write
                         fs.writeFileSync(researchOutput, JSON.stringify(crawler.techniqueEfficiencyMapping, null, 4));
-                        console.log(chalk.green("[✓] Research mode enabled. Technique efficiency written to " + researchOutput));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
                     }
                     // extract the source maps
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "vue") {
-                    console.log(chalk.green("[✓] Vue.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    activeQueue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] Vue.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    activeQueue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     const queue = activeQueue;
-                    const onFilesDiscovered = (files) => queue.push(files);
+                    const onFilesDiscovered = (files) => enqueue(queue, files);
+                    const vueResearchMap = {};
+                    const vueOnTechnique = research ? createTechniqueRecorder(vueResearchMap) : undefined;
                     // run the full discovery pipeline against the entry URL
-                    const { clientSidePaths } = yield vue_discoverJsFiles(url, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, threads);
+                    const { clientSidePaths } = yield vue_discoverJsFiles(url, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, threads, true, true, vueOnTechnique);
+                    if (hardTimeoutReached)
+                        return;
                     // recurse the same pipeline through every client-side path we found
-                    yield vue_recursiveClientSidePathDownload(clientSidePaths, threads, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods);
+                    yield vue_recursiveClientSidePathDownload(clientSidePaths, threads, maxJsSizeMb, onFilesDiscovered, includeMethods, excludeMethods, vueOnTechnique);
+                    if (hardTimeoutReached)
+                        return;
                     yield queue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(vueResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                     // extract the source maps
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "nuxt") {
-                    console.log(chalk.green("[✓] Nuxt.js detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    const queue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] Nuxt.js detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const nuxtResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("nuxt_getFromPageSource", includeMethods, excludeMethods)
                         ? yield nuxt_getFromPageSource(url)
                         : [];
-                    queue.push(jsFilesFromPageSource);
+                    enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_getFromPageSource", jsFilesFromPageSource);
                     const jsFilesFromStringAnalysis = shouldRunMethod("nuxt_stringAnalysisJSFiles", includeMethods, excludeMethods)
                         ? yield nuxt_stringAnalysisJSFiles(url, threads)
                         : [];
-                    queue.push(jsFilesFromStringAnalysis);
+                    enqueue(queue, jsFilesFromStringAnalysis);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_stringAnalysisJSFiles", jsFilesFromStringAnalysis);
                     const firstBatch = [...new Set([...jsFilesFromPageSource, ...jsFilesFromStringAnalysis])];
-                    let jsFilesFromAST = [];
+                    const jsFilesFromAST = [];
                     if (shouldRunMethod("nuxt_astParse", includeMethods, excludeMethods)) {
-                        console.log(chalk.cyan("[i] Analyzing functions in the files found"));
+                        printMsg(MSG.Header, "[i] Analyzing functions in the files found");
                         for (const jsFile of firstBatch) {
                             jsFilesFromAST.push(...(yield nuxt_astParse(jsFile)));
                         }
                     }
-                    queue.push(jsFilesFromAST);
-                    queue.push(lazyLoadGlobals.getJsUrls());
+                    enqueue(queue, jsFilesFromAST);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_astParse", jsFilesFromAST);
+                    enqueue(queue, lazyLoadGlobals.getJsUrls());
                     const buildsManifestFiles = shouldRunMethod("nuxt_getBuildsManifest", includeMethods, excludeMethods)
                         ? yield nuxt_getBuildsManifest(url)
                         : [];
-                    queue.push(buildsManifestFiles);
+                    enqueue(queue, buildsManifestFiles);
+                    if (research)
+                        accumulateTechnique(nuxtResearchMap, "nuxt_getBuildsManifest", buildsManifestFiles);
                     yield queue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(nuxtResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                 }
                 else if (tech.name === "svelte") {
-                    console.log(chalk.green("[✓] Svelte detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    const queue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] Svelte detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const svelteResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("svelte_getFromPageSource", includeMethods, excludeMethods)
                         ? yield svelte_getFromPageSource(url)
                         : [];
-                    queue.push(jsFilesFromPageSource);
+                    enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_getFromPageSource", jsFilesFromPageSource);
                     // probe /<appDir>/version.json — SvelteKit serves this for the `updated` store
                     // but never references it from HTML or JS, so it is invisible to all other steps
                     const appDir = (() => {
@@ -239,8 +324,10 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         ? yield svelte_getVersionJson(url, appDir)
                         : [];
                     if (versionJsonFiles.length > 0) {
-                        queue.push(versionJsonFiles);
+                        enqueue(queue, versionJsonFiles);
                     }
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_getVersionJson", versionJsonFiles);
                     // analyze the strings now
                     let jsFilesFromStringAnalysis = [];
                     let mapFilesFromStringAnalysis = [];
@@ -248,27 +335,36 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         const result = yield svelte_stringAnalysisJSFiles(url, threads);
                         jsFilesFromStringAnalysis = result.jsFiles;
                         mapFilesFromStringAnalysis = result.mapFiles;
-                        queue.push(jsFilesFromStringAnalysis);
+                        enqueue(queue, jsFilesFromStringAnalysis);
                         if (mapFilesFromStringAnalysis.length > 0) {
-                            queue.push(mapFilesFromStringAnalysis);
+                            enqueue(queue, mapFilesFromStringAnalysis);
+                        }
+                        if (research) {
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", jsFilesFromStringAnalysis);
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", mapFilesFromStringAnalysis);
                         }
                     }
                     // recursively follow ESM static imports (import ... from "./chunk.js")
+                    // svelte_followImports reuses react_followImports — tracked under react_followImports key
                     const visited = new Set();
                     let toFollow = [...new Set([...jsFilesFromPageSource, ...jsFilesFromStringAnalysis])];
                     while (toFollow.length > 0) {
                         const newFiles = yield react_followImports(toFollow, maxJsSizeMb, url, visited, threads);
                         if (newFiles.length === 0)
                             break;
-                        console.log(chalk.green(`[✓] Discovered ${newFiles.length} more JS file(s) via imports`));
-                        queue.push(newFiles);
+                        printMsg(MSG.Run, `[✓] Discovered ${newFiles.length} more JS file(s) via imports`);
+                        enqueue(queue, newFiles);
+                        if (research)
+                            accumulateTechnique(svelteResearchMap, "react_followImports", newFiles);
                         toFollow = newFiles;
                     }
                     // crawl same-origin HTML pages found via <a href> and <link href>,
                     // running the full JS-discovery pipeline on each
                     const jsFilesFromPageCrawl = shouldRunMethod("svelte_recursivePageCrawl", includeMethods, excludeMethods)
-                        ? yield svelte_recursivePageCrawl(url, maxJsSizeMb, (files) => queue.push(files))
+                        ? yield svelte_recursivePageCrawl(url, maxJsSizeMb, (files) => enqueue(queue, files))
                         : [];
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_recursivePageCrawl", jsFilesFromPageCrawl);
                     // Svelte/Astro apps use client-side routing — the home page rarely has
                     // <a href> links in its server-rendered HTML. Scan downloaded JS for
                     // embedded page path strings (e.g. "/admin", "/debug") and visit each
@@ -278,31 +374,46 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         ? yield svelte_discoverPagesFromJs(url, threads)
                         : [];
                     if (jsFilesFromPathScan.length > 0) {
-                        queue.push(jsFilesFromPathScan);
+                        enqueue(queue, jsFilesFromPathScan);
                     }
+                    if (research)
+                        accumulateTechnique(svelteResearchMap, "svelte_discoverPagesFromJs", jsFilesFromPathScan);
                     // run string analysis once more to catch JS files discovered during page crawl
                     if ((jsFilesFromPageCrawl.length > 0 || jsFilesFromPathScan.length > 0) &&
                         shouldRunMethod("svelte_stringAnalysisJSFiles", includeMethods, excludeMethods)) {
                         const { jsFiles: jsFilesFromStringAnalysis2, mapFiles: mapFilesFromStringAnalysis2 } = yield svelte_stringAnalysisJSFiles(url, threads);
-                        queue.push(jsFilesFromStringAnalysis2);
+                        enqueue(queue, jsFilesFromStringAnalysis2);
                         if (mapFilesFromStringAnalysis2.length > 0) {
-                            queue.push(mapFilesFromStringAnalysis2);
+                            enqueue(queue, mapFilesFromStringAnalysis2);
+                        }
+                        if (research) {
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", jsFilesFromStringAnalysis2);
+                            accumulateTechnique(svelteResearchMap, "svelte_stringAnalysisJSFiles", mapFilesFromStringAnalysis2);
                         }
                     }
                     yield queue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(svelteResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                     yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
                 else if (tech.name === "angular") {
-                    console.log(chalk.green("[✓] Angular detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    const queue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] Angular detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    const queue = new DownloadQueue(output, threads, { alreadyBatchTargetRoot: isBatch });
                     activeQueue = queue;
+                    const angularResearchMap = {};
                     // find the files from the page source
                     const jsFilesFromPageSource = shouldRunMethod("angular_getFromPageSource", includeMethods, excludeMethods)
                         ? yield angular_getFromPageSource(url)
                         : [];
-                    queue.push(jsFilesFromPageSource);
+                    enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(angularResearchMap, "angular_getFromPageSource", jsFilesFromPageSource);
                     // files using the main.js
                     if (shouldRunMethod("angular_getFromMainJs", includeMethods, excludeMethods)) {
                         let mainJsUrl;
@@ -314,130 +425,155 @@ const lazyLoad = (url_1, output_1, strictScope_1, inputScope_1, threads_1, subse
                         }
                         if (mainJsUrl) {
                             const jsFilesFromMainJs = yield angular_getFromMainJs(mainJsUrl);
-                            queue.push(jsFilesFromMainJs);
+                            enqueue(queue, jsFilesFromMainJs);
+                            if (research)
+                                accumulateTechnique(angularResearchMap, "angular_getFromMainJs", jsFilesFromMainJs);
                             // recursively follow chunk-to-chunk imports beyond the first hop
                             if (shouldRunMethod("angular_recursiveChunkImports", includeMethods, excludeMethods)) {
                                 const transitiveChunks = yield angular_recursiveChunkImports(jsFilesFromMainJs, threads);
-                                queue.push(transitiveChunks);
+                                enqueue(queue, transitiveChunks);
+                                if (research)
+                                    accumulateTechnique(angularResearchMap, "angular_recursiveChunkImports", transitiveChunks);
                             }
                         }
                     }
                     yield queue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     queue.printSummary();
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(angularResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
                 }
                 else if (tech.name === "react") {
-                    console.log(chalk.green("[✓] React detected"));
-                    console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
-                    const queue = new DownloadQueue(output, threads);
+                    printMsg(MSG.Run, "[✓] React detected");
+                    printMsg(MSG.Warn, `Evidence: ${tech.evidence}`);
+                    const queue = new DownloadQueue(output, threads, {
+                        compactOutput: true,
+                        onProgress: (progress) => activeDownloadProgress === null || activeDownloadProgress === void 0 ? void 0 : activeDownloadProgress.update(progress),
+                        alreadyBatchTargetRoot: isBatch,
+                    });
                     activeQueue = queue;
+                    const reactResearchMap = {};
                     // Seed: <script src> tags + <link rel="modulepreload"> (Vite vendor chunks)
                     const jsFilesFromPageSource = shouldRunMethod("react_getScriptTags", includeMethods, excludeMethods)
-                        ? yield react_getScriptTags(url, maxJsSizeMb, output)
+                        ? yield react_getScriptTags(url, maxJsSizeMb, output, isBatch)
                         : [];
-                    queue.push(jsFilesFromPageSource);
+                    if (hardTimeoutReached)
+                        return;
+                    enqueue(queue, jsFilesFromPageSource);
+                    if (research)
+                        accumulateTechnique(reactResearchMap, "react_getScriptTags", jsFilesFromPageSource);
                     // webpack-style chunk path builders (CRA / custom webpack configs)
                     const webpackChunkPaths = shouldRunMethod("react_webpackChunkPaths", includeMethods, excludeMethods)
                         ? yield react_webpackChunkPaths(url, maxJsSizeMb, jsFilesFromPageSource, threads)
                         : [];
-                    queue.push(webpackChunkPaths);
-                    // Recursively follow ESM imports and Vite __vite_mapDeps references.
-                    // visited starts empty so the seed files are fetched and parsed in the first round.
-                    const visited = new Set();
-                    if (shouldRunMethod("react_followImports", includeMethods, excludeMethods)) {
-                        let toFollow = [...new Set([...jsFilesFromPageSource, ...webpackChunkPaths])];
-                        while (toFollow.length > 0) {
-                            const newFiles = yield react_followImports(toFollow, maxJsSizeMb, url, visited, threads);
-                            if (newFiles.length === 0)
-                                break;
-                            console.log(chalk.green(`[✓] Discovered ${newFiles.length} more JS file(s) via imports`));
-                            queue.push(newFiles);
-                            toFollow = newFiles;
+                    if (hardTimeoutReached)
+                        return;
+                    activeDownloadProgress = new DownloadProgress();
+                    activeDownloadProgress.update(queue.progress);
+                    try {
+                        enqueue(queue, webpackChunkPaths);
+                        if (research)
+                            accumulateTechnique(reactResearchMap, "react_webpackChunkPaths", webpackChunkPaths);
+                        // Recursively follow ESM imports and Vite __vite_mapDeps references.
+                        // visited starts empty so the seed files are fetched and parsed in the first round.
+                        const visited = new Set();
+                        if (shouldRunMethod("react_followImports", includeMethods, excludeMethods)) {
+                            let toFollow = [...new Set([...jsFilesFromPageSource, ...webpackChunkPaths])];
+                            while (toFollow.length > 0 && !hardTimeoutReached) {
+                                const newFiles = yield react_followImports(toFollow, maxJsSizeMb, url, visited, threads, true);
+                                if (hardTimeoutReached)
+                                    break;
+                                if (newFiles.length === 0)
+                                    break;
+                                progressLog(chalk.green(`[✓] Discovered ${newFiles.length} more JS file(s) via imports`));
+                                enqueue(queue, newFiles);
+                                if (research)
+                                    accumulateTechnique(reactResearchMap, "react_followImports", newFiles);
+                                toFollow = newFiles;
+                            }
+                        }
+                        // Sourcemaps for everything discovered
+                        if (!hardTimeoutReached &&
+                            shouldRunMethod("react_sourcemapUrls", includeMethods, excludeMethods)) {
+                            const sourcemapUrls = yield react_sourcemapUrls([...visited], threads);
+                            enqueue(queue, sourcemapUrls);
+                            if (research)
+                                accumulateTechnique(reactResearchMap, "react_sourcemapUrls", sourcemapUrls);
                         }
+                        yield queue.drain();
                     }
-                    // Sourcemaps for everything discovered
-                    if (shouldRunMethod("react_sourcemapUrls", includeMethods, excludeMethods)) {
-                        const sourcemapUrls = yield react_sourcemapUrls([...visited], threads);
-                        queue.push(sourcemapUrls);
+                    finally {
+                        activeDownloadProgress === null || activeDownloadProgress === void 0 ? void 0 : activeDownloadProgress.stop();
+                        activeDownloadProgress = null;
                     }
-                    yield queue.drain();
+                    if (hardTimeoutReached)
+                        return;
                     queue.printSummary();
-                    extractSourceMaps(output, join(output, sourcemapDir));
+                    if (research) {
+                        fs.writeFileSync(researchOutput, JSON.stringify(reactResearchMap, null, 4));
+                        printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
+                    }
+                    yield extractSourceMaps(output, join(output, sourcemapDir));
                 }
             }
             else {
-                console.error(chalk.red("[!] Framework not detected :("));
-                console.log(chalk.magenta(CONFIG.notFoundMessage));
-                console.log(chalk.yellow("[i] Trying to download loaded JS files"));
-                const js_urls = yield downloadLoadedJs(url);
-                if (js_urls && js_urls.length > 0) {
-                    console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
-                    // Second-chance tech detection: scan downloaded URL paths for
-                    // framework signatures that Puppeteer may have missed on timeout
-                    // (e.g. Next.js served at a non-root basePath).
-                    let secondChanceTech = null;
-                    let secondChanceEvidence = "";
-                    for (const u of js_urls) {
-                        if (u.includes("/_next/")) {
-                            secondChanceTech = "next";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                        if (u.includes("/_nuxt/")) {
-                            secondChanceTech = "nuxt";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                        if (u.includes("/_app/immutable/")) {
-                            secondChanceTech = "svelte";
-                            secondChanceEvidence = u;
-                            break;
-                        }
-                    }
-                    if (secondChanceTech) {
-                        console.log(chalk.green(`[✓] Detected ${secondChanceTech} from downloaded file paths (evidence: ${secondChanceEvidence})`));
-                        globals.setTech(secondChanceTech);
-                    }
-                    const queue = new DownloadQueue(output, threads);
-                    queue.push(js_urls);
-                    yield queue.drain();
-                    queue.printSummary();
+                printMsg(MSG.Warn, "[i] Framework not detected — falling back to generic JS extraction");
+                globals.setTech("generic");
+                // If the caller didn't customize scope (still the "*" default) and isn't
+                // using --strict-scope, default the generic crawl's scope to the origin
+                // reached after following redirects, rather than crawling the whole web.
+                if (!strictScope && inputScope.length === 1 && inputScope[0] === "*") {
+                    const resolvedScope = yield resolveGenericScope(url, maxRedirects);
+                    lazyLoadGlobals.setScope(resolvedScope);
+                }
+                // Downloads JS incrementally as pages are crawled (see generic_crawlPages.ts) —
+                // the returned list is only used for the final summary count.
+                const genericResearchMap = {};
+                const allUrls = yield generic_crawlPages(url, maxJsSizeMb, output, maxPageVisits, threads, research ? genericResearchMap : undefined, stringsEnabled, stringsMaxIterations, getLastInterceptedUrls(), stagnationTimeinMs, stagnationPercentage, stagnationMonitorMs);
+                if (hardTimeoutReached)
+                    return;
+                if (allUrls.length > 0) {
+                    printMsg(MSG.Run, `[✓] Found ${allUrls.length} JS file(s)`);
+                }
+                else {
+                    printMsg(MSG.Warn, "[i] No JS files discovered");
+                }
+                if (research) {
+                    fs.writeFileSync(researchOutput, JSON.stringify(genericResearchMap, null, 4));
+                    printMsg(MSG.Run, "[✓] Research mode enabled. Technique efficiency written to " + researchOutput);
                 }
             }
         }
     });
     if (hardTimeoutMs === 0) {
-        yield work();
-        return;
+        try {
+            yield withRequestSignal(workController.signal, work);
+            return;
+        }
+        finally {
+            cancellationSignal === null || cancellationSignal === void 0 ? void 0 : cancellationSignal.removeEventListener("abort", handleExternalCancellation);
+            yield cleanupActiveWork();
+        }
+    }
+    const timeoutHandle = setTimeout(() => {
+        requestCancellation();
+        const timeoutLabel = hardTimeoutMs >= 60000 ? `${hardTimeoutMs / 60000} min` : `${hardTimeoutMs}ms`;
+        progressWarn(chalk.yellow(`[!] Lazyload hard timeout reached (${timeoutLabel}). Stopping discovery and waiting for in-flight work...`));
+    }, hardTimeoutMs);
+    try {
+        // Never return while work() can still mutate files, globals, or terminal
+        // state. The timeout requests cooperative cancellation, then this await
+        // provides the lifecycle boundary for all in-flight framework work.
+        yield withRequestSignal(workController.signal, work);
+    }
+    finally {
+        clearTimeout(timeoutHandle);
+        cancellationSignal === null || cancellationSignal === void 0 ? void 0 : cancellationSignal.removeEventListener("abort", handleExternalCancellation);
+        yield cleanupActiveWork();
     }
-    let timeoutHandle;
-    yield Promise.race([
-        work().finally(() => {
-            // work() finished before the timeout — cancel the timer so it never
-            // fires orphaned and doesn't hold the event loop open.
-            if (timeoutHandle !== undefined)
-                clearTimeout(timeoutHandle);
-        }),
-        new Promise((resolve) => {
-            timeoutHandle = setTimeout(() => {
-                console.error(chalk.yellow(`[!] Lazyload hard timeout reached (${hardTimeoutMs / 60000} min). Draining discovered files before moving on...`));
-                // Signal the crawler to stop at its next iteration boundary.
-                activeCrawler === null || activeCrawler === void 0 ? void 0 : activeCrawler.stop();
-                const q = activeQueue;
-                if (q) {
-                    // Wait for already-queued downloads to finish, then move on.
-                    q.drain()
-                        .then(() => {
-                        q.printSummary();
-                        resolve();
-                    })
-                        .catch(() => resolve());
-                }
-                else {
-                    resolve();
-                }
-            }, hardTimeoutMs);
-        }),
-    ]);
 });
 export default lazyLoad;
 //# sourceMappingURL=index.js.map