@js-recon/js-recon 1.3.1-beta.2

package/build/lazyLoad/next_js/NextJsCrawler.js

@@ -0,0 +1,332 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import chalk from "chalk";
+import { URL } from "url";
+import next_getJSScript from "./next_GetJSScript.js";
+import next_GetLazyResourcesWebpackJs from "./next_GetLazyResourcesWebpackJs.js";
+import next_getLazyResourcesBuildManifestJs from "./next_GetLazyResourcesBuildManifestJs.js";
+import subsequentRequests from "./next_SubsequentRequests.js";
+import next_promiseResolve from "./next_promiseResolve.js";
+import next_parseLayoutJs from "./next_parseLayoutJs.js";
+import next_scriptTagsSubsequentRequests from "./next_scriptTagsSubsequentRequests.js";
+import next_bruteForceJsFiles from "./next_bruteForceJsFiles.js";
+import next_getClientSidePaths from "./next_getClientSidePaths.js";
+import * as lazyLoadGlobals from "../globals.js";
+/**
+ * Encapsulates the Next.js JS file discovery logic with recursive crawling.
+ *
+ * After running all discovery methods once, the crawler re-runs path-sensitive
+ * methods (promiseResolve, parseLayoutJs, getJSScript) on any *newly* found
+ * URLs until no new JS files are discovered, ensuring full coverage.
+ */
+class NextJsCrawler {
+    constructor(options) {
+        /**
+         * Maximum number of page URLs visited across the entire crawl instance.
+         * Prevents runaway crawls on sites with many locale/language variants where
+         * each locale page links to every other locale, causing exponential growth.
+         */
+        this.MAX_VISITED_PAGES = 200;
+        this.visitedPageCount = 0;
+        /** Set to true by stop() to abort the crawl loop gracefully. */
+        this.stopped = false;
+        this.url = options.url;
+        this.output = options.output;
+        this.subsequentRequestsFlag = options.subsequentRequestsFlag;
+        this.urlsFile = options.urlsFile;
+        this.threads = options.threads;
+        this.research = options.research;
+        this.MAX_ITERATIONS = options.maxIterations;
+        this.onUrlsDiscovered = options.onUrlsDiscovered;
+        this.discoveredUrls = new Set();
+        this.pageScriptFingerprints = new Map();
+        this.techniqueEfficiencyMapping = {};
+    }
+    // ── helpers ──────────────────────────────────────────────────────────
+    /**
+     * Normalizes a page URL to origin+pathname, stripping query and fragment.
+     * Used as the key for pageScriptFingerprints.
+     */
+    normalizePageUrl(u) {
+        try {
+            const parsed = new URL(u);
+            return parsed.origin + parsed.pathname;
+        }
+        catch (_a) {
+            return u;
+        }
+    }
+    /** Produces a stable fingerprint for a set of script URLs. */
+    scriptFingerprint(scripts) {
+        return [...new Set(scripts)].sort().join(",");
+    }
+    /** Signal the crawl loop to stop after the current iteration. */
+    stop() {
+        this.stopped = true;
+    }
+    /** Register newly found URLs and return only the ones that are truly new. */
+    registerUrls(urls) {
+        const newUrls = [];
+        for (const u of urls) {
+            if (!this.discoveredUrls.has(u)) {
+                this.discoveredUrls.add(u);
+                newUrls.push(u);
+            }
+        }
+        return newUrls;
+    }
+    /** Emit newly discovered downloadable assets to the onUrlsDiscovered callback. */
+    emitDownloadable(urls) {
+        if (!this.onUrlsDiscovered)
+            return;
+        const downloadable = urls.filter((u) => {
+            try {
+                const p = new URL(u).pathname;
+                return p.endsWith(".js") || p.endsWith(".js.map");
+            }
+            catch (_a) {
+                return false;
+            }
+        });
+        if (downloadable.length > 0)
+            this.onUrlsDiscovered(downloadable);
+    }
+    /** Snapshot the current size so we can detect growth. */
+    get size() {
+        return this.discoveredUrls.size;
+    }
+    // ── initial discovery (run-once methods) ─────────────────────────────
+    /**
+     * Runs the heavyweight / one-shot discovery methods that only need to
+     * execute once (puppeteer-based webpack analysis, build-manifest, etc.).
+     */
+    initialDiscovery() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 1. Script tags on the landing page
+            const jsFromScriptTag = yield next_getJSScript(this.url);
+            this.techniqueEfficiencyMapping["next_getJSScript"] = [
+                ...(this.techniqueEfficiencyMapping["next_getJSScript"] || []),
+                ...jsFromScriptTag,
+            ];
+            this.emitDownloadable(this.registerUrls(jsFromScriptTag));
+            // Record the landing page's script fingerprint so the recursive loop
+            // never re-probes it.
+            const landingNorm = this.normalizePageUrl(this.url);
+            this.pageScriptFingerprints.set(landingNorm, new Set([this.scriptFingerprint(jsFromScriptTag)]));
+            // 1b. Client-side paths from <a href> tags on the landing page.
+            // These are page URLs (not JS), so they'll be picked up by the
+            // recursivePass loop which visits any newly registered non-JS URL.
+            const pathsFromAnchors = yield next_getClientSidePaths(this.url);
+            this.techniqueEfficiencyMapping["next_getClientSidePaths"] = [
+                ...(this.techniqueEfficiencyMapping["next_getClientSidePaths"] || []),
+                ...pathsFromAnchors,
+            ];
+            this.emitDownloadable(this.registerUrls(pathsFromAnchors));
+            // 2. Webpack runtime analysis (puppeteer – expensive, run once per target).
+            // Skip in subsequent-requests passes: the webpack chunk URL builders are static
+            // and were already resolved in the initial lazyload call; re-running this costs
+            // 3-6 minutes per call without discovering new URLs.
+            if (!this.subsequentRequestsFlag) {
+                const jsFromWebpack = yield next_GetLazyResourcesWebpackJs(this.url);
+                this.techniqueEfficiencyMapping["next_GetLazyResourcesWebpackJs"] = jsFromWebpack;
+                lazyLoadGlobals.pushToJsUrls(jsFromWebpack);
+                this.emitDownloadable(this.registerUrls(jsFromWebpack));
+            }
+            // 3. _buildManifest.js
+            const jsFromBuildManifest = yield next_getLazyResourcesBuildManifestJs(this.url);
+            this.techniqueEfficiencyMapping["next_getLazyResourcesBuildManifestJs"] = jsFromBuildManifest;
+            lazyLoadGlobals.pushToJsUrls(jsFromBuildManifest);
+            this.emitDownloadable(this.registerUrls(jsFromBuildManifest));
+            // 4. Subsequent requests (RSC / script-tags on known endpoints)
+            if (this.subsequentRequestsFlag) {
+                const jsFromSubsequent = yield subsequentRequests(this.url, this.urlsFile, this.threads, this.output, lazyLoadGlobals.getJsUrls());
+                this.techniqueEfficiencyMapping["subsequentRequests"] = [...jsFromSubsequent];
+                this.emitDownloadable(this.registerUrls([...jsFromSubsequent]));
+                const jsFromScriptTagsSR = yield next_scriptTagsSubsequentRequests(this.url, this.urlsFile);
+                this.techniqueEfficiencyMapping["next_scriptTagsSubsequentRequests"] = jsFromScriptTagsSR;
+                lazyLoadGlobals.pushToJsUrls(jsFromScriptTagsSR);
+                this.emitDownloadable(this.registerUrls(jsFromScriptTagsSR));
+            }
+            // Also pull in anything the globals accumulated
+            this.emitDownloadable(this.registerUrls(lazyLoadGlobals.getJsUrls()));
+            this.emitDownloadable(this.registerUrls(lazyLoadGlobals.getJsonUrls()));
+        });
+    }
+    // ── recursive discovery ──────────────────────────────────────────────
+    /**
+     * Runs the lightweight methods that can discover *more* URLs from an
+     * existing set of JS file URLs. These are cheap enough to repeat.
+     *
+     * @param jsUrls The JS URLs to analyse in this pass.
+     * @returns Newly discovered URLs in this pass.
+     */
+    recursivePass(jsUrls) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let newInThisPass = [];
+            // Promise.all pattern analysis on JS file contents
+            const jsFromPromise = yield next_promiseResolve(jsUrls);
+            this.techniqueEfficiencyMapping["next_promiseResolve"] = [
+                ...(this.techniqueEfficiencyMapping["next_promiseResolve"] || []),
+                ...jsFromPromise,
+            ];
+            const newFromPromise = this.registerUrls(jsFromPromise);
+            this.emitDownloadable(newFromPromise);
+            newInThisPass.push(...newFromPromise);
+            // Layout.js parsing → discovers new client-side page paths → visits them
+            const jsFromLayout = yield next_parseLayoutJs(this.url, jsUrls);
+            this.techniqueEfficiencyMapping["next_parseLayoutJs"] = [
+                ...(this.techniqueEfficiencyMapping["next_parseLayoutJs"] || []),
+                ...jsFromLayout,
+            ];
+            const newFromLayout = this.registerUrls(jsFromLayout);
+            this.emitDownloadable(newFromLayout);
+            newInThisPass.push(...newFromLayout);
+            // Build a queue of unvisited page URLs to walk. Seed it with:
+            //   - unvisited page URLs from the input batch (anchor-derived URLs
+            //     registered by initialDiscovery, or pages from earlier passes
+            //     that haven't been visited yet)
+            //   - new page URLs discovered above (parseLayoutJs etc.)
+            // For every page URL visited we run script-tag extraction AND
+            // <a href> extraction, then enqueue any new page URLs that surfaces.
+            // visitedPageUrls breaks cycles.
+            const isPageUrl = (u) => {
+                let parsed;
+                try {
+                    parsed = new URL(u);
+                }
+                catch (_a) {
+                    return false;
+                }
+                return !parsed.pathname.endsWith(".js") && !parsed.pathname.endsWith(".js.map");
+            };
+            const pageQueue = [];
+            const enqueued = new Set(); // tracks full URLs to avoid duplicate entries per pass
+            const enqueueIfPage = (u) => {
+                if (this.visitedPageCount + pageQueue.length >= this.MAX_VISITED_PAGES)
+                    return;
+                if (enqueued.has(u))
+                    return; // exact URL already queued this pass
+                if (!isPageUrl(u))
+                    return;
+                enqueued.add(u);
+                pageQueue.push(u);
+            };
+            for (const u of jsUrls)
+                enqueueIfPage(u);
+            for (const u of newInThisPass)
+                enqueueIfPage(u);
+            for (const u of pageQueue) {
+                if (this.stopped)
+                    break; // honour stop() at each iteration boundary
+                if (this.visitedPageCount >= this.MAX_VISITED_PAGES) {
+                    console.error(chalk.yellow(`[!] Visited page limit reached (${this.MAX_VISITED_PAGES}). Skipping remaining page queue entries.`));
+                    break;
+                }
+                this.visitedPageCount++;
+                const normalized = this.normalizePageUrl(u);
+                // Fetch scripts first — used for both fingerprinting and URL registration.
+                const extra = yield next_getJSScript(u);
+                if (!extra || !Array.isArray(extra)) {
+                    console.error(`[NextJsCrawler] Invalid return value from next_getJSScript for URL: ${u}`);
+                    console.error(`[NextJsCrawler] Returned value:`, extra);
+                    process.exit(1);
+                }
+                // Content-entropy dedup: skip this variant if its script set matches
+                // a fingerprint already recorded for this pathname.
+                const fp = this.scriptFingerprint(extra);
+                const knownFPs = this.pageScriptFingerprints.get(normalized);
+                if (knownFPs === null || knownFPs === void 0 ? void 0 : knownFPs.has(fp))
+                    continue;
+                // New content for this pathname — record fingerprint and process.
+                if (!this.pageScriptFingerprints.has(normalized)) {
+                    this.pageScriptFingerprints.set(normalized, new Set());
+                }
+                this.pageScriptFingerprints.get(normalized).add(fp);
+                this.techniqueEfficiencyMapping["next_getJSScript"] = [
+                    ...(this.techniqueEfficiencyMapping["next_getJSScript"] || []),
+                    ...extra,
+                ];
+                const newFromScripts = this.registerUrls(extra);
+                this.emitDownloadable(newFromScripts);
+                newInThisPass.push(...newFromScripts);
+                for (const x of newFromScripts)
+                    enqueueIfPage(x);
+                // Harvest <a href> links from this page so we keep expanding
+                // the crawl frontier.
+                const morePaths = yield next_getClientSidePaths(u);
+                this.techniqueEfficiencyMapping["next_getClientSidePaths"] = [
+                    ...(this.techniqueEfficiencyMapping["next_getClientSidePaths"] || []),
+                    ...morePaths,
+                ];
+                const newFromAnchors = this.registerUrls(morePaths);
+                this.emitDownloadable(newFromAnchors);
+                newInThisPass.push(...newFromAnchors);
+                for (const x of newFromAnchors)
+                    enqueueIfPage(x);
+            }
+            return newInThisPass;
+        });
+    }
+    // ── public API ───────────────────────────────────────────────────────
+    /**
+     * Main entry-point.  Runs initial discovery, then iterates recursive
+     * passes until convergence (no new URLs) or the iteration cap.
+     *
+     * @returns The complete, deduplicated list of JS/asset URLs to download.
+     */
+    crawl() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Phase 1 – initial heavyweight discovery
+            yield this.initialDiscovery();
+            // Phase 2 – recursive lightweight passes
+            let currentBatch = [...this.discoveredUrls];
+            let iteration = 0;
+            while (iteration < this.MAX_ITERATIONS && !this.stopped) {
+                iteration++;
+                const sizeBefore = this.size;
+                console.log(chalk.cyan(`[i] Recursive crawl pass ${iteration} – ${sizeBefore} URLs known`));
+                const newUrls = yield this.recursivePass(currentBatch);
+                if (newUrls.length === 0) {
+                    console.log(chalk.green(`[✓] Recursive crawl converged after ${iteration} pass(es)`));
+                    break;
+                }
+                console.log(chalk.green(`[+] Pass ${iteration} discovered ${newUrls.length} new URL(s)`));
+                // Next pass only analyses the newly found URLs
+                currentBatch = newUrls;
+            }
+            if (this.stopped) {
+                console.error(chalk.yellow(`[!] Crawler stopped — downloading all discovered files`));
+            }
+            else if (iteration >= this.MAX_ITERATIONS) {
+                console.error(chalk.yellow(`[!] Reached max recursive crawl iterations (${this.MAX_ITERATIONS})`));
+            }
+            // Phase 3 – brute-force .map files on the final set (skip if stopped by timeout)
+            if (!this.stopped) {
+                const allJsUrls = [...this.discoveredUrls].filter((u) => u.endsWith(".js") || u.endsWith(".js.map"));
+                const jsFromBrute = yield next_bruteForceJsFiles(allJsUrls);
+                this.techniqueEfficiencyMapping["next_bruteForceJsFiles"] = jsFromBrute;
+                this.emitDownloadable(this.registerUrls(jsFromBrute));
+            }
+            // Only return downloadable assets. Anchor-derived page URLs live in
+            // discoveredUrls to drive the crawl, but must not reach downloadFiles.
+            return [...this.discoveredUrls].filter((u) => {
+                try {
+                    const p = new URL(u).pathname;
+                    return p.endsWith(".js") || p.endsWith(".js.map");
+                }
+                catch (_a) {
+                    return false;
+                }
+            });
+        });
+    }
+}
+export default NextJsCrawler;
+//# sourceMappingURL=NextJsCrawler.js.map

package/build/lazyLoad/next_js/NextJsCrawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NextJsCrawler.js","sourceRoot":"","sources":["../../../src/lazyLoad/next_js/NextJsCrawler.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,gBAAgB,MAAM,uBAAuB,CAAC;AACrD,OAAO,8BAA8B,MAAM,qCAAqC,CAAC;AACjF,OAAO,oCAAoC,MAAM,2CAA2C,CAAC;AAC7F,OAAO,kBAAkB,MAAM,8BAA8B,CAAC;AAC9D,OAAO,mBAAmB,MAAM,0BAA0B,CAAC;AAC3D,OAAO,kBAAkB,MAAM,yBAAyB,CAAC;AACzD,OAAO,iCAAiC,MAAM,wCAAwC,CAAC;AACvF,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,uBAAuB,MAAM,8BAA8B,CAAC;AAEnE,OAAO,KAAK,eAAe,MAAM,eAAe,CAAC;AAcjD;;;;;;GAMG;AACH,MAAM,aAAa;IAwCf,YAAY,OAA6B;QAdzC;;;;WAIG;QACc,sBAAiB,GAAG,GAAG,CAAC;QACjC,qBAAgB,GAAG,CAAC,CAAC;QAE7B,gEAAgE;QACxD,YAAO,GAAG,KAAK,CAAC;QAMpB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC;QAC7D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QAEjD,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC,sBAAsB,GAAG,IAAI,GAAG,EAAE,CAAC;QACxC,IAAI,CAAC,0BAA0B,GAAG,EAAE,CAAC;IACzC,CAAC;IAED,wEAAwE;IAExE;;;OAGG;IACK,gBAAgB,CAAC,CAAS;QAC9B,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1B,OAAO,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC3C,CAAC;QAAC,WAAM,CAAC;YACL,OAAO,CAAC,CAAC;QACb,CAAC;IACL,CAAC;IAED,8DAA8D;IACtD,iBAAiB,CAAC,OAAiB;QACvC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,iEAAiE;IAC1D,IAAI;QACP,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,6EAA6E;IACrE,YAAY,CAAC,IAAc;QAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,kFAAkF;IAC1E,gBAAgB,CAAC,IAAc;QACnC,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO;QACnC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACnC,IAAI,CAAC;gBACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC9B,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;YAAC,WAAM,CAAC;gBACL,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;IACrE,CAAC;IAED,yDAAyD;IACzD,IAAY,IAAI;QACZ,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;IACpC,CAAC;IAED,wEAAwE;IAExE;;;OAGG;IACW,gBAAgB;;YAC1B,qCAAqC;YACrC,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,0BAA0B,CAAC,kBAAkB,CAAC,GAAG;gBAClD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;gBAC9D,GAAG,eAAe;aACrB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC;YAC1D,qEAAqE;YACrE,sBAAsB;YACtB,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpD,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAEjG,gEAAgE;YAChE,+DAA+D;YAC/D,mEAAmE;YACnE,MAAM,gBAAgB,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjE,IAAI,CAAC,0BAA0B,CAAC,yBAAyB,CAAC,GAAG;gBACzD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC;gBACrE,GAAG,gBAAgB;aACtB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE3D,4EAA4E;YAC5E,gFAAgF;YAChF,gFAAgF;YAChF,qDAAqD;YACrD,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC/B,MAAM,aAAa,GAAG,MAAM,8BAA8B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrE,IAAI,CAAC,0BAA0B,CAAC,gCAAgC,CAAC,GAAG,aAAa,CAAC;gBAClF,eAAe,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC5C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC;YAC5D,CAAC;YAED,uBAAuB;YACvB,MAAM,mBAAmB,GAAG,MAAM,oCAAoC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjF,IAAI,CAAC,0BAA0B,CAAC,sCAAsC,CAAC,GAAG,mBAAmB,CAAC;YAC9F,eAAe,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;YAClD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAE9D,gEAAgE;YAChE,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,MAAM,gBAAgB,GAAG,MAAM,kBAAkB,CAC7C,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,EACX,eAAe,CAAC,SAAS,EAAE,CAC9B,CAAC;gBACF,IAAI,CAAC,0BAA0B,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC;gBAC9E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAEhE,MAAM,kBAAkB,GAAG,MAAM,iCAAiC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC5F,IAAI,CAAC,0BAA0B,CAAC,mCAAmC,CAAC,GAAG,kBAAkB,CAAC;gBAC1F,eAAe,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;gBACjD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACjE,CAAC;YAED,gDAAgD;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5E,CAAC;KAAA;IAED,wEAAwE;IAExE;;;;;;OAMG;IACW,aAAa,CAAC,MAAgB;;YACxC,IAAI,aAAa,GAAa,EAAE,CAAC;YAEjC,mDAAmD;YACnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,CAAC,GAAG;gBACrD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;gBACjE,GAAG,aAAa;aACnB,CAAC;YACF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;YAEtC,yEAAyE;YACzE,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChE,IAAI,CAAC,0BAA0B,CAAC,oBAAoB,CAAC,GAAG;gBACpD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;gBAChE,GAAG,YAAY;aAClB,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YACtD,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YACrC,aAAa,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YAErC,8DAA8D;YAC9D,oEAAoE;YACpE,mEAAmE;YACnE,qCAAqC;YACrC,0DAA0D;YAC1D,8DAA8D;YAC9D,qEAAqE;YACrE,iCAAiC;YACjC,MAAM,SAAS,GAAG,CAAC,CAAS,EAAW,EAAE;gBACrC,IAAI,MAAW,CAAC;gBAChB,IAAI,CAAC;oBACD,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;gBACxB,CAAC;gBAAC,WAAM,CAAC;oBACL,OAAO,KAAK,CAAC;gBACjB,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpF,CAAC,CAAC;YAEF,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,uDAAuD;YAC3F,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,EAAE;gBAChC,IAAI,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB;oBAAE,OAAO;gBAC/E,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,OAAO,CAAC,qCAAqC;gBAClE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;oBAAE,OAAO;gBAC1B,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC;YAEF,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,aAAa,CAAC,CAAC,CAAC,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,aAAa;gBAAE,aAAa,CAAC,CAAC,CAAC,CAAC;YAEhD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,OAAO;oBAAE,MAAM,CAAC,2CAA2C;gBAEpE,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAClD,OAAO,CAAC,KAAK,CACT,KAAK,CAAC,MAAM,CACR,mCAAmC,IAAI,CAAC,iBAAiB,2CAA2C,CACvG,CACJ,CAAC;oBACF,MAAM;gBACV,CAAC;gBACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAExB,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAE5C,2EAA2E;gBAC3E,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAExC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClC,OAAO,CAAC,KAAK,CAAC,uEAAuE,CAAC,EAAE,CAAC,CAAC;oBAC1F,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;oBACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,CAAC;gBAED,qEAAqE;gBACrE,oDAAoD;gBACpD,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC7D,IAAI,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,GAAG,CAAC,EAAE,CAAC;oBAAE,SAAS;gBAEhC,kEAAkE;gBAClE,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC/C,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBACD,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErD,IAAI,CAAC,0BAA0B,CAAC,kBAAkB,CAAC,GAAG;oBAClD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;oBAC9D,GAAG,KAAK;iBACX,CAAC;gBAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAChD,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;gBACtC,aAAa,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;gBACtC,KAAK,MAAM,CAAC,IAAI,cAAc;oBAAE,aAAa,CAAC,CAAC,CAAC,CAAC;gBAEjD,6DAA6D;gBAC7D,sBAAsB;gBACtB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,CAAC,CAAC,CAAC;gBACnD,IAAI,CAAC,0BAA0B,CAAC,yBAAyB,CAAC,GAAG;oBACzD,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC;oBACrE,GAAG,SAAS;iBACf,CAAC;gBACF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACpD,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;gBACtC,aAAa,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;gBACtC,KAAK,MAAM,CAAC,IAAI,cAAc;oBAAE,aAAa,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC;YAED,OAAO,aAAa,CAAC;QACzB,CAAC;KAAA;IAED,wEAAwE;IAExE;;;;;OAKG;IACG,KAAK;;YACP,0CAA0C;YAC1C,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAE9B,yCAAyC;YACzC,IAAI,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;YAC5C,IAAI,SAAS,GAAG,CAAC,CAAC;YAElB,OAAO,SAAS,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtD,SAAS,EAAE,CAAC;gBACZ,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;gBAE7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,SAAS,MAAM,UAAU,aAAa,CAAC,CAAC,CAAC;gBAE5F,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;gBAEvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uCAAuC,SAAS,WAAW,CAAC,CAAC,CAAC;oBACtF,MAAM;gBACV,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,SAAS,eAAe,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC;gBAE1F,+CAA+C;gBAC/C,YAAY,GAAG,OAAO,CAAC;YAC3B,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,wDAAwD,CAAC,CAAC,CAAC;YAC1F,CAAC;iBAAM,IAAI,SAAS,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,+CAA+C,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;YACvG,CAAC;YAED,iFAAiF;YACjF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChB,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrG,MAAM,WAAW,GAAG,MAAM,sBAAsB,CAAC,SAAS,CAAC,CAAC;gBAC5D,IAAI,CAAC,0BAA0B,CAAC,wBAAwB,CAAC,GAAG,WAAW,CAAC;gBACxE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,oEAAoE;YACpE,uEAAuE;YACvE,OAAO,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACzC,IAAI,CAAC;oBACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAC9B,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACtD,CAAC;gBAAC,WAAM,CAAC;oBACL,OAAO,KAAK,CAAC;gBACjB,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;CACJ;AAED,eAAe,aAAa,CAAC"}

package/build/lazyLoad/next_js/next_GetJSScript.js

@@ -0,0 +1,126 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+// lazyLoad/nextGetJSScript.js
+import chalk from "chalk";
+import { URL } from "url";
+import * as cheerio from "cheerio";
+import makeRequest from "../../utility/makeReq.js";
+import { getJsUrls } from "../globals.js";
+import { getCacheOnly } from "../../utility/globals.js";
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * Scrapes all lazy-loaded JavaScript file URLs from the provided Next.js page.
+ *
+ * The function fetches the HTML with `makeRequest`, parses `script` tags via `cheerio`,
+ * and normalizes discovered `src` attributes (absolute, relative, or inline chunk hints)
+ * into absolute URLs using the shared lazy-load URL registry.
+ *
+ * @param {string} url - The URL of the webpage to fetch and parse.
+ * @returns {Promise<string[]>} - A promise that resolves to an array of absolute URLs
+ * pointing to JavaScript files found in the page.
+ */
+const MAX_FETCH_ATTEMPTS = 5;
+const next_getJSScript = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    const toReturn = [];
+    // get the page source — bounded retry so an unreachable host can't hang the crawler
+    let res = null;
+    const maxAttempts = getCacheOnly() ? 1 : MAX_FETCH_ATTEMPTS;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        res = yield makeRequest(url, {});
+        if (res)
+            break;
+        if (attempt < maxAttempts) {
+            console.error(chalk.yellow(`[!] Request to ${url} failed, retrying (${attempt}/${maxAttempts})...`));
+            yield sleep(1000);
+        }
+    }
+    if (!res) {
+        if (!getCacheOnly()) {
+            console.error(chalk.red(`[!] Giving up on ${url} after ${maxAttempts} attempts`));
+        }
+        return toReturn;
+    }
+    const pageSource = yield res.text();
+    // cheerio to parse the page source
+    const $ = cheerio.load(pageSource);
+    // find all script tags
+    const scriptTags = $("script");
+    // iterate through script tags
+    for (const scriptTag of scriptTags) {
+        // get the src attribute
+        const src = $(scriptTag).attr("src");
+        // see if the src is a JS file
+        if (src !== undefined && src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)) {
+            // if the src starts with /, like `/static/js/a.js` find the absolute URL
+            if (src.startsWith("/")) {
+                const absoluteUrl = new URL(url).origin + src;
+                if (!getJsUrls().includes(absoluteUrl)) {
+                    toReturn.push(absoluteUrl);
+                }
+            }
+            else if (src.startsWith("http")) {
+                const urlObj = new URL(src);
+                const ext = urlObj.pathname.split(".").pop();
+                if (ext === "js") {
+                    if (!getJsUrls().includes(src)) {
+                        toReturn.push(src);
+                    }
+                }
+            }
+            else if (src.match(/^[^/]/)) {
+                // if the src is a relative URL, like `static/js/a.js` find the absolute URL
+                // Get directory URL (origin + path without filename)
+                const pathParts = new URL(url).pathname.split("/");
+                pathParts.pop(); // remove filename from last
+                const directory = new URL(url).origin + pathParts.join("/") + "/";
+                if (!getJsUrls().includes(directory + src)) {
+                    toReturn.push(directory + src);
+                }
+            }
+            else {
+                if (!getJsUrls().includes(src)) {
+                    toReturn.push(src);
+                }
+            }
+        }
+        else {
+            // if the script tag is inline, it could contain static JS URL
+            // to get these, simply regex from the JS script
+            const js_script = $(scriptTag).html();
+            const matches = js_script === null || js_script === void 0 ? void 0 : js_script.match(/static\/chunks\/[a-zA-Z0-9_\-~.]+\.js/g);
+            if (matches) {
+                const uniqueMatches = [...new Set(matches)];
+                for (const match of uniqueMatches) {
+                    // if it is using that static/chunks/ pattern, I can just get the filename
+                    const filename = match.replace("static/chunks/", "");
+                    // go through the already found URLs, coz they will have it (src attribute
+                    // is there before inline things)
+                    let js_path_dir;
+                    for (const js_url of getJsUrls()) {
+                        if (!js_path_dir &&
+                            new URL(js_url).host === new URL(url).host &&
+                            new URL(js_url).pathname.includes("static/chunks/")) {
+                            js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
+                        }
+                    }
+                    if (js_path_dir) {
+                        // Ensure js_path_dir was found
+                        toReturn.push(js_path_dir + "/" + filename);
+                    }
+                }
+            }
+        }
+    }
+    // causes too much noise after using it with subsequent requests
+    // console.log(chalk.green(`[✓] Found ${getJsUrls().length} JS files from the script tags`));
+    return toReturn;
+});
+export default next_getJSScript;
+//# sourceMappingURL=next_GetJSScript.js.map

package/build/lazyLoad/next_js/next_GetJSScript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"next_GetJSScript.js","sourceRoot":"","sources":["../../../src/lazyLoad/next_js/next_GetJSScript.ts"],"names":[],"mappings":";;;;;;;;;AAAA,8BAA8B;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF;;;;;;;;;;GAUG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,MAAM,gBAAgB,GAAG,CAAO,GAAW,EAAqB,EAAE;IAC9D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,oFAAoF;IACpF,IAAI,GAAG,GAAoB,IAAI,CAAC;IAChC,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC5D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACtD,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,IAAI,GAAG;YAAE,MAAM;QACf,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,kBAAkB,GAAG,sBAAsB,OAAO,IAAI,WAAW,MAAM,CAAC,CAAC,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IACD,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,GAAG,UAAU,WAAW,WAAW,CAAC,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAEpC,mCAAmC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEnC,uBAAuB;IACvB,MAAM,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE/B,8BAA8B;IAC9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,wBAAwB;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAErC,8BAA8B;QAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,CAAC,0DAA0D,CAAC,EAAE,CAAC;YAC7F,yEAAyE;YACzE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC9C,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBACrC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;gBAC7C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;oBACf,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACL,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,4EAA4E;gBAC5E,qDAAqD;gBACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnD,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,4BAA4B;gBAC7C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAElE,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;oBACzC,QAAQ,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;YACL,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,8DAA8D;YAC9D,gDAAgD;YAEhD,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAE3E,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC5C,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChC,0EAA0E;oBAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;oBAErD,0EAA0E;oBAC1E,iCAAiC;oBAEjC,IAAI,WAAW,CAAC;oBAEhB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,EAAE,CAAC;wBAC/B,IACI,CAAC,WAAW;4BACZ,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI;4BAC1C,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACrD,CAAC;4BACC,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;wBACxD,CAAC;oBACL,CAAC;oBACD,IAAI,WAAW,EAAE,CAAC;wBACd,+BAA+B;wBAC/B,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC;oBAChD,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,gEAAgE;IAChE,6FAA6F;IAE7F,OAAO,QAAQ,CAAC;AACpB,CAAC,CAAA,CAAC;AAEF,eAAe,gBAAgB,CAAC"}

package/build/lazyLoad/next_js/next_GetLazyResourcesBuildManifestJs.js

@@ -0,0 +1,83 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import chalk from "chalk";
+import * as globals from "../globals.js";
+import makeRequest from "../../utility/makeReq.js";
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+const traverse = _traverse.default;
+/**
+ * Finds lazy-loaded JavaScript files from Next.js `_buildManifest.js`.
+ *
+ * Given a URL, this function finds the `_buildManifest.js` file in the given URL or file containing URLs.
+ * It then parses the file with the Babel parser and extracts all string literals from the AST.
+ * Finally, it iterates over the strings and finds any chunks (i.e., URLs containing "static/chunks/").
+ * The function returns an array of absolute URLs pointing to the lazy-loaded JS files found.
+ *
+ * @param {string} url - The URL or path to a file containing a list of URLs to process.
+ * @returns {Promise<string[] | any>} - A promise that resolves to an array of absolute URLs pointing to lazy-loaded JS files found, or undefined for invalid URL.
+ */
+const next_getLazyResourcesBuildManifestJs = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a;
+    // get the JS URLs
+    const foundUrls = globals.getJsUrls();
+    let toReturn = [];
+    let buildManifestUrl = "";
+    // iterate over them, and find the build manifest
+    for (const jsUrl of foundUrls) {
+        if (jsUrl.endsWith("_buildManifest.js")) {
+            buildManifestUrl = jsUrl;
+            break;
+        }
+    }
+    if (buildManifestUrl === "") {
+        return [];
+    }
+    // get the contents of that
+    try {
+        const response = yield makeRequest(buildManifestUrl, {});
+        if (!response)
+            return [];
+        let buildManifestContent = yield response.text();
+        // parse it with babel parser
+        const ast = parser.parse(buildManifestContent, {
+            sourceType: "unambiguous",
+            plugins: ["jsx", "typescript"],
+            errorRecovery: true,
+        });
+        let strings = [];
+        traverse(ast, {
+            StringLiteral(path) {
+                strings.push(path.node.value);
+            },
+        });
+        // iterate over the strings, and find the chunks
+        for (const stringTxt of strings) {
+            if (stringTxt.includes("static/chunks/")) {
+                // a chunk is found
+                // bui;d the relative URL
+                const foundUrl = new URL(`../../${stringTxt}`, buildManifestUrl).href;
+                globals.pushToJsUrls(foundUrl);
+                toReturn.push(foundUrl);
+            }
+        }
+        if (toReturn.length > 0) {
+            console.log(chalk.green(`[✓] Found ${toReturn.length} JS files from _buildManifest.js`));
+        }
+    }
+    catch (err) {
+        if ((err === null || err === void 0 ? void 0 : err.code) !== "ENOENT") {
+            console.error(chalk.red(`[!] Failed to parse _buildManifest.js: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`));
+        }
+    }
+    return toReturn;
+});
+export default next_getLazyResourcesBuildManifestJs;
+//# sourceMappingURL=next_GetLazyResourcesBuildManifestJs.js.map

package/build/lazyLoad/next_js/next_GetLazyResourcesBuildManifestJs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"next_GetLazyResourcesBuildManifestJs.js","sourceRoot":"","sources":["../../../src/lazyLoad/next_js/next_GetLazyResourcesBuildManifestJs.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,eAAe,CAAC;AACzC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC;AAEnC;;;;;;;;;;GAUG;AACH,MAAM,oCAAoC,GAAG,CAAO,GAAW,EAA2B,EAAE;;IACxF,kBAAkB;IAClB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IACtC,IAAI,QAAQ,GAAa,EAAE,CAAC;IAE5B,IAAI,gBAAgB,GAAW,EAAE,CAAC;IAClC,iDAAiD;IACjD,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACtC,gBAAgB,GAAG,KAAK,CAAC;YACzB,MAAM;QACV,CAAC;IACL,CAAC;IAED,IAAI,gBAAgB,KAAK,EAAE,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAEzB,IAAI,oBAAoB,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEjD,6BAA6B;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE;YAC3C,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC;YAC9B,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,OAAO,GAAa,EAAE,CAAC;QAE3B,QAAQ,CAAC,GAAG,EAAE;YACV,aAAa,CAAC,IAAI;gBACd,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;SACJ,CAAC,CAAC;QAEH,gDAAgD;QAEhD,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACvC,mBAAmB;gBACnB,yBAAyB;gBACzB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,SAAS,EAAE,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC;gBACtE,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAC/B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,kCAAkC,CAAC,CAAC,CAAC;QAC7F,CAAC;IACL,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,mCAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC,CAAA,CAAC;AAEF,eAAe,oCAAoC,CAAC"}