@js-recon/js-recon 1.4.4 → 2.0.1-alpha.2

package/build/lazyLoad/generic/generic_extractLinks.js

@@ -0,0 +1,79 @@
+import * as cheerio from "cheerio";
+const resolvePageUrl = (rawValue, baseUrl) => {
+    if (!rawValue)
+        return null;
+    const trimmed = rawValue.trim();
+    if (!trimmed || trimmed.startsWith("#"))
+        return null;
+    if (/^(mailto|tel|javascript|data|blob):/i.test(trimmed))
+        return null;
+    let resolved;
+    try {
+        resolved = new URL(trimmed, baseUrl);
+    }
+    catch (_a) {
+        return null;
+    }
+    if (resolved.protocol !== "http:" && resolved.protocol !== "https:")
+        return null;
+    resolved.hash = "";
+    return resolved.href;
+};
+// Matches an absolute http(s) URL embedded anywhere in a string — e.g. inside an
+// onclick handler's JS: onclick="return powerpress_pinw('https://site/?pinw=123')".
+// Deliberately not tied to a specific function name (window.open, a plugin helper,
+// etc.) — popup/widget links are triggered through all sorts of call shapes, but the
+// URL argument itself is always a plain quoted absolute URL, which this catches
+// regardless of what function it's passed to.
+const EMBEDDED_URL_RE = /https?:\/\/[^\s'"<>)]+/g;
+/**
+ * Extracts every absolute http(s) URL substring embedded in a string value (typically
+ * an onclick handler or other inline JS attribute). See EMBEDDED_URL_RE for why this
+ * isn't tied to a specific JS call pattern like window.open(...).
+ */
+export const extractEmbeddedUrls = (attrValue) => [...attrValue.matchAll(EMBEDDED_URL_RE)].map((m) => m[0]);
+/**
+ * Extracts navigable page URLs from <a href>/<iframe src> tags and absolute URLs
+ * embedded in any other attribute value (typically an onclick handler) — not JS
+ * files. Used to recursively discover more HTML pages to scan for JS, per
+ * internal#66. <iframe src> and onclick-embedded URLs are included alongside
+ * <a href> because embedded widgets (a podcast player, a survey, a booking form) are
+ * commonly reachable only via an iframe or a JS-driven popup, never a plain link —
+ * WordPress's PowerPress podcast plugin is one real example: its "pop out player"
+ * link is `onclick="return powerpress_pinw('https://site/?powerpress_pinw=123-podcast')"`,
+ * invisible to href-only or iframe-only crawling. Filters out fragment-only links,
+ * non-http(s) schemes (mailto/tel/javascript/data/blob), and strips the hash so
+ * #section variants of the same page dedupe to one entry.
+ */
+export const extractPageLinks = (html, baseUrl) => {
+    const $ = cheerio.load(html);
+    const links = new Set();
+    $("a[href], iframe[src]").each((_, elem) => {
+        var _a;
+        const rawValue = (_a = $(elem).attr("href")) !== null && _a !== void 0 ? _a : $(elem).attr("src");
+        const resolved = resolvePageUrl(rawValue, baseUrl);
+        if (resolved)
+            links.add(resolved);
+    });
+    $("*").each((_, elem) => {
+        const attribs = elem.attribs;
+        if (!attribs)
+            return;
+        for (const [attrName, value] of Object.entries(attribs)) {
+            // href/src already handled above via cheerio's typed accessors; re-scanning
+            // them here would just re-add the same URL, which the Set dedupes anyway,
+            // but skipping is cheaper and avoids false "embedded URL" matches inside a
+            // long querystring value that itself contains another URL as a parameter.
+            if (attrName === "href" || attrName === "src" || !value)
+                continue;
+            for (const embeddedUrl of extractEmbeddedUrls(value)) {
+                const resolved = resolvePageUrl(embeddedUrl, baseUrl);
+                if (resolved)
+                    links.add(resolved);
+            }
+        }
+    });
+    return [...links];
+};
+export default extractPageLinks;
+//# sourceMappingURL=generic_extractLinks.js.map

package/build/lazyLoad/generic/generic_extractLinks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_extractLinks.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_extractLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG,CAAC,QAA4B,EAAE,OAAe,EAAiB,EAAE;IACpF,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,sCAAsC,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtE,IAAI,QAAa,CAAC;IAClB,IAAI,CAAC;QACD,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjF,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,CAAC;AACzB,CAAC,CAAC;AAEF,iFAAiF;AACjF,oFAAoF;AACpF,mFAAmF;AACnF,qFAAqF;AACrF,gFAAgF;AAChF,8CAA8C;AAC9C,MAAM,eAAe,GAAG,yBAAyB,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,SAAiB,EAAY,EAAE,CAC/D,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE9D;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAE,OAAe,EAAY,EAAE;IACxE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,CAAC,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;;QACvC,MAAM,QAAQ,GAAG,MAAA,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,QAAQ;YAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,OAAO,GAAI,IAAY,CAAC,OAA6C,CAAC;QAC5E,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACtD,4EAA4E;YAC5E,0EAA0E;YAC1E,2EAA2E;YAC3E,0EAA0E;YAC1E,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,KAAK;gBAAE,SAAS;YAClE,KAAK,MAAM,WAAW,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBACtD,IAAI,QAAQ;oBAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,eAAe,gBAAgB,CAAC"}

package/build/lazyLoad/generic/generic_getScriptTags.js

@@ -0,0 +1,152 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import makeRequest from "../../utility/makeReq.js";
+import * as cheerio from "cheerio";
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import * as lazyLoadGlobals from "../globals.js";
+import { printMsg, MSG } from "../../utility/printMsg.js";
+// <script type="..."> values known to hold non-JS content. Browsers never execute
+// these as script — WordPress/plugins routinely stash JSON-LD structured data or
+// browser resource-hint configs in a bare <script> tag with no src, and saving
+// that content as a .js file breaks any JS parser later run against it (Babel
+// can't parse a top-level JSON object as a JS Program).
+const NON_JS_SCRIPT_TYPES = [
+    "application/ld+json",
+    "application/json",
+    "speculationrules",
+    "importmap",
+    "text/template",
+    "text/x-template",
+    "text/x-handlebars-template",
+    "text/x-jquery-tmpl",
+    "text/ng-template",
+    "text/html",
+];
+/**
+ * Whether a <script type="..."> value indicates the tag's content should be
+ * treated as JS. Missing/empty type defaults to JS per the HTML spec. Deliberately
+ * permissive for unrecognized types (e.g. Cloudflare Rocket Loader rewrites
+ * type="text/javascript" to a randomized "<hash>-text/javascript" to defer
+ * execution — still real JS) — only the known non-JS types above are excluded.
+ */
+export const isLikelyJsScriptType = (type) => {
+    if (!type)
+        return true;
+    const normalized = type.trim().toLowerCase();
+    if (!normalized)
+        return true;
+    if (normalized.includes("javascript") || normalized === "module" || normalized.includes("ecmascript")) {
+        return true;
+    }
+    return !NON_JS_SCRIPT_TYPES.some((t) => normalized.includes(t));
+};
+/**
+ * Decodes a data: URI script src (e.g. `data:text/javascript;base64,...`) into
+ * its JS source text. Returns null if the URI doesn't match the expected shape
+ * or the payload can't be decoded.
+ */
+export const decodeDataUriScript = (dataUri) => {
+    const match = /^data:([^,]*),([\s\S]*)$/.exec(dataUri);
+    if (!match)
+        return null;
+    const [, meta, payload] = match;
+    const isBase64 = /;base64$/i.test(meta);
+    try {
+        return isBase64 ? Buffer.from(payload, "base64").toString("utf-8") : decodeURIComponent(payload);
+    }
+    catch (_a) {
+        return null;
+    }
+};
+/**
+ * Extracts JS chunk URLs from <script src> and <link rel="modulepreload"> tags,
+ * and saves inline <script> bodies (inline text and decoded data: URIs) to disk.
+ * Every saved file carries a `// File Source: <url>` header — this is what lets
+ * generic_stringsDiscovery.ts later resolve string-literal JS paths found inside
+ * these files against the URL they actually came from. Mirrors react_getScriptTags.ts
+ * — the same seeding pattern applies to any site regardless of framework.
+ */
+const generic_getScriptTags = (url, maxJsSizeMb, outputDir) => __awaiter(void 0, void 0, void 0, function* () {
+    let toReturn = [];
+    const req = yield makeRequest(url);
+    const pageSource = req ? yield req.text() : "";
+    const $ = cheerio.load(pageSource);
+    const host = new URL(url).host.replace(":", "_");
+    // Mirrors the page's own path, not just its host — so inline scripts from
+    // different pages of a multi-page generic crawl (internal#66) land in
+    // separate directories instead of colliding on inline-0.js, inline-1.js, etc.
+    const pageDir = new URL(url).pathname;
+    let inlineIndex = 0;
+    $("script").each((_, elem) => {
+        const src = $(elem).attr("src");
+        if (src) {
+            let resolved;
+            try {
+                resolved = new URL(src, url);
+            }
+            catch (_a) {
+                return;
+            }
+            // A data: URI script src embeds the JS source directly — it isn't a
+            // network-fetchable URL, so pushing it to toReturn would make the
+            // downloader try (and fail) to request/save it as a file.
+            if (resolved.protocol === "data:") {
+                if (!outputDir)
+                    return;
+                const content = decodeDataUriScript(resolved.href);
+                if (!content)
+                    return;
+                const hostDir = path.join(outputDir, host, pageDir);
+                fs.mkdirSync(hostDir, { recursive: true });
+                const filename = `inline-${inlineIndex++}.js`;
+                const filePath = path.join(hostDir, filename);
+                fs.writeFileSync(filePath, `// File Source: ${url} (data URI script #${inlineIndex - 1})\n${content}`);
+                lazyLoadGlobals.recordJsFileHash(crypto.createHash("sha256").update(content).digest("hex"));
+                printMsg(MSG.Run, `[✓] Saved data URI script to ${filePath}`);
+                return;
+            }
+            // blob:, javascript:, etc — not something we can fetch over the network either.
+            if (resolved.protocol !== "http:" && resolved.protocol !== "https:")
+                return;
+            toReturn.push(resolved.href);
+        }
+        else if (outputDir) {
+            if (!isLikelyJsScriptType($(elem).attr("type")))
+                return;
+            const content = $(elem).text().trim();
+            if (!content)
+                return;
+            const hostDir = path.join(outputDir, host, pageDir);
+            fs.mkdirSync(hostDir, { recursive: true });
+            const filename = `inline-${inlineIndex++}.js`;
+            const filePath = path.join(hostDir, filename);
+            fs.writeFileSync(filePath, `// File Source: ${url} (inline script #${inlineIndex - 1})\n${content}`);
+            lazyLoadGlobals.recordJsFileHash(crypto.createHash("sha256").update(content).digest("hex"));
+            printMsg(MSG.Run, `[✓] Saved inline script to ${filePath}`);
+        }
+    });
+    $("link[rel='modulepreload']").each((_, elem) => {
+        const href = $(elem).attr("href");
+        if (href) {
+            try {
+                toReturn.push(new URL(href, url).href);
+            }
+            catch (_a) {
+                /* not a resolvable URL — skip */
+            }
+        }
+    });
+    toReturn = [...new Set(toReturn)];
+    return { urls: toReturn, pageSource };
+});
+export default generic_getScriptTags;
+//# sourceMappingURL=generic_getScriptTags.js.map

package/build/lazyLoad/generic/generic_getScriptTags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_getScriptTags.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_getScriptTags.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,eAAe,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAE1D,kFAAkF;AAClF,iFAAiF;AACjF,+EAA+E;AAC/E,8EAA8E;AAC9E,wDAAwD;AACxD,MAAM,mBAAmB,GAAG;IACxB,qBAAqB;IACrB,kBAAkB;IAClB,kBAAkB;IAClB,WAAW;IACX,eAAe;IACf,iBAAiB;IACjB,4BAA4B;IAC5B,oBAAoB;IACpB,kBAAkB;IAClB,WAAW;CACd,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAwB,EAAW,EAAE;IACtE,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAC7B,IAAI,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACpG,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,OAAe,EAAiB,EAAE;IAClE,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC;QACD,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACrG,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,qBAAqB,GAAG,CAC1B,GAAW,EACX,WAAmB,EACnB,SAAkB,EAC6B,EAAE;IACjD,IAAI,QAAQ,GAAa,EAAE,CAAC;IAE5B,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACjD,0EAA0E;IAC1E,sEAAsE;IACtE,8EAA8E;IAC9E,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACzB,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,GAAG,EAAE,CAAC;YACN,IAAI,QAAa,CAAC;YAClB,IAAI,CAAC;gBACD,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjC,CAAC;YAAC,WAAM,CAAC;gBACL,OAAO;YACX,CAAC;YAED,oEAAoE;YACpE,kEAAkE;YAClE,0DAA0D;YAC1D,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAChC,IAAI,CAAC,SAAS;oBAAE,OAAO;gBACvB,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACnD,IAAI,CAAC,OAAO;oBAAE,OAAO;gBAErB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBACpD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3C,MAAM,QAAQ,GAAG,UAAU,WAAW,EAAE,KAAK,CAAC;gBAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC9C,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,sBAAsB,WAAW,GAAG,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC;gBACvG,eAAe,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC5F,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,gCAAgC,QAAQ,EAAE,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YAED,gFAAgF;YAChF,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YAE5E,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAAE,OAAO;YAExD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAErB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,UAAU,WAAW,EAAE,KAAK,CAAC;YAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC9C,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,oBAAoB,WAAW,GAAG,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC;YACrG,eAAe,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,8BAA8B,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAC5C,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;YAAC,WAAM,CAAC;gBACL,iCAAiC;YACrC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;AAC1C,CAAC,CAAA,CAAC;AAEF,eAAe,qBAAqB,CAAC"}

package/build/lazyLoad/generic/generic_importMapDiscovery.js

@@ -0,0 +1,87 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from "fs";
+import path from "path";
+import { confirmJsContentType } from "./generic_scanAttributesForJs.js";
+/**
+ * Parses a downloaded file's content for a Webpack Module Federation "import map"
+ * manifest shape:
+ *   { "microFrontends": { "<scope>/<name>": [{ "url": "https://.../remote-entry.js", ... }] } }
+ * and returns every remote-entry-style URL it lists. Tolerates the
+ * `// File Source: <url>\n` header `generic_downloadFiles.ts` prefixes onto every
+ * downloaded file — these manifests are typically referenced via a literal
+ * `<script src="...prod.json">` tag (picked up like any other script by
+ * `generic_getScriptTags.ts` regardless of extension) and saved under a synthesized
+ * `.js` filename despite being JSON (see `generic_downloadFiles.ts::synthesizeFilename`),
+ * so the header must be stripped before `JSON.parse`.
+ */
+export const extractMicroFrontendUrls = (content) => {
+    const withoutHeader = content.replace(/^\/\/ File Source: \S+\n/, "");
+    let data;
+    try {
+        data = JSON.parse(withoutHeader);
+    }
+    catch (_a) {
+        return [];
+    }
+    if (typeof data !== "object" || data === null || !("microFrontends" in data))
+        return [];
+    const microFrontends = data.microFrontends;
+    if (typeof microFrontends !== "object" || microFrontends === null)
+        return [];
+    const urls = new Set();
+    for (const entries of Object.values(microFrontends)) {
+        if (!Array.isArray(entries))
+            continue;
+        for (const entry of entries) {
+            const url = entry === null || entry === void 0 ? void 0 : entry.url;
+            if (typeof url === "string" && /^https?:\/\//i.test(url) && url.toLowerCase().endsWith(".js")) {
+                urls.add(url);
+            }
+        }
+    }
+    return [...urls];
+};
+/**
+ * Scans every file `generic` tech has already downloaded under outputDir for a
+ * micro-frontend import-map manifest (see `extractMicroFrontendUrls`) and confirms each
+ * referenced remote-entry URL via Content-Type before returning it as a new download
+ * candidate. Internal#74: js-recon was already downloading these manifests (picked up
+ * like any other `<script src>`) but never reading them — every remote a
+ * module-federation app can dynamically load is listed here, including remotes whose
+ * components never rendered (and so never fired a request) during the crawl.
+ */
+const generic_importMapDiscovery = (outputDir_1, ...args_1) => __awaiter(void 0, [outputDir_1, ...args_1], void 0, function* (outputDir, alreadyKnownUrls = new Set(), threads = 1) {
+    if (!fs.existsSync(outputDir))
+        return [];
+    const files = fs.readdirSync(outputDir, { recursive: true, encoding: "utf8" });
+    const candidates = new Set();
+    for (const relPath of files) {
+        if (!/\.(js|mjs|json)$/i.test(relPath))
+            continue;
+        const filePath = path.join(outputDir, relPath);
+        let content;
+        try {
+            content = fs.readFileSync(filePath, "utf-8");
+        }
+        catch (_a) {
+            continue;
+        }
+        for (const url of extractMicroFrontendUrls(content)) {
+            if (!alreadyKnownUrls.has(url))
+                candidates.add(url);
+        }
+    }
+    if (candidates.size === 0)
+        return [];
+    return confirmJsContentType([...candidates], threads);
+});
+export default generic_importMapDiscovery;
+//# sourceMappingURL=generic_importMapDiscovery.js.map

package/build/lazyLoad/generic/generic_importMapDiscovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_importMapDiscovery.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_importMapDiscovery.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AAExE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,OAAe,EAAY,EAAE;IAClE,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;IAEtE,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACrC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,EAAE,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACxF,MAAM,cAAc,GAAI,IAAgC,CAAC,cAAc,CAAC;IACxE,IAAI,OAAO,cAAc,KAAK,QAAQ,IAAI,cAAc,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAE7E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,cAAyC,CAAC,EAAE,CAAC;QAC7E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QACtC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAI,KAAwC,aAAxC,KAAK,uBAAL,KAAK,CAAqC,GAAG,CAAC;YAC3D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5F,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,0BAA0B,GAAG,yBAId,EAAE,gEAHnB,SAAiB,EACjB,mBAAgC,IAAI,GAAG,EAAE,EACzC,UAAkB,CAAC;IAEnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAEzC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAa,CAAC;IAC3F,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAErC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,SAAS;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACD,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;QAAC,WAAM,CAAC;YACL,SAAS;QACb,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,wBAAwB,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;IACL,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,oBAAoB,CAAC,CAAC,GAAG,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC,CAAA,CAAC;AAEF,eAAe,0BAA0B,CAAC"}

package/build/lazyLoad/generic/generic_resolveScope.js

@@ -0,0 +1,66 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { printMsg, MSG } from "../../utility/printMsg.js";
+import { customHeadersToRecord } from "../../utility/customHeaders.js";
+import * as globals from "../../utility/globals.js";
+/**
+ * Extracts the host (scheme-independent, matches the format used throughout
+ * `lazyLoad/globals.ts`'s scope list) from a URL.
+ */
+export const hostFromUrl = (url) => new URL(url).host;
+/**
+ * Manually follows HTTP redirects starting at startUrl, up to maxRedirects hops,
+ * and returns the final URL reached. Uses redirect: "manual" (rather than the
+ * platform default of automatically following) so the hop count is actually
+ * enforced by this function instead of an opaque undici-internal limit.
+ */
+export const resolveRedirectChain = (startUrl_1, maxRedirects_1, ...args_1) => __awaiter(void 0, [startUrl_1, maxRedirects_1, ...args_1], void 0, function* (startUrl, maxRedirects, fetchImpl = fetch) {
+    let currentUrl = startUrl;
+    for (let i = 0; i < maxRedirects; i++) {
+        let res;
+        try {
+            res = yield fetchImpl(currentUrl, {
+                redirect: "manual",
+                headers: customHeadersToRecord(globals.getCustomHeaders()),
+            });
+        }
+        catch (_a) {
+            return currentUrl;
+        }
+        if (res.status >= 300 && res.status < 400) {
+            const location = res.headers.get("location");
+            if (!location)
+                return currentUrl;
+            try {
+                currentUrl = new URL(location, currentUrl).href;
+            }
+            catch (_b) {
+                return currentUrl;
+            }
+            continue;
+        }
+        return currentUrl;
+    }
+    return currentUrl;
+});
+/**
+ * Default scope for the `generic` tech: fetch the target URL, follow redirects
+ * (capped at maxRedirects), and scope the crawl to the host of the final
+ * destination. Without this, an unscoped generic crawl would follow every
+ * in-page link across the entire web.
+ */
+const resolveGenericScope = (url, maxRedirects) => __awaiter(void 0, void 0, void 0, function* () {
+    const finalUrl = yield resolveRedirectChain(url, maxRedirects);
+    const host = hostFromUrl(finalUrl);
+    printMsg(MSG.Header, `[i] Generic crawl scope resolved to: ${host}`);
+    return [host];
+});
+export default resolveGenericScope;
+//# sourceMappingURL=generic_resolveScope.js.map

package/build/lazyLoad/generic/generic_resolveScope.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_resolveScope.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_resolveScope.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,0BAA0B,CAAC;AAEpD;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,wCAIjB,EAAE,+EAHjB,QAAgB,EAChB,YAAoB,EACpB,YAA0B,KAAK;IAE/B,IAAI,UAAU,GAAG,QAAQ,CAAC;IAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACD,GAAG,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE;gBAC9B,QAAQ,EAAE,QAAQ;gBAClB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;aAC7D,CAAC,CAAC;QACP,CAAC;QAAC,WAAM,CAAC;YACL,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI,CAAC,QAAQ;gBAAE,OAAO,UAAU,CAAC;YACjC,IAAI,CAAC;gBACD,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;YACpD,CAAC;YAAC,WAAM,CAAC;gBACL,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,SAAS;QACb,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,OAAO,UAAU,CAAC;AACtB,CAAC,CAAA,CAAC;AAEF;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,CAAO,GAAW,EAAE,YAAoB,EAAqB,EAAE;IACvF,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,wCAAwC,IAAI,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,IAAI,CAAC,CAAC;AAClB,CAAC,CAAA,CAAC;AAEF,eAAe,mBAAmB,CAAC"}

package/build/lazyLoad/generic/generic_scanAttributesForJs.js

@@ -0,0 +1,122 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as cheerio from "cheerio";
+import makeRequest from "../../utility/makeReq.js";
+import { isJsContentType } from "./generic_jsMimeTypes.js";
+import { printMsg, MSG } from "../../utility/printMsg.js";
+/**
+ * Resolves value against baseUrl and returns the href if it's a network-fetchable
+ * (http/https) URL with a path segment ending in ".js" — this is what catches assets
+ * served like ".../beacon.min.js/v124/token" (a ".js"-suffixed segment that isn't the
+ * last one, so a plain endsWith(".js") check on the whole path would miss it). Returns
+ * null for anything unparseable or non-http(s) (data:/blob:/javascript: etc — these
+ * aren't requests confirmJsContentType could ever meaningfully make).
+ */
+export const resolveJsPathCandidate = (value, baseUrl) => {
+    if (!value)
+        return null;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return null;
+    // A fragment-only ("#selector") or query-only ("?x=1") value resolves against
+    // baseUrl by inheriting its pathname completely unchanged. When baseUrl is itself
+    // a .js file (e.g. generic_stringsDiscovery.ts resolving strings found INSIDE an
+    // already-downloaded JS file), a value that isn't a path reference at all — a CSS
+    // selector string embedded in a Vue bundle's scoped styles, say — would otherwise
+    // falsely look like a legitimate JS-path candidate purely because it inherited the
+    // base's own ".js"-ending pathname.
+    if (trimmed.startsWith("#") || trimmed.startsWith("?"))
+        return null;
+    let parsed;
+    try {
+        parsed = new URL(trimmed, baseUrl);
+    }
+    catch (_a) {
+        return null;
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+        return null;
+    const segments = parsed.pathname.split("/");
+    if (!segments.some((seg) => seg.toLowerCase().endsWith(".js")))
+        return null;
+    return parsed.href;
+};
+/**
+ * Pure candidate extraction: walks every element attribute in the given HTML and
+ * keeps the ones that resolve to a JS-looking path (see resolveJsPathCandidate).
+ */
+export const findJsPathSegmentCandidates = (html, baseUrl) => {
+    const $ = cheerio.load(html);
+    const candidates = new Set();
+    $("*").each((_, elem) => {
+        const attribs = elem.attribs;
+        if (!attribs)
+            return;
+        for (const value of Object.values(attribs)) {
+            const resolved = resolveJsPathCandidate(value, baseUrl);
+            if (resolved)
+                candidates.add(resolved);
+        }
+    });
+    return [...candidates];
+};
+/**
+ * Confirms each candidate is actually JavaScript via Content-Type (per RFC 9239 /
+ * RFC 4329 and legacy variants — see generic_jsMimeTypes.ts), not just the URL shape.
+ * Uses GET rather than HEAD: makeRequest's response cache is keyed on URL only (not
+ * method), so a cached HEAD response would poison the later GET that downloads the
+ * body — every other crawler in this codebase uses GET for the same reason.
+ *
+ * `concurrency` (default 1, matching every other generic-tech call site's default
+ * single-threaded behavior) controls how many candidates are in flight at once — a
+ * worker-pool over the candidate list, same shape as generic_downloadFiles.ts's
+ * `processOne`/`worker` pattern. Internal#75: a webpack chunk-hash-map can list far
+ * more candidates than a string-literal scan ever would (every chunk ID an entry
+ * chunk knows about, not just ones referenced by a literal path), and each failing
+ * candidate costs up to `10 retries * requestTimeout` in makeRequest — sequential
+ * confirmation of a large candidate batch, with even a handful of dead chunk IDs in
+ * it, becomes prohibitively slow. Passing a caller-supplied `threads` value here
+ * (already threaded through generic tech for downloads) keeps that path proportional.
+ */
+export const confirmJsContentType = (candidates_1, ...args_1) => __awaiter(void 0, [candidates_1, ...args_1], void 0, function* (candidates, concurrency = 1) {
+    const confirmed = [];
+    let cursor = 0;
+    const worker = () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        while (true) {
+            const idx = cursor++;
+            if (idx >= candidates.length)
+                break;
+            const url = candidates[idx];
+            let contentType = null;
+            try {
+                const res = yield makeRequest(url, { method: "GET" });
+                contentType = (_a = res === null || res === void 0 ? void 0 : res.headers.get("content-type")) !== null && _a !== void 0 ? _a : null;
+            }
+            catch (_b) {
+                continue;
+            }
+            if (isJsContentType(contentType)) {
+                printMsg(MSG.Run, `[✓] Confirmed JS via Content-Type: ${url}`);
+                confirmed.push(url);
+            }
+        }
+    });
+    yield Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+    return confirmed;
+});
+const generic_scanAttributesForJs = (html_1, baseUrl_1, ...args_1) => __awaiter(void 0, [html_1, baseUrl_1, ...args_1], void 0, function* (html, baseUrl, alreadyKnownUrls = new Set()) {
+    const candidates = findJsPathSegmentCandidates(html, baseUrl).filter((u) => !alreadyKnownUrls.has(u));
+    if (candidates.length === 0)
+        return [];
+    return confirmJsContentType(candidates);
+});
+export default generic_scanAttributesForJs;
+//# sourceMappingURL=generic_scanAttributesForJs.js.map

package/build/lazyLoad/generic/generic_scanAttributesForJs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic_scanAttributesForJs.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_scanAttributesForJs.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAE1D;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,KAAa,EAAE,OAAe,EAAiB,EAAE;IACpF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,8EAA8E;IAC9E,kFAAkF;IAClF,iFAAiF;IACjF,kFAAkF;IAClF,kFAAkF;IAClF,mFAAmF;IACnF,oCAAoC;IACpC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpE,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACD,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,MAAM,CAAC,IAAI,CAAC;AACvB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,IAAY,EAAE,OAAe,EAAY,EAAE;IACnF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAErC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,OAAO,GAAI,IAAY,CAAC,OAA6C,CAAC;QAC5E,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACxD,IAAI,QAAQ;gBAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAAyE,EAAE,iEAApE,UAAoB,EAAE,cAAsB,CAAC;IACpF,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,MAAM,GAAG,GAAS,EAAE;;QACtB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM;gBAAE,MAAM;YACpC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAE5B,IAAI,WAAW,GAAkB,IAAI,CAAC;YACtC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBACtD,WAAW,GAAG,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,mCAAI,IAAI,CAAC;YAC3D,CAAC;YAAC,WAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/B,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,sCAAsC,GAAG,EAAE,CAAC,CAAC;gBAC/D,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEpF,OAAO,SAAS,CAAC;AACrB,CAAC,CAAA,CAAC;AAEF,MAAM,2BAA2B,GAAG,+BAIf,EAAE,sEAHnB,IAAY,EACZ,OAAe,EACf,mBAAgC,IAAI,GAAG,EAAE;IAEzC,MAAM,UAAU,GAAG,2BAA2B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACtG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,oBAAoB,CAAC,UAAU,CAAC,CAAC;AAC5C,CAAC,CAAA,CAAC;AAEF,eAAe,2BAA2B,CAAC"}

package/build/lazyLoad/generic/generic_stringsDiscovery.js

@@ -0,0 +1,103 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from "fs";
+import path from "path";
+import strings from "../../strings/index.js";
+import { resolveJsPathCandidate, confirmJsContentType } from "./generic_scanAttributesForJs.js";
+import { printMsg, MSG } from "../../utility/printMsg.js";
+/**
+ * Parses the `// File Source: <url>` header every generic-downloaded file carries
+ * as its first line (written by generic_downloadFiles.ts and generic_getScriptTags.ts)
+ * to recover the URL a downloaded file actually came from. This is what lets string
+ * literals found inside that file resolve against ITS OWN url — unlike Next.js's
+ * subsequent-requests pass (next_SubsequentRequests.ts), which resolves every
+ * extracted path against a single fixed target URL for the whole crawl.
+ */
+export const extractFileSourceUrl = (fileContent) => {
+    const match = /^\/\/ File Source: (\S+)/.exec(fileContent);
+    return match ? match[1] : null;
+};
+/**
+ * Pure candidate extraction: given the strings module's raw per-file output
+ * ({ [filePath]: string[] }), resolves each string against the URL its own file
+ * came from and keeps the ones that look like a JS path. readFile is injected so
+ * this stays unit-testable without real filesystem access.
+ */
+export const findJsPathCandidatesFromStrings = (allStrings, readFile) => {
+    var _a;
+    const candidates = new Set();
+    for (const filePath of Object.keys(allStrings)) {
+        let content;
+        try {
+            content = readFile(filePath);
+        }
+        catch (_b) {
+            continue;
+        }
+        const sourceUrl = extractFileSourceUrl(content);
+        if (!sourceUrl)
+            continue;
+        for (const str of (_a = allStrings[filePath]) !== null && _a !== void 0 ? _a : []) {
+            const resolved = resolveJsPathCandidate(str, sourceUrl);
+            if (resolved)
+                candidates.add(resolved);
+        }
+    }
+    return [...candidates];
+};
+let stringsPassCounter = 0;
+/**
+ * Runs the existing `strings` module against the generic crawl's whole output
+ * directory (files span multiple host subdirectories — the page's own host plus
+ * any third-party asset hosts), recovers per-file source URLs, and confirms
+ * JS-looking string-literal candidates via Content-Type. Internal#66: WordPress
+ * plugin configs routinely reference further JS assets only as a string inside
+ * an already-downloaded file (inline script or external JS), invisible to
+ * attribute scanning or <script src> discovery.
+ *
+ * alreadyKnownUrls (typically the caller's running downloadedJsUrls set) is
+ * filtered out BEFORE confirmJsContentType, not after — each subsequent pass
+ * rescans the same growing directory (strings() has no incremental-scan mode),
+ * so without this every already-confirmed URL from a prior pass would otherwise
+ * trigger a redundant live GET request every single pass.
+ */
+const generic_stringsDiscovery = (outputDir_1, ...args_1) => __awaiter(void 0, [outputDir_1, ...args_1], void 0, function* (outputDir, alreadyKnownUrls = new Set()) {
+    if (!fs.existsSync(outputDir))
+        return [];
+    const tmpStringsFile = path.join(outputDir, `.generic-strings-pass-${stringsPassCounter++}.json`);
+    try {
+        yield strings(outputDir, tmpStringsFile, false, "", false, false, false);
+    }
+    catch (err) {
+        printMsg(MSG.Warn, `[!] Generic strings discovery pass failed: ${err}`);
+        return [];
+    }
+    let allStrings;
+    try {
+        allStrings = JSON.parse(fs.readFileSync(tmpStringsFile, "utf-8"));
+    }
+    catch (_a) {
+        return [];
+    }
+    finally {
+        try {
+            fs.unlinkSync(tmpStringsFile);
+        }
+        catch (_b) {
+            /* best-effort cleanup — a leftover temp file isn't fatal */
+        }
+    }
+    const candidates = findJsPathCandidatesFromStrings(allStrings, (p) => fs.readFileSync(p, "utf-8")).filter((u) => !alreadyKnownUrls.has(u));
+    if (candidates.length === 0)
+        return [];
+    return confirmJsContentType(candidates);
+});
+export default generic_stringsDiscovery;
+//# sourceMappingURL=generic_stringsDiscovery.js.map