@js-recon/js-recon 1.4.1-beta.1 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-sync/capture-new.sh +61 -0
- package/.claude-sync/capture-scan.sh +55 -0
- package/.claude-sync/hooks/post-checkout +16 -0
- package/.claude-sync/hooks/post-merge +4 -0
- package/.claude-sync/hooks/pre-commit +3 -0
- package/.claude-sync/merge-agentic.sh +77 -0
- package/.claude-sync/setup.sh +17 -0
- package/.claude-sync/sync.sh +64 -0
- package/.claude-sync/watch.sh +17 -0
- package/.github/workflows/build-and-prettify.yaml +40 -0
- package/.github/workflows/promote-js-recon.yml +154 -10
- package/.vscode/tasks.json +42 -0
- package/CHANGELOG.md +27 -0
- package/Dockerfile.w-rules +13 -0
- package/README.md +1 -1
- package/build/analyze/helpers/initRules.js +27 -5
- package/build/analyze/helpers/initRules.js.map +1 -1
- package/build/analyze/index.js +3 -2
- package/build/analyze/index.js.map +1 -1
- package/build/completion/index.js +17 -11
- package/build/completion/index.js.map +1 -1
- package/build/globalConfig.js +1 -1
- package/build/globalConfig.js.map +1 -1
- package/build/index.js +55 -23
- package/build/index.js.map +1 -1
- package/build/lazyLoad/angular/angular_getFromMainJs.js +11 -29
- package/build/lazyLoad/angular/angular_getFromMainJs.js.map +1 -1
- package/build/lazyLoad/angular/angular_recursiveChunkImports.js +104 -0
- package/build/lazyLoad/angular/angular_recursiveChunkImports.js.map +1 -0
- package/build/lazyLoad/generic/generic_jsMimeTypes.js +19 -0
- package/build/lazyLoad/generic/generic_jsMimeTypes.js.map +1 -0
- package/build/lazyLoad/index.js +16 -9
- package/build/lazyLoad/index.js.map +1 -1
- package/build/lazyLoad/methodFilter.js +1 -1
- package/build/lazyLoad/methodFilter.js.map +1 -1
- package/build/lazyLoad/next_js/NextJsCrawler.js +5 -5
- package/build/lazyLoad/next_js/NextJsCrawler.js.map +1 -1
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js +17 -10
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_bruteForceJsFiles.js +4 -3
- package/build/lazyLoad/next_js/next_bruteForceJsFiles.js.map +1 -1
- package/build/lazyLoad/next_js/next_parseLayoutJs.js +31 -31
- package/build/lazyLoad/next_js/next_parseLayoutJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_promiseResolve.js +4 -3
- package/build/lazyLoad/next_js/next_promiseResolve.js.map +1 -1
- package/build/lazyLoad/next_js/next_scriptTagsSubsequentRequests.js +7 -8
- package/build/lazyLoad/next_js/next_scriptTagsSubsequentRequests.js.map +1 -1
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +6 -7
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js.map +1 -1
- package/build/lazyLoad/react/react_followImports.js +9 -8
- package/build/lazyLoad/react/react_followImports.js.map +1 -1
- package/build/lazyLoad/react/react_sourcemapUrls.js +5 -11
- package/build/lazyLoad/react/react_sourcemapUrls.js.map +1 -1
- package/build/lazyLoad/react/react_webpackChunkPaths.js +7 -6
- package/build/lazyLoad/react/react_webpackChunkPaths.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_discoverPagesFromJs.js +13 -12
- package/build/lazyLoad/svelte/svelte_discoverPagesFromJs.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js +6 -7
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js.map +1 -1
- package/build/lazyLoad/techDetect/checkInterceptedEvidence.js +18 -0
- package/build/lazyLoad/techDetect/checkInterceptedEvidence.js.map +1 -0
- package/build/lazyLoad/techDetect/index.js +58 -13
- package/build/lazyLoad/techDetect/index.js.map +1 -1
- package/build/lazyLoad/vue/vue_discoverJsFiles.js +6 -6
- package/build/lazyLoad/vue/vue_discoverJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_getClientSidePaths.js +8 -7
- package/build/lazyLoad/vue/vue_getClientSidePaths.js.map +1 -1
- package/build/lazyLoad/vue/vue_jsImports.js +16 -16
- package/build/lazyLoad/vue/vue_jsImports.js.map +1 -1
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js +5 -4
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js.map +1 -1
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js +8 -1
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js.map +1 -1
- package/build/lazyLoad/vue/vue_stringJsFiles.js +15 -8
- package/build/lazyLoad/vue/vue_stringJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_viteMapDeps.js +6 -5
- package/build/lazyLoad/vue/vue_viteMapDeps.js.map +1 -1
- package/build/mcp/mcpServer.js +1 -2
- package/build/mcp/mcpServer.js.map +1 -1
- package/build/mcp/tools.js +1 -2
- package/build/mcp/tools.js.map +1 -1
- package/build/proxy/awsConfig.js +45 -0
- package/build/proxy/awsConfig.js.map +1 -0
- package/build/proxy/checkFeasibility.js.map +1 -0
- package/build/proxy/checkFireWallBlocking.js.map +1 -0
- package/build/proxy/configFile.js +31 -0
- package/build/proxy/configFile.js.map +1 -0
- package/build/{api_gateway → proxy}/genReq.js +4 -5
- package/build/proxy/genReq.js.map +1 -0
- package/build/proxy/genericProxy.js +21 -0
- package/build/proxy/genericProxy.js.map +1 -0
- package/build/proxy/index.js +421 -0
- package/build/proxy/index.js.map +1 -0
- package/build/proxy/oxylabsProxy.js +34 -0
- package/build/proxy/oxylabsProxy.js.map +1 -0
- package/build/proxy/resolveProxyConfig.js +38 -0
- package/build/proxy/resolveProxyConfig.js.map +1 -0
- package/build/refactor/index.js +23 -28
- package/build/refactor/index.js.map +1 -1
- package/build/report/utility/initGlobalReportDb.js +111 -0
- package/build/report/utility/initGlobalReportDb.js.map +1 -0
- package/build/report/utility/mergeDomainIntoGlobalDb.js +39 -0
- package/build/report/utility/mergeDomainIntoGlobalDb.js.map +1 -0
- package/build/run/index.js +129 -75
- package/build/run/index.js.map +1 -1
- package/build/run/interruptHandler.js +17 -1
- package/build/run/interruptHandler.js.map +1 -1
- package/build/utility/concurrency.js +35 -0
- package/build/utility/concurrency.js.map +1 -0
- package/build/utility/configureProxy.js +60 -0
- package/build/utility/configureProxy.js.map +1 -0
- package/build/utility/globals.js +108 -13
- package/build/utility/globals.js.map +1 -1
- package/build/utility/makeReq.js +103 -10
- package/build/utility/makeReq.js.map +1 -1
- package/package.json +9 -2
- package/patches/puppeteer-extra-plugin-user-data-dir+2.4.1.patch +31 -0
- package/CLAUDE.md +0 -641
- package/build/api_gateway/checkFeasibility.js.map +0 -1
- package/build/api_gateway/checkFireWallBlocking.js.map +0 -1
- package/build/api_gateway/genReq.js.map +0 -1
- package/build/api_gateway/index.js +0 -291
- package/build/api_gateway/index.js.map +0 -1
- /package/build/{api_gateway → proxy}/checkFeasibility.js +0 -0
- /package/build/{api_gateway → proxy}/checkFireWallBlocking.js +0 -0
|
@@ -13,6 +13,7 @@ import makeRequest from "../../utility/makeReq.js";
|
|
|
13
13
|
import resolvePath from "../../utility/resolvePath.js";
|
|
14
14
|
import { getJsUrls, pushToJsUrls } from "../globals.js";
|
|
15
15
|
import { URL } from "url";
|
|
16
|
+
import { runWithConcurrency } from "../../utility/concurrency.js";
|
|
16
17
|
const ASSET_EXTENSIONS = [".js", ".css", ".json", ".png", ".svg", ".woff", ".woff2", ".ico", ".jpg", ".gif"];
|
|
17
18
|
const SKIP_PREFIXES = ["/_astro/", "/_next/", "/__"];
|
|
18
19
|
/**
|
|
@@ -93,8 +94,7 @@ const MAX_ROUNDS = 5;
|
|
|
93
94
|
* @param entryUrl - The target's base URL (used to build absolute page URLs)
|
|
94
95
|
* @returns Array of all newly discovered JS file URLs across all rounds
|
|
95
96
|
*/
|
|
96
|
-
const svelte_discoverPagesFromJs = (
|
|
97
|
-
var _a;
|
|
97
|
+
const svelte_discoverPagesFromJs = (entryUrl_1, ...args_1) => __awaiter(void 0, [entryUrl_1, ...args_1], void 0, function* (entryUrl, threads = 1) {
|
|
98
98
|
const origin = new URL(entryUrl).origin;
|
|
99
99
|
const allDiscovered = [];
|
|
100
100
|
const visitedPaths = new Set();
|
|
@@ -105,37 +105,38 @@ const svelte_discoverPagesFromJs = (entryUrl) => __awaiter(void 0, void 0, void
|
|
|
105
105
|
break;
|
|
106
106
|
// Collect candidate page paths from all unscanned JS URLs
|
|
107
107
|
const candidatePaths = new Set();
|
|
108
|
-
|
|
108
|
+
yield runWithConcurrency(currentJsUrls, threads, (jsUrl) => __awaiter(void 0, void 0, void 0, function* () {
|
|
109
109
|
scannedJsUrls.add(jsUrl);
|
|
110
110
|
try {
|
|
111
111
|
const res = yield makeRequest(jsUrl, {});
|
|
112
112
|
if (!res)
|
|
113
|
-
|
|
113
|
+
return;
|
|
114
114
|
const content = yield res.text();
|
|
115
115
|
for (const p of extractPagePaths(content)) {
|
|
116
116
|
candidatePaths.add(p);
|
|
117
117
|
}
|
|
118
118
|
}
|
|
119
|
-
catch (
|
|
120
|
-
|
|
119
|
+
catch (_a) {
|
|
120
|
+
return;
|
|
121
121
|
}
|
|
122
|
-
}
|
|
122
|
+
}));
|
|
123
123
|
// Filter to only unvisited paths
|
|
124
124
|
const newPaths = [...candidatePaths].filter((p) => !visitedPaths.has(p));
|
|
125
125
|
if (newPaths.length === 0)
|
|
126
126
|
break;
|
|
127
127
|
console.log(chalk.cyan(`[i] Found ${newPaths.length} candidate page path(s) in JS — visiting to discover more chunks`));
|
|
128
128
|
let foundThisRound = 0;
|
|
129
|
-
|
|
129
|
+
yield runWithConcurrency(newPaths, threads, (pagePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
130
|
+
var _a;
|
|
130
131
|
visitedPaths.add(pagePath);
|
|
131
132
|
const pageUrl = `${origin}${pagePath}`;
|
|
132
133
|
try {
|
|
133
134
|
const res = yield makeRequest(pageUrl, {});
|
|
134
135
|
if (!res)
|
|
135
|
-
|
|
136
|
+
return;
|
|
136
137
|
const contentType = (_a = res.headers.get("content-type")) !== null && _a !== void 0 ? _a : "";
|
|
137
138
|
if (!contentType.includes("text/html"))
|
|
138
|
-
|
|
139
|
+
return;
|
|
139
140
|
const html = yield res.text();
|
|
140
141
|
const jsFromPage = yield extractJsFromHtml(pageUrl, html);
|
|
141
142
|
for (const jsUrl of jsFromPage) {
|
|
@@ -149,10 +150,10 @@ const svelte_discoverPagesFromJs = (entryUrl) => __awaiter(void 0, void 0, void
|
|
|
149
150
|
console.log(chalk.green(`[✓] ${pageUrl}: found ${jsFromPage.length} JS file(s)`));
|
|
150
151
|
}
|
|
151
152
|
}
|
|
152
|
-
catch (
|
|
153
|
+
catch (_b) {
|
|
153
154
|
// page not found or network error — skip
|
|
154
155
|
}
|
|
155
|
-
}
|
|
156
|
+
}));
|
|
156
157
|
if (foundThisRound === 0)
|
|
157
158
|
break;
|
|
158
159
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"svelte_discoverPagesFromJs.js","sourceRoot":"","sources":["../../../src/lazyLoad/svelte/svelte_discoverPagesFromJs.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,WAAW,MAAM,8BAA8B,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"svelte_discoverPagesFromJs.js","sourceRoot":"","sources":["../../../src/lazyLoad/svelte/svelte_discoverPagesFromJs.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,WAAW,MAAM,8BAA8B,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAC7G,MAAM,aAAa,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;AAErD;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAY,EAAE;IACnD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,mDAAmD;IACnD,MAAM,SAAS,GAAG,+BAA+B,CAAC;IAClD,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QAC1D,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,SAAS;QAC7D,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACtC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,iEAAiE;IACjE,0FAA0F;IAC1F,MAAM,WAAW,GAAG,2DAA2D,CAAC;IAChF,OAAO,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB;QACrC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,SAAS;QAClE,gDAAgD;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,IAAI,IAAI,CAAC;QAC5B,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,iBAAiB,GAAG,CAAO,OAAe,EAAE,IAAY,EAAqB,EAAE;IACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,GAAG,KAAK,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC9F,CAAC;IACL,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,KAAK;gBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAClG,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,wBAAiE,EAAE,+DAA5D,QAAgB,EAAE,UAAkB,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAExC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEtC,0DAA0D;QAC1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,MAAM,kBAAkB,CAAC,aAAa,EAAE,OAAO,EAAE,CAAO,KAAK,EAAE,EAAE;YAC7D,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACzC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBACjB,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjC,KAAK,MAAM,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC1B,CAAC;YACL,CAAC;YAAC,WAAM,CAAC;gBACL,OAAO;YACX,CAAC;QACL,CAAC,CAAA,CAAC,CAAC;QAEH,iCAAiC;QACjC,MAAM,QAAQ,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEjC,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,kEAAkE,CAAC,CAC7G,CAAC;QAEF,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAO,QAAQ,EAAE,EAAE;;YAC3D,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC3B,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,CAAC,GAAG;oBAAE,OAAO;gBACjB,MAAM,WAAW,GAAG,MAAA,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,mCAAI,EAAE,CAAC;gBAC1D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;oBAAE,OAAO;gBAE/C,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAE1D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC7B,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC/B,YAAY,CAAC,KAAK,CAAC,CAAC;wBACpB,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC1B,cAAc,EAAE,CAAC;oBACrB,CAAC;gBACL,CAAC;gBAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,OAAO,WAAW,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC;gBACtF,CAAC;YACL,CAAC;YAAC,WAAM,CAAC;gBACL,yCAAyC;YAC7C,CAAC;QACL,CAAC,CAAA,CAAC,CAAC;QAEH,IAAI,cAAc,KAAK,CAAC;YAAE,MAAM;IACpC,CAAC;IAED,OAAO,aAAa,CAAC;AACzB,CAAC,CAAA,CAAC;AAEF,eAAe,0BAA0B,CAAC"}
|
|
@@ -15,6 +15,7 @@ import resolvePath from "../../utility/resolvePath.js";
|
|
|
15
15
|
// for parsing
|
|
16
16
|
import parser from "@babel/parser";
|
|
17
17
|
import _traverse from "@babel/traverse";
|
|
18
|
+
import { runWithConcurrency } from "../../utility/concurrency.js";
|
|
18
19
|
const traverse = ((_a = _traverse.default) !== null && _a !== void 0 ? _a : _traverse);
|
|
19
20
|
let analyzedFiles = [];
|
|
20
21
|
let filesFound = [];
|
|
@@ -61,7 +62,7 @@ export const parseJSFileContent = (content) => __awaiter(void 0, void 0, void 0,
|
|
|
61
62
|
* @param {string} url - The URL of the webpage to fetch and parse.
|
|
62
63
|
* @returns {Promise<string[]>} - A promise that resolves to an array of absolute URLs pointing to JavaScript files found in the page.
|
|
63
64
|
*/
|
|
64
|
-
const svelte_stringAnalysisJSFiles = (
|
|
65
|
+
const svelte_stringAnalysisJSFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...args_1], void 0, function* (url, threads = 1) {
|
|
65
66
|
console.log(chalk.cyan("[i] Analyzing strings in the files found"));
|
|
66
67
|
while (true) {
|
|
67
68
|
// get all the JS URLs
|
|
@@ -82,11 +83,9 @@ const svelte_stringAnalysisJSFiles = (url) => __awaiter(void 0, void 0, void 0,
|
|
|
82
83
|
if (everythingAnalyzed) {
|
|
83
84
|
break;
|
|
84
85
|
}
|
|
85
|
-
// iterate through the JS URLs
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
continue;
|
|
89
|
-
}
|
|
86
|
+
// iterate through the JS URLs not yet analyzed, concurrently
|
|
87
|
+
const toAnalyze = js_urls.filter((js_url) => !analyzedFiles.includes(js_url));
|
|
88
|
+
yield runWithConcurrency(toAnalyze, threads, (js_url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
90
89
|
const response = yield makeRequest(js_url, {});
|
|
91
90
|
const respText = yield response.text();
|
|
92
91
|
const foundJsFiles = yield parseJSFileContent(respText);
|
|
@@ -109,7 +108,7 @@ const svelte_stringAnalysisJSFiles = (url) => __awaiter(void 0, void 0, void 0,
|
|
|
109
108
|
filesFound.push(resolvedPath);
|
|
110
109
|
}
|
|
111
110
|
analyzedFiles.push(js_url);
|
|
112
|
-
}
|
|
111
|
+
}));
|
|
113
112
|
}
|
|
114
113
|
// dedupe the files
|
|
115
114
|
filesFound = [...new Set(filesFound)];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"svelte_stringAnalysisJSFiles.js","sourceRoot":"","sources":["../../../src/lazyLoad/svelte/svelte_stringAnalysisJSFiles.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,WAAW,MAAM,8BAA8B,CAAC;AAEvD,cAAc;AACd,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AAExC,MAAM,QAAQ,GAAG,CAAC,MAAA,SAAS,CAAC,OAAO,mCAAI,SAAS,CAA6B,CAAC;AAE9E,IAAI,aAAa,GAAG,EAAE,CAAC;AACvB,IAAI,UAAU,GAAG,EAAE,CAAC;AACpB,IAAI,aAAa,GAAG,EAAE,CAAC;AAEvB;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAO,OAAO,EAAE,EAAE;IAChD,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YAC9B,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC;YAC9B,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,EAAE,CAAC;QAEtB,QAAQ,CAAC,GAAG,EAAE;YACV,aAAa,CAAC,IAAI;gBACd,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC9B,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClD,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;qBAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1D,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,wBAAwB;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,
|
|
1
|
+
{"version":3,"file":"svelte_stringAnalysisJSFiles.js","sourceRoot":"","sources":["../../../src/lazyLoad/svelte/svelte_stringAnalysisJSFiles.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,WAAW,MAAM,8BAA8B,CAAC;AAEvD,cAAc;AACd,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,MAAM,QAAQ,GAAG,CAAC,MAAA,SAAS,CAAC,OAAO,mCAAI,SAAS,CAA6B,CAAC;AAE9E,IAAI,aAAa,GAAG,EAAE,CAAC;AACvB,IAAI,UAAU,GAAG,EAAE,CAAC;AACpB,IAAI,aAAa,GAAG,EAAE,CAAC;AAEvB;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAO,OAAO,EAAE,EAAE;IAChD,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YAC9B,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC;YAC9B,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,EAAE,CAAC;QAEtB,QAAQ,CAAC,GAAG,EAAE;YACV,aAAa,CAAC,IAAI;gBACd,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC9B,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAClD,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;qBAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1D,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAChC,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,wBAAwB;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,mBAAiC,EAAE,0DAA5B,GAAG,EAAE,UAAkB,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC,CAAC;IAEpE,OAAO,IAAI,EAAE,CAAC;QACV,sBAAsB;QACtB,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;QAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC,CAAC;YACtE,MAAM;QACV,CAAC;QAED,6DAA6D;QAC7D,IAAI,kBAAkB,GAAG,IAAI,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,wEAAwE;YACxE,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,kBAAkB,GAAG,KAAK,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM;QACV,CAAC;QAED,6DAA6D;QAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,MAAM,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,CAAO,MAAM,EAAE,EAAE;YAC1D,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,YAAY,GAAiB,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAEtE,wDAAwD;YACxD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACvE,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;gBAC5C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;YAED,yDAAyD;YACzD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtD,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACtD,IAAI,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACvC,SAAS;gBACb,CAAC;gBACD,YAAY,CAAC,YAAY,CAAC,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAClC,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,CAAA,CAAC,CAAC;IACP,CAAC;IAED,mBAAmB;IACnB,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IACtC,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAE5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,UAAU,CAAC,MAAM,gCAAgC,CAAC,CAAC,CAAC;IACzF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,aAAa,CAAC,MAAM,6BAA6B,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC;AAC5D,CAAC,CAAA,CAAC;AAEF,eAAe,4BAA4B,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { isJsContentType } from "../generic/generic_jsMimeTypes.js";
|
|
2
|
+
/**
|
|
3
|
+
* A framework-shaped intercepted URL (e.g. matching /_nuxt/, /_next/) is only real
|
|
4
|
+
* detection evidence if its response actually looks like JS — a path-shape match alone
|
|
5
|
+
* isn't enough, since an error/maintenance page served at that path shape would
|
|
6
|
+
* otherwise be enough to mis-fingerprint the whole target.
|
|
7
|
+
*/
|
|
8
|
+
export const isValidInterceptedJsEvidence = (status, contentType, body) => {
|
|
9
|
+
if (status < 200 || status >= 300)
|
|
10
|
+
return false;
|
|
11
|
+
if (!isJsContentType(contentType))
|
|
12
|
+
return false;
|
|
13
|
+
const trimmed = body.trimStart().toLowerCase();
|
|
14
|
+
if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html"))
|
|
15
|
+
return false;
|
|
16
|
+
return true;
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=checkInterceptedEvidence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkInterceptedEvidence.js","sourceRoot":"","sources":["../../../src/lazyLoad/techDetect/checkInterceptedEvidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAEpE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,CACxC,MAAc,EACd,WAAsC,EACtC,IAAY,EACL,EAAE;IACT,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC;IAChD,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/C,IAAI,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC"}
|
|
@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
9
9
|
};
|
|
10
10
|
import chalk from "chalk";
|
|
11
11
|
import * as cheerio from "cheerio";
|
|
12
|
-
import makeRequest from "../../utility/makeReq.js";
|
|
12
|
+
import makeRequest, { buildPuppeteerProxyArgs, getResolvedProxyConfigFromGlobals } from "../../utility/makeReq.js";
|
|
13
13
|
import puppeteer from "../../utility/puppeteerInstance.js";
|
|
14
14
|
import * as globalsUtil from "../../utility/globals.js";
|
|
15
15
|
import { getChromiumPath } from "../../utility/getChromiumPath.js";
|
|
@@ -19,6 +19,8 @@ import { checkSvelte } from "./checkSvelte.js";
|
|
|
19
19
|
import { checkVueJS } from "./checkVueJS.js";
|
|
20
20
|
import { checkAngularJS } from "./checkAngularJS.js";
|
|
21
21
|
import { checkReact } from "./checkReact.js";
|
|
22
|
+
import { isValidInterceptedJsEvidence } from "./checkInterceptedEvidence.js";
|
|
23
|
+
import { isSigintHandlerActive } from "../../run/interruptHandler.js";
|
|
22
24
|
/**
|
|
23
25
|
* Detects the front-end framework used in a webpage.
|
|
24
26
|
* It does this by iterating through all HTML tags and checking if any attribute name starts with "data-v-".
|
|
@@ -52,15 +54,27 @@ const frameworkDetect = (url) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
52
54
|
// get the page source in the browser (skipped in cache-only mode — no network allowed)
|
|
53
55
|
let pageSource = "";
|
|
54
56
|
const interceptedUrls = [];
|
|
57
|
+
// Keyed by URL: the response status/content-type/body for every request Puppeteer
|
|
58
|
+
// saw during page load. Populated below so the intercepted-URL fallback can confirm
|
|
59
|
+
// a framework-shaped URL's response actually looks like JS, not just its path shape.
|
|
60
|
+
const responseEvidence = new Map();
|
|
55
61
|
if (!globalsUtil.getCacheOnly()) {
|
|
56
62
|
const chromiumPath = getChromiumPath();
|
|
63
|
+
const proxyArgs = buildPuppeteerProxyArgs(getResolvedProxyConfigFromGlobals());
|
|
57
64
|
const browser = yield puppeteer.launch({
|
|
58
65
|
executablePath: chromiumPath,
|
|
59
|
-
args:
|
|
60
|
-
|
|
61
|
-
|
|
66
|
+
args: [
|
|
67
|
+
...(globalsUtil.getDisableSandbox()
|
|
68
|
+
? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
|
|
69
|
+
: []),
|
|
70
|
+
...(proxyArgs.arg ? [proxyArgs.arg] : []),
|
|
71
|
+
],
|
|
72
|
+
handleSIGINT: !isSigintHandlerActive(),
|
|
62
73
|
});
|
|
63
74
|
const page = yield browser.newPage();
|
|
75
|
+
if (proxyArgs.authenticate) {
|
|
76
|
+
yield page.authenticate(proxyArgs.authenticate);
|
|
77
|
+
}
|
|
64
78
|
page.setDefaultNavigationTimeout(30000);
|
|
65
79
|
const cdp = yield page.createCDPSession();
|
|
66
80
|
yield cdp.send("Page.setDownloadBehavior", { behavior: "deny" });
|
|
@@ -77,6 +91,23 @@ const frameworkDetect = (url) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
77
91
|
req.abort().catch(() => { });
|
|
78
92
|
}
|
|
79
93
|
});
|
|
94
|
+
// Captures each response's status/content-type/body as it arrives — reading the
|
|
95
|
+
// body must happen while the page/CDP session is still alive, so this can't be
|
|
96
|
+
// deferred until after browser.close(). Body-read failures (redirects, aborted
|
|
97
|
+
// responses) are swallowed to an empty string rather than dropping the entry.
|
|
98
|
+
const responseBodyPromises = [];
|
|
99
|
+
page.on("response", (httpRes) => {
|
|
100
|
+
var _a;
|
|
101
|
+
const responseUrl = httpRes.url();
|
|
102
|
+
const status = httpRes.status();
|
|
103
|
+
const contentType = (_a = httpRes.headers()["content-type"]) !== null && _a !== void 0 ? _a : null;
|
|
104
|
+
responseBodyPromises.push(httpRes
|
|
105
|
+
.text()
|
|
106
|
+
.catch(() => "")
|
|
107
|
+
.then((body) => {
|
|
108
|
+
responseEvidence.set(responseUrl, { status, contentType, body });
|
|
109
|
+
}));
|
|
110
|
+
});
|
|
80
111
|
try {
|
|
81
112
|
yield page.goto(url, {
|
|
82
113
|
waitUntil: "load",
|
|
@@ -99,6 +130,9 @@ const frameworkDetect = (url) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
99
130
|
log(chalk.yellow("[!] Page navigation/content failed, falling back to fetch response if available"));
|
|
100
131
|
}
|
|
101
132
|
finally {
|
|
133
|
+
// Wait for in-flight response body reads so responseEvidence is populated
|
|
134
|
+
// before the CDP session backing them goes away.
|
|
135
|
+
yield Promise.allSettled(responseBodyPromises);
|
|
102
136
|
yield browser.close().catch(() => { });
|
|
103
137
|
}
|
|
104
138
|
}
|
|
@@ -167,21 +201,32 @@ const frameworkDetect = (url) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
167
201
|
// Fallback: check URLs intercepted by Puppeteer during page load.
|
|
168
202
|
// Some sites load framework chunks dynamically via JavaScript rather than referencing
|
|
169
203
|
// them in static HTML attributes — those won't be caught by the HTML-based checks above.
|
|
170
|
-
//
|
|
204
|
+
// A path-shape match alone isn't sufficient evidence, though — a framework-shaped path
|
|
205
|
+
// that actually resolves to an error/maintenance page would otherwise be enough to
|
|
206
|
+
// mis-fingerprint the whole target. Only accept the match once the captured response
|
|
207
|
+
// for that URL confirms it's a genuine 2xx JS response.
|
|
171
208
|
for (const interceptedUrl of interceptedUrls) {
|
|
209
|
+
let candidateName = null;
|
|
172
210
|
if (interceptedUrl.includes("/_nuxt/")) {
|
|
173
|
-
|
|
211
|
+
candidateName = "nuxt";
|
|
212
|
+
}
|
|
213
|
+
else if (interceptedUrl.includes("/_next/")) {
|
|
214
|
+
candidateName = "next";
|
|
174
215
|
}
|
|
175
|
-
if (interceptedUrl.includes("/
|
|
176
|
-
|
|
216
|
+
else if (interceptedUrl.includes("/_app/immutable/")) {
|
|
217
|
+
candidateName = "svelte";
|
|
177
218
|
}
|
|
178
|
-
if (interceptedUrl.includes("
|
|
179
|
-
|
|
219
|
+
else if (interceptedUrl.includes("/@react-refresh")) {
|
|
220
|
+
// Vite React plugin dev-mode HMR runtime — requested by every Vite/React dev server.
|
|
221
|
+
candidateName = "react";
|
|
180
222
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
223
|
+
if (candidateName === null)
|
|
224
|
+
continue;
|
|
225
|
+
const evidence = responseEvidence.get(interceptedUrl);
|
|
226
|
+
if (evidence && isValidInterceptedJsEvidence(evidence.status, evidence.contentType, evidence.body)) {
|
|
227
|
+
return { name: candidateName, evidence: `intercepted request: ${interceptedUrl}` };
|
|
184
228
|
}
|
|
229
|
+
log(chalk.yellow(`[!] Framework-shaped URL rejected as detection evidence (response is not JS): ${interceptedUrl}`));
|
|
185
230
|
}
|
|
186
231
|
return null;
|
|
187
232
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/lazyLoad/techDetect/index.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/lazyLoad/techDetect/index.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,EAAE,EAAE,uBAAuB,EAAE,iCAAiC,EAAE,MAAM,0BAA0B,CAAC;AACnH,OAAO,SAAS,MAAM,oCAAoC,CAAC;AAC3D,OAAO,KAAK,WAAW,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAEtE;;;;;;;;;GASG;AACH,MAAM,eAAe,GAAG,CAAO,GAAW,EAA+C,EAAE;IACvF,MAAM,GAAG,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;QAC3B,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC,CAAC;IAErD,wEAAwE;IACxE,yEAAyE;IACzE,wEAAwE;IACxE,4DAA4D;IAC5D,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACf,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CACC,KAAK,CAAC,MAAM,CACR,8CAA8C,GAAG,KAAK,CAAC,GAAW,aAAX,GAAG,uBAAH,GAAG,CAAU,OAAO,KAAI,GAAG,wCAAwC,CAC7H,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,uFAAuF;IACvF,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,kFAAkF;IAClF,oFAAoF;IACpF,qFAAqF;IACrF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAwE,CAAC;IACzG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,eAAe,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,uBAAuB,CAAC,iCAAiC,EAAE,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;YACnC,cAAc,EAAE,YAAY;YAC5B,IAAI,EAAE;gBACF,GAAG,CAAC,WAAW,CAAC,iBAAiB,EAAE;oBAC/B,CAAC,CAAC,CAAC,cAAc,EAAE,0BAA0B,EAAE,yBAAyB,CAAC;oBACzE,CAAC,CAAC,EAAE,CAAC;gBACT,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;YACD,YAAY,EAAE,CAAC,qBAAqB,EAAE;SACzC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,SAAS,CAAC,YAAY,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;QAExC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1C,MAAM,GAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QAEjE,oFAAoF;QACpF,gFAAgF;QAChF,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAChC,gFAAgF;YAChF,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;gBAClC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACJ,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAChC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,gFAAgF;QAChF,+EAA+E;QAC/E,+EAA+E;QAC/E,8EAA8E;QAC9E,MAAM,oBAAoB,GAAoB,EAAE,CAAC;QACjD,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE;;YAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,mCAAI,IAAI,CAAC;YAC9D,oBAAoB,CAAC,IAAI,CACrB,OAAO;iBACF,IAAI,EAAE;iBACN,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBACf,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;gBACX,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,CAAC,CAAC,CACT,CAAC;QACN,CAAC,CAAC,CAAC;QACH,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACjB,SAAS,EAAE,MAAM;gBACjB,OAAO,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,mEAAmE;YACnE,oEAAoE;YACpE,qEAAqE;YACrE,qEAAqE;YACrE,6CAA6C;YAC7C,MAAM,kBAAkB,GAAG,eAAe,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAC1F,CAAC;YACF,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,iBAAiB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACvF,CAAC;YACD,uDAAuD;YACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,iFAAiF,CAAC,CAAC,CAAC;QACzG,CAAC;gBAAS,CAAC;YACP,0EAA0E;YAC1E,iDAAiD;YACjD,MAAM,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;YAC/C,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,2CAA2C;IAC3C,YAAY;IACZ,IAAI;IAEJ,uCAAuC;IAEvC,mCAAmC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEnC,uBAAuB;IACvB,iFAAiF;IACjF,oHAAoH;IAEpH,oCAAoC;IACpC,MAAM,kBAAkB,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,kBAAkB,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,mBAAmB,GAAG,MAAM,cAAc,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzD,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAEnD,wCAAwC;IACxC,IAAI,sBAAsB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC/D,IAAI,qBAAqB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC9D,IAAI,sBAAsB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC/D,IAAI,yBAAyB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAClE,IAAI,qBAAqB,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAE9D,IAAI,IAAI,CAAC;IACT,2CAA2C;IAC3C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;IAC7D,CAAC;SAAM,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QAC1B,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,sBAAsB,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;QACjD,qBAAqB,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpD,sBAAsB,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;QACjD,yBAAyB,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5D,qBAAqB,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,kBAAkB,CAAC,QAAQ,KAAK,IAAI,IAAI,sBAAsB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QACnF,MAAM,QAAQ,GACV,kBAAkB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,QAAQ,CAAC;QACvG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACtC,CAAC;SAAM,IAAI,iBAAiB,CAAC,QAAQ,KAAK,IAAI,IAAI,qBAAqB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxF,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;QACxC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACnF,MAAM,kBAAkB,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,sBAAsB,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAClG,IAAI,kBAAkB,CAAC,QAAQ,KAAK,IAAI,IAAI,sBAAsB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YACnF,MAAM,QAAQ,GACV,kBAAkB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,QAAQ,CAAC;YACvG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACtC,CAAC;QACD,MAAM,QAAQ,GACV,iBAAiB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC;QACpG,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IACrC,CAAC;SAAM,IAAI,kBAAkB,CAAC,QAAQ,KAAK,IAAI,IAAI,sBAAsB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC1F,MAAM,QAAQ,GACV,kBAAkB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,QAAQ,CAAC;QACvG,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACxC,CAAC;SAAM,IAAI,mBAAmB,CAAC,QAAQ,KAAK,IAAI,IAAI,yBAAyB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC9F,MAAM,QAAQ,GACV,mBAAmB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB,CAAC,QAAQ,CAAC;QAC5G,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACzC,CAAC;SAAM,IAAI,iBAAiB,CAAC,QAAQ,KAAK,IAAI,IAAI,qBAAqB,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QACxF,MAAM,QAAQ,GACV,iBAAiB,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC;QACpG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACvC,CAAC;IAED,kEAAkE;IAClE,sFAAsF;IACtF,yFAAyF;IACzF,uFAAuF;IACvF,mFAAmF;IACnF,qFAAqF;IACrF,wDAAwD;IACxD,KAAK,MAAM,cAAc,IAAI,eAAe,EAAE,CAAC;QAC3C,IAAI,aAAa,GAAkB,IAAI,CAAC;QACxC,IAAI,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,aAAa,GAAG,MAAM,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,aAAa,GAAG,MAAM,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACrD,aAAa,GAAG,QAAQ,CAAC;QAC7B,CAAC;aAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACpD,qFAAqF;YACrF,aAAa,GAAG,OAAO,CAAC;QAC5B,CAAC;QACD,IAAI,aAAa,KAAK,IAAI;YAAE,SAAS;QAErC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACtD,IAAI,QAAQ,IAAI,4BAA4B,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjG,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,wBAAwB,cAAc,EAAE,EAAE,CAAC;QACvF,CAAC;QACD,GAAG,CACC,KAAK,CAAC,MAAM,CACR,iFAAiF,cAAc,EAAE,CACpG,CACJ,CAAC;IACN,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAA,CAAC;AAEF,eAAe,eAAe,CAAC"}
|
|
@@ -30,7 +30,7 @@ import { shouldRunMethod } from "../methodFilter.js";
|
|
|
30
30
|
* caller can start downloading them immediately rather than waiting for the
|
|
31
31
|
* full pipeline to finish.
|
|
32
32
|
*/
|
|
33
|
-
const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...args_1], void 0, function* (url, maxJsSizeMb = 2, onFilesDiscovered, includeMethods = [], excludeMethods = []) {
|
|
33
|
+
const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...args_1], void 0, function* (url, maxJsSizeMb = 2, onFilesDiscovered, includeMethods = [], excludeMethods = [], threads = 1) {
|
|
34
34
|
let jsFiles = [];
|
|
35
35
|
const inc = includeMethods;
|
|
36
36
|
const exc = excludeMethods;
|
|
@@ -67,7 +67,7 @@ const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...a
|
|
|
67
67
|
// scan page-loaded JS files for Vite's __vite__mapDeps chunk manifest
|
|
68
68
|
if (shouldRunMethod("vue_viteMapDeps", inc, exc)) {
|
|
69
69
|
const beforeViteMapDeps = new Set(jsFiles.map(normalize));
|
|
70
|
-
const fromViteMapDeps = yield vue_viteMapDeps(jsFiles, maxJsSizeMb);
|
|
70
|
+
const fromViteMapDeps = yield vue_viteMapDeps(jsFiles, maxJsSizeMb, threads);
|
|
71
71
|
emit(fromViteMapDeps);
|
|
72
72
|
const newViteMapDeps = countNew(fromViteMapDeps, beforeViteMapDeps);
|
|
73
73
|
if (newViteMapDeps > 0) {
|
|
@@ -77,7 +77,7 @@ const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...a
|
|
|
77
77
|
// walk the import graph of everything found so far
|
|
78
78
|
if (shouldRunMethod("vue_jsImports", inc, exc)) {
|
|
79
79
|
const beforeJsImports = new Set(jsFiles.map(normalize));
|
|
80
|
-
const fromImports = yield vue_jsImports(url, jsFiles, maxJsSizeMb);
|
|
80
|
+
const fromImports = yield vue_jsImports(url, jsFiles, maxJsSizeMb, threads);
|
|
81
81
|
emit(fromImports);
|
|
82
82
|
const newJsImports = countNew(fromImports, beforeJsImports);
|
|
83
83
|
if (newJsImports > 0) {
|
|
@@ -87,7 +87,7 @@ const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...a
|
|
|
87
87
|
// scan string literals inside known JS files for .js references
|
|
88
88
|
if (shouldRunMethod("vue_stringJsFiles", inc, exc)) {
|
|
89
89
|
const beforeStringRefs = new Set(jsFiles.map(normalize));
|
|
90
|
-
const fromStringRefs = yield vue_stringJsFiles(jsFiles, maxJsSizeMb);
|
|
90
|
+
const fromStringRefs = yield vue_stringJsFiles(jsFiles, maxJsSizeMb, threads);
|
|
91
91
|
emit(fromStringRefs);
|
|
92
92
|
const newStringRefs = countNew(fromStringRefs, beforeStringRefs);
|
|
93
93
|
if (newStringRefs > 0) {
|
|
@@ -97,7 +97,7 @@ const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...a
|
|
|
97
97
|
// reconstruct sourceMappingURL references
|
|
98
98
|
if (shouldRunMethod("vue_reconstructSourceMaps", inc, exc)) {
|
|
99
99
|
const beforeSourceMaps = new Set(jsFiles.map(normalize));
|
|
100
|
-
const fromSourceMaps = yield vue_reconstructSourceMaps(url, jsFiles);
|
|
100
|
+
const fromSourceMaps = yield vue_reconstructSourceMaps(url, jsFiles, threads);
|
|
101
101
|
emit(fromSourceMaps);
|
|
102
102
|
const newSourceMaps = countNew(fromSourceMaps, beforeSourceMaps);
|
|
103
103
|
if (newSourceMaps > 0) {
|
|
@@ -108,7 +108,7 @@ const vue_discoverJsFiles = (url_1, ...args_1) => __awaiter(void 0, [url_1, ...a
|
|
|
108
108
|
// surface client-side paths so the caller can recurse into them
|
|
109
109
|
let clientSidePaths = [];
|
|
110
110
|
if (shouldRunMethod("vue_getClientSidePaths", inc, exc)) {
|
|
111
|
-
clientSidePaths = yield vue_getClientSidePaths(url, jsFiles, maxJsSizeMb);
|
|
111
|
+
clientSidePaths = yield vue_getClientSidePaths(url, jsFiles, maxJsSizeMb, threads);
|
|
112
112
|
}
|
|
113
113
|
return { jsFiles, clientSidePaths: [...new Set(clientSidePaths)] };
|
|
114
114
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vue_discoverJsFiles.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_discoverJsFiles.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAC3C,OAAO,aAAa,MAAM,oBAAoB,CAAC;AAC/C,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,aAAa,MAAM,oBAAoB,CAAC;AAC/C,OAAO,yBAAyB,MAAM,gCAAgC,CAAC;AACvE,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,eAAe,MAAM,sBAAsB,CAAC;AACnD,OAAO,iBAAiB,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAOrD;;;;;;;;;;;GAWG;AACH,MAAM,mBAAmB,GAAG,
|
|
1
|
+
{"version":3,"file":"vue_discoverJsFiles.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_discoverJsFiles.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAC3C,OAAO,aAAa,MAAM,oBAAoB,CAAC;AAC/C,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,aAAa,MAAM,oBAAoB,CAAC;AAC/C,OAAO,yBAAyB,MAAM,gCAAgC,CAAC;AACvE,OAAO,sBAAsB,MAAM,6BAA6B,CAAC;AACjE,OAAO,eAAe,MAAM,sBAAsB,CAAC;AACnD,OAAO,iBAAiB,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAOrD;;;;;;;;;;;GAWG;AACH,MAAM,mBAAmB,GAAG,mBAOG,EAAE,0DAN7B,GAAW,EACX,cAAsB,CAAC,EACvB,iBAA6C,EAC7C,iBAA2B,EAAE,EAC7B,iBAA2B,EAAE,EAC7B,UAAkB,CAAC;IAEnB,IAAI,OAAO,GAAa,EAAE,CAAC;IAE3B,MAAM,GAAG,GAAG,cAAc,CAAC;IAC3B,MAAM,GAAG,GAAG,cAAc,CAAC;IAE3B,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,CAAC,KAAe,EAAE,MAAmB,EAAU,EAAE,CAC9D,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAE1D,MAAM,IAAI,GAAG,CAAC,KAAe,EAAE,EAAE;QAC7B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QACvB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,EAAE,CAAC;YACxC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,CAAC;IACL,CAAC,CAAC;IAEF,mDAAmD;IACnD,IAAI,eAAe,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,sCAAsC;IACtC,IAAI,eAAe,CAAC,eAAe,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,kDAAkD;IAClD,IAAI,eAAe,CAAC,wBAAwB,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;QACvD,IAAI,CAAC,YAAY,CAAC,CAAC;QACnB,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QAC3D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,WAAW,4CAA4C,CAAC,CAAC,CAAC;QACnG,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,IAAI,eAAe,CAAC,wBAAwB,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,sEAAsE;IACtE,IAAI,eAAe,CAAC,iBAAiB,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1D,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QAC7E,IAAI,CAAC,eAAe,CAAC,CAAC;QACtB,MAAM,cAAc,GAAG,QAAQ,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;QACpE,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,cAAc,iCAAiC,CAAC,CAAC,CAAC;QAC3F,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,IAAI,eAAe,CAAC,eAAe,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QAC5E,IAAI,CAAC,WAAW,CAAC,CAAC;QAClB,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QAC5D,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,YAAY,mCAAmC,CAAC,CAAC,CAAC;QAC3F,CAAC;IACL,CAAC;IAED,gEAAgE;IAChE,IAAI,eAAe,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QAC9E,IAAI,CAAC,cAAc,CAAC,CAAC;QACrB,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QACjE,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,aAAa,8CAA8C,CAAC,CAAC,CAAC;QACvG,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,IAAI,eAAe,CAAC,2BAA2B,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACzD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,yBAAyB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9E,IAAI,CAAC,cAAc,CAAC,CAAC;QACrB,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QACjE,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,aAAa,4CAA4C,CAAC,CAAC,CAAC;QACrG,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpF,gEAAgE;IAChE,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,eAAe,CAAC,wBAAwB,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACtD,eAAe,GAAG,MAAM,sBAAsB,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;AACvE,CAAC,CAAA,CAAC;AAEF,eAAe,mBAAmB,CAAC"}
|
|
@@ -15,8 +15,9 @@ import cliProgress from "cli-progress";
|
|
|
15
15
|
import parser from "@babel/parser";
|
|
16
16
|
import t from "@babel/types";
|
|
17
17
|
import { setActiveBarLogger, computeBarSize, watchBarResize } from "../../utility/progressLog.js";
|
|
18
|
+
import { runWithConcurrency } from "../../utility/concurrency.js";
|
|
18
19
|
const traverse = ((_a = _traverse.default) !== null && _a !== void 0 ? _a : _traverse);
|
|
19
|
-
const vue_getClientSidePaths = (url_1, jsFiles_1, ...args_1) => __awaiter(void 0, [url_1, jsFiles_1, ...args_1], void 0, function* (url, jsFiles, maxJsSizeMb = 2) {
|
|
20
|
+
const vue_getClientSidePaths = (url_1, jsFiles_1, ...args_1) => __awaiter(void 0, [url_1, jsFiles_1, ...args_1], void 0, function* (url, jsFiles, maxJsSizeMb = 2, threads = 1) {
|
|
20
21
|
const MAX_JS_SIZE_BYTES = maxJsSizeMb * 1024 * 1024;
|
|
21
22
|
let toReturn = [];
|
|
22
23
|
const baseOrigin = new URL(url).origin;
|
|
@@ -36,26 +37,26 @@ const vue_getClientSidePaths = (url_1, jsFiles_1, ...args_1) => __awaiter(void 0
|
|
|
36
37
|
const stopBarWatcher = watchBarResize(bar, 86);
|
|
37
38
|
setActiveBarLogger({ log: (s) => process.stdout.write("\r\x1b[K" + s) });
|
|
38
39
|
// iterate through all those
|
|
39
|
-
|
|
40
|
+
yield runWithConcurrency(jsFiles, threads, (jsFile) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
41
|
if (!jsFile.endsWith(".js")) {
|
|
41
42
|
processed++;
|
|
42
43
|
skipped++;
|
|
43
44
|
bar.update(processed, { paths: toReturn.length, skipped });
|
|
44
|
-
|
|
45
|
+
return;
|
|
45
46
|
}
|
|
46
47
|
const req = yield makeRequest(jsFile);
|
|
47
48
|
if (req == null) {
|
|
48
49
|
processed++;
|
|
49
50
|
skipped++;
|
|
50
51
|
bar.update(processed, { paths: toReturn.length, skipped });
|
|
51
|
-
|
|
52
|
+
return;
|
|
52
53
|
}
|
|
53
54
|
const jsContent = yield req.text();
|
|
54
55
|
if (jsContent.length > MAX_JS_SIZE_BYTES) {
|
|
55
56
|
processed++;
|
|
56
57
|
skipped++;
|
|
57
58
|
bar.update(processed, { paths: toReturn.length, skipped });
|
|
58
|
-
|
|
59
|
+
return;
|
|
59
60
|
}
|
|
60
61
|
// load in ast
|
|
61
62
|
let ast;
|
|
@@ -70,7 +71,7 @@ const vue_getClientSidePaths = (url_1, jsFiles_1, ...args_1) => __awaiter(void 0
|
|
|
70
71
|
processed++;
|
|
71
72
|
skipped++;
|
|
72
73
|
bar.update(processed, { paths: toReturn.length, skipped });
|
|
73
|
-
|
|
74
|
+
return;
|
|
74
75
|
}
|
|
75
76
|
const jsFileOrigin = new URL(jsFile).origin;
|
|
76
77
|
const pathsBefore = toReturn.length;
|
|
@@ -143,7 +144,7 @@ const vue_getClientSidePaths = (url_1, jsFiles_1, ...args_1) => __awaiter(void 0
|
|
|
143
144
|
if (toReturn.length === pathsBefore)
|
|
144
145
|
skipped++;
|
|
145
146
|
bar.update(processed, { paths: toReturn.length, skipped });
|
|
146
|
-
}
|
|
147
|
+
}));
|
|
147
148
|
bar.stop();
|
|
148
149
|
stopBarWatcher();
|
|
149
150
|
setActiveBarLogger(null);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vue_getClientSidePaths.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_getClientSidePaths.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,CAAC,MAAM,cAAc,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"vue_getClientSidePaths.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_getClientSidePaths.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,CAAC,MAAM,cAAc,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAClG,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,QAAQ,GAAG,CAAC,MAAA,SAAS,CAAC,OAAO,mCAAI,SAAS,CAA6B,CAAC;AAE9E,MAAM,sBAAsB,GAAG,8BAKV,EAAE,qEAJnB,GAAW,EACX,OAAiB,EACjB,cAAsB,CAAC,EACvB,UAAkB,CAAC;IAEnB,MAAM,iBAAiB,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC;IACpD,IAAI,QAAQ,GAAa,EAAE,CAAC;IAE5B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IAEvC,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACjC;QACI,MAAM,EACF,KAAK,CAAC,IAAI,CAAC,mCAAmC,CAAC;YAC/C,mFAAmF;QACvF,eAAe,EAAE,GAAG;QACpB,iBAAiB,EAAE,GAAG;QACtB,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QAC3B,UAAU,EAAE,KAAK;QACjB,eAAe,EAAE,KAAK;QACtB,cAAc,EAAE,KAAK;KACxB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACrC,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACvD,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC/C,kBAAkB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAEjF,4BAA4B;IAC5B,MAAM,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAO,MAAM,EAAE,EAAE;QACxD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3D,OAAO;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;QAEtC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACd,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3D,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;YACvC,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3D,OAAO;QACX,CAAC;QAED,cAAc;QACd,IAAI,GAAG,CAAC;QAER,IAAI,CAAC;YACD,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE;gBAC1B,UAAU,EAAE,aAAa;gBACzB,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC;gBAC9B,aAAa,EAAE,IAAI;aACtB,CAAC,CAAC;QACP,CAAC;QAAC,WAAM,CAAC;YACL,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3D,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;QAEpC,QAAQ,CAAC,GAAG,EAAE;YACV,cAAc,CAAC,IAAI;gBACf,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;gBAEjC,IACI,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;oBACrC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC;oBACxB,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EACnC,CAAC;oBACC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC;oBAE5B,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC3B,QAAQ,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC;oBACtC,CAAC;yBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjC,QAAQ,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;oBACxC,CAAC;yBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;wBAChF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;gBACL,CAAC;YACL,CAAC;YAED,cAAc,CAAC,IAAI;gBACf,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;gBAE9C,2BAA2B;gBAC3B,IACI,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC;oBAC7B,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAClD,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAEpD,OAAO;gBAEX,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO;gBAE5B,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,IAAI,CAAC;gBAEnC,kCAAkC;gBAClC,IACI,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC;oBAC/B,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;oBACtD,CAAC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACtC,CAAC,CAAC,CAAC,YAAY,CAAE,QAAQ,CAAC,MAA6B,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAEnF,OAAO;gBAEX,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,CAAC;oBAAE,OAAO;gBAE7C,KAAK,MAAM,UAAU,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;oBAC5C,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,UAAU,CAAC,KAAK,CAAC;wBAAE,SAAS;oBAEzF,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;wBAClD,IACI,CAAC,CAAC,CAAC,gBAAgB,CAAC,SAAS,CAAC;4BAC9B,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;4BAClD,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC;4BAErC,SAAS;wBAEb,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;4BAC7C,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC;gCAAE,SAAS;4BAE5C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC;4BAE3C,IAAI,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;gCAC7E,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC;gCAE9B,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oCAC3B,QAAQ,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC;gCACtC,CAAC;qCAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oCACjC,QAAQ,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;gCACxC,CAAC;qCAAM,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;oCAChF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gCAC3B,CAAC;4BACL,CAAC;wBACL,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,SAAS,EAAE,CAAC;QACZ,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO,EAAE,CAAC;QAC/C,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC,CAAA,CAAC,CAAC;IAEH,GAAG,CAAC,IAAI,EAAE,CAAC;IACX,cAAc,EAAE,CAAC;IACjB,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAEzB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,mCAAmC,CAAC,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC,CAAA,CAAC;AAEF,eAAe,sBAAsB,CAAC"}
|
|
@@ -12,6 +12,7 @@ import makeRequest from "../../utility/makeReq.js";
|
|
|
12
12
|
import parser from "@babel/parser";
|
|
13
13
|
import _traverse from "@babel/traverse";
|
|
14
14
|
import chalk from "chalk";
|
|
15
|
+
import { runWithConcurrency } from "../../utility/concurrency.js";
|
|
15
16
|
const traverse = ((_a = _traverse.default) !== null && _a !== void 0 ? _a : _traverse);
|
|
16
17
|
const parseJsFile = (url, maxJsSizeMb) => __awaiter(void 0, void 0, void 0, function* () {
|
|
17
18
|
const MAX_JS_SIZE_BYTES = maxJsSizeMb * 1024 * 1024;
|
|
@@ -49,27 +50,26 @@ const parseJsFile = (url, maxJsSizeMb) => __awaiter(void 0, void 0, void 0, func
|
|
|
49
50
|
});
|
|
50
51
|
return foundUrls;
|
|
51
52
|
});
|
|
52
|
-
const vue_jsImports = (url_1, foundJsFiles_1, ...args_1) => __awaiter(void 0, [url_1, foundJsFiles_1, ...args_1], void 0, function* (url, foundJsFiles, maxJsSizeMb = 2) {
|
|
53
|
+
const vue_jsImports = (url_1, foundJsFiles_1, ...args_1) => __awaiter(void 0, [url_1, foundJsFiles_1, ...args_1], void 0, function* (url, foundJsFiles, maxJsSizeMb = 2, threads = 1) {
|
|
53
54
|
const allDiscoveredUrls = new Set();
|
|
54
55
|
const crawledUrls = new Set(foundJsFiles);
|
|
55
|
-
|
|
56
|
+
yield runWithConcurrency(foundJsFiles, threads, (jsfile) => __awaiter(void 0, void 0, void 0, function* () {
|
|
56
57
|
const discovered = yield parseJsFile(jsfile, maxJsSizeMb);
|
|
57
58
|
for (const u of discovered)
|
|
58
59
|
allDiscoveredUrls.add(u);
|
|
59
|
-
}
|
|
60
|
-
// crawl newly found URLs until no uncrawled ones remain
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
for (const u of
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
}
|
|
60
|
+
}));
|
|
61
|
+
// crawl newly found URLs until no uncrawled ones remain, one round of
|
|
62
|
+
// concurrent fetches at a time
|
|
63
|
+
let toCrawl = [...allDiscoveredUrls].filter((u) => !crawledUrls.has(u));
|
|
64
|
+
while (toCrawl.length > 0) {
|
|
65
|
+
for (const u of toCrawl)
|
|
66
|
+
crawledUrls.add(u);
|
|
67
|
+
yield runWithConcurrency(toCrawl, threads, (u) => __awaiter(void 0, void 0, void 0, function* () {
|
|
68
|
+
const discovered = yield parseJsFile(u, maxJsSizeMb);
|
|
69
|
+
for (const newU of discovered)
|
|
70
|
+
allDiscoveredUrls.add(newU);
|
|
71
|
+
}));
|
|
72
|
+
toCrawl = [...allDiscoveredUrls].filter((u) => !crawledUrls.has(u));
|
|
73
73
|
}
|
|
74
74
|
return [...allDiscoveredUrls];
|
|
75
75
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vue_jsImports.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_jsImports.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,KAAK,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"vue_jsImports.js","sourceRoot":"","sources":["../../../src/lazyLoad/vue/vue_jsImports.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,MAAM,MAAM,eAAe,CAAC;AACnC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,QAAQ,GAAG,CAAC,MAAA,SAAS,CAAC,OAAO,mCAAI,SAAS,CAA6B,CAAC;AAE9E,MAAM,WAAW,GAAG,CAAO,GAAW,EAAE,WAAmB,EAAE,EAAE;IAC3D,MAAM,iBAAiB,GAAG,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC;QACnD,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAEjC,IAAI,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACrC,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,IAAI,GAAG,CAAC;IACR,IAAI,CAAC;QACD,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YACxB,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,CAAC,kBAAkB,CAAC;SAChC,CAAC,CAAC;IACP,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,gCAAgC;IAChC,QAAQ,CAAC,GAAG,EAAE;QACV,iBAAiB,CAAC,IAAI;YAClB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YAEtC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChF,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;YACnG,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,SAAS,CAAC;AACrB,CAAC,CAAA,CAAC;AAEF,MAAM,aAAa,GAAG,mCAA0F,EAAE,0EAArF,GAAW,EAAE,YAAsB,EAAE,cAAsB,CAAC,EAAE,UAAkB,CAAC;IAC1G,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IAElD,MAAM,kBAAkB,CAAC,YAAY,EAAE,OAAO,EAAE,CAAO,MAAM,EAAE,EAAE;QAC7D,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,UAAU;YAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAA,CAAC,CAAC;IAEH,sEAAsE;IACtE,+BAA+B;IAC/B,IAAI,OAAO,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAO,CAAC,EAAE,EAAE;YACnD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;YACrD,KAAK,MAAM,IAAI,IAAI,UAAU;gBAAE,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/D,CAAC,CAAA,CAAC,CAAC;QACH,OAAO,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,CAAC,GAAG,iBAAiB,CAAC,CAAC;AAClC,CAAC,CAAA,CAAC;AAEF,eAAe,aAAa,CAAC"}
|