@shriyanss/js-recon 1.0.0 → 1.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/build-and-prettify.yaml +65 -0
- package/.github/workflows/npm-publish.yml +35 -0
- package/.prettierignore +2 -0
- package/.prettierrc +4 -0
- package/CHANGELOG.md +50 -0
- package/README.md +35 -139
- package/build/api_gateway/checkFeasibility.js +32 -0
- package/build/api_gateway/checkFeasibility.js.map +1 -0
- package/build/api_gateway/checkFireWallBlocking.js +24 -0
- package/build/api_gateway/checkFireWallBlocking.js.map +1 -0
- package/build/api_gateway/genReq.js +199 -0
- package/build/api_gateway/genReq.js.map +1 -0
- package/build/api_gateway/index.js +275 -0
- package/build/api_gateway/index.js.map +1 -0
- package/build/endpoints/gen_report/gen_json.js +22 -0
- package/build/endpoints/gen_report/gen_json.js.map +1 -0
- package/build/endpoints/gen_report/gen_markdown.js +66 -0
- package/build/endpoints/gen_report/gen_markdown.js.map +1 -0
- package/build/endpoints/gen_report/utility/iterate_n_store.js +46 -0
- package/build/endpoints/gen_report/utility/iterate_n_store.js.map +1 -0
- package/build/endpoints/index.js +89 -0
- package/build/endpoints/index.js.map +1 -0
- package/build/endpoints/next_js/client_jsFilesHref.js +91 -0
- package/build/endpoints/next_js/client_jsFilesHref.js.map +1 -0
- package/build/endpoints/next_js/client_jsonParse.js +78 -0
- package/build/endpoints/next_js/client_jsonParse.js.map +1 -0
- package/build/endpoints/next_js/client_subsequentRequests.js +199 -0
- package/build/endpoints/next_js/client_subsequentRequests.js.map +1 -0
- package/build/endpoints/next_js/getWebpacks.js +45 -0
- package/build/endpoints/next_js/getWebpacks.js.map +1 -0
- package/build/globalConfig.js +11 -0
- package/build/globalConfig.js.map +1 -0
- package/build/index.js +166 -0
- package/build/index.js.map +1 -0
- package/build/lazyLoad/downloadFilesUtil.js +128 -0
- package/build/lazyLoad/downloadFilesUtil.js.map +1 -0
- package/build/lazyLoad/downloadLoadedJsUtil.js +51 -0
- package/build/lazyLoad/downloadLoadedJsUtil.js.map +1 -0
- package/build/lazyLoad/globals.js +22 -0
- package/build/lazyLoad/globals.js.map +1 -0
- package/build/lazyLoad/index.js +170 -0
- package/build/lazyLoad/index.js.map +1 -0
- package/build/lazyLoad/next_js/next_GetJSScript.js +94 -0
- package/build/lazyLoad/next_js/next_GetJSScript.js.map +1 -0
- package/build/lazyLoad/next_js/next_GetLazyResources.js +202 -0
- package/build/lazyLoad/next_js/next_GetLazyResources.js.map +1 -0
- package/build/lazyLoad/next_js/next_SubsequentRequests.js +120 -0
- package/build/lazyLoad/next_js/next_SubsequentRequests.js.map +1 -0
- package/build/lazyLoad/nuxt_js/nuxt_astParse.js +188 -0
- package/build/lazyLoad/nuxt_js/nuxt_astParse.js.map +1 -0
- package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js +75 -0
- package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js.map +1 -0
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +94 -0
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js.map +1 -0
- package/build/lazyLoad/svelte/svelte_getFromPageSource.js +68 -0
- package/build/lazyLoad/svelte/svelte_getFromPageSource.js.map +1 -0
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js +95 -0
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js.map +1 -0
- package/build/map/index.js +58 -0
- package/build/map/index.js.map +1 -0
- package/build/map/next_js/getFetchInstances.js +108 -0
- package/build/map/next_js/getFetchInstances.js.map +1 -0
- package/build/map/next_js/getWebpackConnections.js +227 -0
- package/build/map/next_js/getWebpackConnections.js.map +1 -0
- package/build/map/next_js/interactive.js +32 -0
- package/build/map/next_js/interactive.js.map +1 -0
- package/build/map/next_js/interactive_helpers/commandHandler.js +190 -0
- package/build/map/next_js/interactive_helpers/commandHandler.js.map +1 -0
- package/build/map/next_js/interactive_helpers/commandHelpers.js +91 -0
- package/build/map/next_js/interactive_helpers/commandHelpers.js.map +1 -0
- package/build/map/next_js/interactive_helpers/helpMenu.js +11 -0
- package/build/map/next_js/interactive_helpers/helpMenu.js.map +1 -0
- package/build/map/next_js/interactive_helpers/keybindings.js +80 -0
- package/build/map/next_js/interactive_helpers/keybindings.js.map +1 -0
- package/build/map/next_js/interactive_helpers/printer.js +17 -0
- package/build/map/next_js/interactive_helpers/printer.js.map +1 -0
- package/build/map/next_js/interactive_helpers/ui.js +81 -0
- package/build/map/next_js/interactive_helpers/ui.js.map +1 -0
- package/build/map/next_js/resolveFetch.js +201 -0
- package/build/map/next_js/resolveFetch.js.map +1 -0
- package/build/run/index.js +62 -0
- package/build/run/index.js.map +1 -0
- package/build/strings/index.js +238 -0
- package/build/strings/index.js.map +1 -0
- package/build/strings/openapi.js +55 -0
- package/build/strings/openapi.js.map +1 -0
- package/build/strings/permutate.js +55 -0
- package/build/strings/permutate.js.map +1 -0
- package/build/strings/secrets.js +89 -0
- package/build/strings/secrets.js.map +1 -0
- package/build/techDetect/index.js +229 -0
- package/build/techDetect/index.js.map +1 -0
- package/build/utility/ai.js +69 -0
- package/build/utility/ai.js.map +1 -0
- package/build/utility/globals.js +84 -0
- package/build/utility/globals.js.map +1 -0
- package/build/utility/interfaces.js +2 -0
- package/build/utility/interfaces.js.map +1 -0
- package/build/utility/makeReq.js +265 -0
- package/build/utility/makeReq.js.map +1 -0
- package/build/utility/resolvePath.js +44 -0
- package/build/utility/resolvePath.js.map +1 -0
- package/{utility → build/utility}/runSandboxed.js +10 -13
- package/build/utility/runSandboxed.js.map +1 -0
- package/{utility → build/utility}/urlUtils.js +9 -11
- package/build/utility/urlUtils.js.map +1 -0
- package/docs/CNAME +1 -0
- package/docs/README.md +20 -0
- package/docs/api-gateway.md +68 -0
- package/docs/endpoints.md +49 -0
- package/docs/example-scenario.md +258 -0
- package/docs/interactive-mode.md +76 -0
- package/docs/lazyload.md +56 -0
- package/docs/map.md +53 -0
- package/docs/run.md +54 -0
- package/docs/strings.md +75 -0
- package/package.json +50 -38
- package/api_gateway/checkFeasibility.js +0 -25
- package/api_gateway/checkFireWallBlocking.js +0 -17
- package/api_gateway/genReq.js +0 -214
- package/api_gateway/index.js +0 -325
- package/endpoints/index.js +0 -7
- package/globalConfig.js +0 -12
- package/index.js +0 -69
- package/lazyLoad/downloadFilesUtil.js +0 -122
- package/lazyLoad/downloadLoadedJsUtil.js +0 -54
- package/lazyLoad/globals.js +0 -15
- package/lazyLoad/index.js +0 -167
- package/lazyLoad/next_js/next_GetJSScript.js +0 -99
- package/lazyLoad/next_js/next_GetLazyResources.js +0 -201
- package/lazyLoad/next_js/next_SubsequentRequests.js +0 -138
- package/lazyLoad/nuxt_js/nuxt_astParse.js +0 -194
- package/lazyLoad/nuxt_js/nuxt_getFromPageSource.js +0 -77
- package/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +0 -99
- package/research/firewall_bypass.md +0 -38
- package/research/next_js.md +0 -116
- package/research/nuxt_js.md +0 -125
- package/research/vue_js.md +0 -9
- package/strings/index.js +0 -145
- package/techDetect/index.js +0 -156
- package/utility/globals.js +0 -6
- package/utility/makeReq.js +0 -179
- package/utility/resolvePath.js +0 -43
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import chalk from "chalk";
|
|
11
|
+
import path from "path";
|
|
12
|
+
import fs from "fs";
|
|
13
|
+
import prettier from "prettier";
|
|
14
|
+
import makeRequest from "../utility/makeReq.js";
|
|
15
|
+
import { getURLDirectory } from "../utility/urlUtils.js";
|
|
16
|
+
import { getScope, getMaxReqQueue } from "./globals.js"; // Import scope and max_req_queue functions
|
|
17
|
+
/**
|
|
18
|
+
* Downloads a list of URLs and saves them as files in the specified output directory.
|
|
19
|
+
* It creates the necessary subdirectories based on the URL's host and path.
|
|
20
|
+
* If the URL does not end with `.js`, it is skipped.
|
|
21
|
+
* The function logs the progress and any errors to the console.
|
|
22
|
+
* @param {string[]} urls - An array of URLs to be downloaded.
|
|
23
|
+
* @param {string} output - The directory where the downloaded files will be saved.
|
|
24
|
+
* @returns {Promise<void>}
|
|
25
|
+
*/
|
|
26
|
+
const downloadFiles = (urls, output) => __awaiter(void 0, void 0, void 0, function* () {
|
|
27
|
+
console.log(chalk.cyan(`[i] Attempting to download ${urls.length} JS chunks`));
|
|
28
|
+
fs.mkdirSync(output, { recursive: true });
|
|
29
|
+
// to store ignored JS domain
|
|
30
|
+
let ignoredJSFiles = [];
|
|
31
|
+
let ignoredJSDomains = [];
|
|
32
|
+
let download_count = 0;
|
|
33
|
+
let queue = 0;
|
|
34
|
+
const downloadPromises = urls.map((url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
35
|
+
try {
|
|
36
|
+
yield new Promise((resolve) => setTimeout(resolve, Math.random() * 4950 + 50));
|
|
37
|
+
if (url.match(/(\.js|\.json)/)) {
|
|
38
|
+
// get the directory of the url
|
|
39
|
+
const { host, directory } = getURLDirectory(url);
|
|
40
|
+
// check scope of file. Only if in scope, download it
|
|
41
|
+
if (!getScope().includes("*")) {
|
|
42
|
+
if (!getScope().includes(host)) {
|
|
43
|
+
ignoredJSFiles.push(url);
|
|
44
|
+
if (!ignoredJSDomains.includes(host)) {
|
|
45
|
+
ignoredJSDomains.push(host);
|
|
46
|
+
}
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// make the directory inside the output folder
|
|
51
|
+
const childDir = path.join(output, host, directory);
|
|
52
|
+
fs.mkdirSync(childDir, { recursive: true });
|
|
53
|
+
let res;
|
|
54
|
+
try {
|
|
55
|
+
// Wait until there is an available slot in the request queue
|
|
56
|
+
while (queue >= getMaxReqQueue()) {
|
|
57
|
+
yield new Promise((resolve) => setTimeout(resolve, Math.random() * 250 + 50));
|
|
58
|
+
}
|
|
59
|
+
queue++; // acquire a slot in the queue
|
|
60
|
+
res = yield makeRequest(url, {});
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
console.error(chalk.red(`[!] Failed to download: ${url}`));
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
queue--;
|
|
67
|
+
}
|
|
68
|
+
// only push it if the file is a JS file
|
|
69
|
+
let file;
|
|
70
|
+
if (!url.match(/\.json/)) {
|
|
71
|
+
file = `// JS Source: ${url}\n${yield res.text()}`;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
file = yield res.text();
|
|
75
|
+
}
|
|
76
|
+
let filename;
|
|
77
|
+
try {
|
|
78
|
+
filename = url
|
|
79
|
+
.split("/")
|
|
80
|
+
.pop()
|
|
81
|
+
.match(/[a-zA-Z0-9\.\-_]+\.js(on)?/)[0];
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
// split the URL into multiple chunks. then iterate
|
|
85
|
+
// through it, and find whatever matches with JS or JSON ext
|
|
86
|
+
const chunks = url.split("/");
|
|
87
|
+
for (const chunk of chunks) {
|
|
88
|
+
if (chunk.match(/\.js(on)?$/)) {
|
|
89
|
+
filename = chunk;
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (!filename) {
|
|
95
|
+
// Handle cases where filename might not be found
|
|
96
|
+
console.warn(chalk.yellow(`[!] Could not determine filename for URL: ${url}. Skipping.`));
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const filePath = path.join(childDir, filename);
|
|
100
|
+
try {
|
|
101
|
+
// if it's json, then use different parser
|
|
102
|
+
if (url.match(/\.json/)) {
|
|
103
|
+
fs.writeFileSync(filePath, yield prettier.format(file, { parser: "json" }));
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
fs.writeFileSync(filePath, yield prettier.format(file, { parser: "babel" }));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
console.error(chalk.red(`[!] Failed to write file: ${filePath}`));
|
|
111
|
+
}
|
|
112
|
+
download_count++;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
console.error(chalk.red(`[!] Failed to download: ${url} : ${err}`));
|
|
117
|
+
}
|
|
118
|
+
}));
|
|
119
|
+
yield Promise.all(downloadPromises);
|
|
120
|
+
if (ignoredJSFiles.length > 0) {
|
|
121
|
+
console.log(chalk.yellow(`[i] Ignored ${ignoredJSFiles.length} JS files across ${ignoredJSDomains.length} domain(s) - ${ignoredJSDomains.join(", ")}`));
|
|
122
|
+
}
|
|
123
|
+
if (download_count > 0) {
|
|
124
|
+
console.log(chalk.green(`[✓] Downloaded ${download_count} JS chunks to ${output} directory`));
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
export default downloadFiles;
|
|
128
|
+
//# sourceMappingURL=downloadFilesUtil.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"downloadFilesUtil.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadFilesUtil.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,WAAW,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC,CAAC,2CAA2C;AAEpG;;;;;;;;GAQG;AACH,MAAM,aAAa,GAAG,CAAO,IAAc,EAAE,MAAc,EAAE,EAAE;IAC3D,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,MAAM,YAAY,CAAC,CACpE,CAAC;IACF,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,6BAA6B;IAC7B,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAE1B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAO,GAAG,EAAE,EAAE;QAC5C,IAAI,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAE,CAAC,CACjD,CAAC;YACF,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC7B,+BAA+B;gBAC/B,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBAEjD,qDAAqD;gBACrD,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC7B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACzB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;4BACnC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAChC,CAAC;wBACD,OAAO;oBACX,CAAC;gBACL,CAAC;gBAED,8CAA8C;gBAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;gBACpD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE5C,IAAI,GAAG,CAAC;gBACR,IAAI,CAAC;oBACD,6DAA6D;oBAC7D,OAAO,KAAK,IAAI,cAAc,EAAE,EAAE,CAAC;wBAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,CAChD,CAAC;oBACN,CAAC;oBACD,KAAK,EAAE,CAAC,CAAC,8BAA8B;oBAEvC,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,CAAC;wBAAS,CAAC;oBACP,KAAK,EAAE,CAAC;gBACZ,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACvB,IAAI,GAAG,iBAAiB,GAAG,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvD,CAAC;qBAAM,CAAC;oBACJ,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,CAAC;gBACD,IAAI,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACD,QAAQ,GAAG,GAAG;yBACT,KAAK,CAAC,GAAG,CAAC;yBACV,GAAG,EAAE;yBACL,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,mDAAmD;oBACnD,4DAA4D;oBAC5D,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACzB,IAAI,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC5B,QAAQ,GAAG,KAAK,CAAC;4BACjB,MAAM;wBACV,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACZ,iDAAiD;oBACjD,OAAO,CAAC,IAAI,CACR,KAAK,CAAC,MAAM,CACR,6CAA6C,GAAG,aAAa,CAChE,CACJ,CAAC;oBACF,OAAO;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC/C,IAAI,CAAC;oBACD,0CAA0C;oBAC1C,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,EAAE,CAAC,aAAa,CACZ,QAAQ,EACR,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,EAAE,CAAC,aAAa,CACZ,QAAQ,EACR,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CACnD,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,KAAK,CAAC,GAAG,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CACrD,CAAC;gBACN,CAAC;gBACD,cAAc,EAAE,CAAC;YACrB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC;IACL,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAEpC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,MAAM,CACR,eAAe,cAAc,CAAC,MAAM,oBAAoB,gBAAgB,CAAC,MAAM,gBAAgB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/H,CACJ,CAAC;IACN,CAAC;IAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CACP,kBAAkB,cAAc,iBAAiB,MAAM,YAAY,CACtE,CACJ,CAAC;IACN,CAAC;AACL,CAAC,CAAA,CAAC;AAEF,eAAe,aAAa,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import chalk from "chalk";
|
|
11
|
+
import puppeteer from "puppeteer";
|
|
12
|
+
/**
|
|
13
|
+
* Downloads all the lazy loaded JS files from a given URL.
|
|
14
|
+
*
|
|
15
|
+
* It opens a headless browser instance, navigates to the given URL, and
|
|
16
|
+
* intercepts all the requests. It checks if the request is a JS file
|
|
17
|
+
* and if it is a GET request. If both conditions are satisfied, the URL
|
|
18
|
+
* is added to the array of URLs. Finally, it closes the browser instance
|
|
19
|
+
* and returns the array of URLs.
|
|
20
|
+
*
|
|
21
|
+
* @param {string} url - The URL of the webpage to fetch and parse.
|
|
22
|
+
* @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
|
|
23
|
+
* absolute URLs pointing to JavaScript files found in the page, or undefined for invalid URL.
|
|
24
|
+
*/
|
|
25
|
+
const downloadLoadedJs = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
26
|
+
if (!url.match(/https?:\/\/[a-zA-Z0-9\._\-]+/)) {
|
|
27
|
+
console.log(chalk.red("[!] Invalid URL"));
|
|
28
|
+
return; // Return undefined as per JSDoc
|
|
29
|
+
}
|
|
30
|
+
const browser = yield puppeteer.launch({
|
|
31
|
+
headless: true,
|
|
32
|
+
});
|
|
33
|
+
const page = yield browser.newPage();
|
|
34
|
+
yield page.setRequestInterception(true);
|
|
35
|
+
let js_urls_local = []; // Use a local variable, not the global one
|
|
36
|
+
page.on("request", (request) => __awaiter(void 0, void 0, void 0, function* () {
|
|
37
|
+
// get the request url
|
|
38
|
+
const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
|
|
39
|
+
// see if the request is a JS file, and is a get request
|
|
40
|
+
if (request.method() === "GET" &&
|
|
41
|
+
req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)) {
|
|
42
|
+
js_urls_local.push(req_url);
|
|
43
|
+
}
|
|
44
|
+
yield request.continue();
|
|
45
|
+
}));
|
|
46
|
+
yield page.goto(url);
|
|
47
|
+
yield browser.close();
|
|
48
|
+
return js_urls_local;
|
|
49
|
+
});
|
|
50
|
+
export default downloadLoadedJs;
|
|
51
|
+
//# sourceMappingURL=downloadLoadedJsUtil.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"downloadLoadedJsUtil.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadLoadedJsUtil.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,SAAS,MAAM,WAAW,CAAC;AAElC;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,CAAO,GAAG,EAAE,EAAE;IACnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,gCAAgC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;QACnC,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAExC,IAAI,aAAa,GAAG,EAAE,CAAC,CAAC,2CAA2C;IACnE,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAO,OAAO,EAAE,EAAE;QACjC,sBAAsB;QACtB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,6CAA6C;QAE5E,wDAAwD;QACxD,IACI,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK;YAC1B,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,EACtD,CAAC;YACC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC7B,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,aAAa,CAAC;AACzB,CAAC,CAAA,CAAC;AAEF,eAAe,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
let scope = [];
|
|
2
|
+
let js_urls = [];
|
|
3
|
+
let json_urls = [];
|
|
4
|
+
let max_req_queue;
|
|
5
|
+
export const getScope = () => scope;
|
|
6
|
+
export const setScope = (newScope) => {
|
|
7
|
+
scope = newScope;
|
|
8
|
+
};
|
|
9
|
+
export const pushToScope = (item) => {
|
|
10
|
+
scope.push(item);
|
|
11
|
+
};
|
|
12
|
+
export const getJsUrls = () => js_urls;
|
|
13
|
+
export const pushToJsUrls = (url) => {
|
|
14
|
+
js_urls.push(url);
|
|
15
|
+
};
|
|
16
|
+
export const getJsonUrls = () => json_urls;
|
|
17
|
+
export const pushToJsonUrls = (url) => json_urls.push(url);
|
|
18
|
+
export const getMaxReqQueue = () => max_req_queue;
|
|
19
|
+
export const setMaxReqQueue = (newMax) => {
|
|
20
|
+
max_req_queue = newMax;
|
|
21
|
+
};
|
|
22
|
+
//# sourceMappingURL=globals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"globals.js","sourceRoot":"","sources":["../../src/lazyLoad/globals.ts"],"names":[],"mappings":"AAAA,IAAI,KAAK,GAAG,EAAE,CAAC;AACf,IAAI,OAAO,GAAa,EAAE,CAAC;AAC3B,IAAI,SAAS,GAAG,EAAE,CAAC;AACnB,IAAI,aAAa,CAAC;AAElB,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;AACpC,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,EAAE;IACjC,KAAK,GAAG,QAAQ,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE;IAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;AACvC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,EAAE;IAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC;AAC3C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEnE,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC;AAClD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,EAAE;IACrC,aAAa,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC"}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import chalk from "chalk";
|
|
11
|
+
import fs from "fs";
|
|
12
|
+
import frameworkDetect from "../techDetect/index.js";
|
|
13
|
+
import CONFIG from "../globalConfig.js";
|
|
14
|
+
import _traverse from "@babel/traverse";
|
|
15
|
+
const traverse = _traverse.default;
|
|
16
|
+
import { URL } from "url";
|
|
17
|
+
// Next.js
|
|
18
|
+
import subsequentRequests from "./next_js/next_SubsequentRequests.js";
|
|
19
|
+
import next_getJSScript from "./next_js/next_GetJSScript.js";
|
|
20
|
+
import next_getLazyResources from "./next_js/next_GetLazyResources.js";
|
|
21
|
+
// Nuxt.js
|
|
22
|
+
import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
|
|
23
|
+
import nuxt_stringAnalysisJSFiles from "./nuxt_js/nuxt_stringAnalysisJSFiles.js";
|
|
24
|
+
import nuxt_astParse from "./nuxt_js/nuxt_astParse.js";
|
|
25
|
+
// Svelte
|
|
26
|
+
import svelte_getFromPageSource from "./svelte/svelte_getFromPageSource.js";
|
|
27
|
+
import svelte_stringAnalysisJSFiles from "./svelte/svelte_stringAnalysisJSFiles.js";
|
|
28
|
+
// generic
|
|
29
|
+
import downloadFiles from "./downloadFilesUtil.js";
|
|
30
|
+
import downloadLoadedJs from "./downloadLoadedJsUtil.js";
|
|
31
|
+
// import global vars
|
|
32
|
+
import * as lazyLoadGlobals from "./globals.js";
|
|
33
|
+
import * as globals from "../utility/globals.js";
|
|
34
|
+
/**
|
|
35
|
+
* Downloads all lazy-loaded JavaScript files from the specified URL or file containing URLs.
|
|
36
|
+
*
|
|
37
|
+
* The function detects the JavaScript framework used by the webpage (e.g., Next.js, Nuxt.js)
|
|
38
|
+
* and utilizes specific techniques to find and download lazy-loaded JS files.
|
|
39
|
+
* It supports subsequent requests for additional JS files if specified.
|
|
40
|
+
*
|
|
41
|
+
* @param {string} url - The URL or path to a file containing a list of URLs to process.
|
|
42
|
+
* @param {string} output - The directory where downloaded files will be saved.
|
|
43
|
+
* @param {boolean} strictScope - Whether to restrict downloads to the input URL domain.
|
|
44
|
+
* @param {string[]} inputScope - Specific domains to download JS files from.
|
|
45
|
+
* @param {number} threads - The number of threads to use for downloading files.
|
|
46
|
+
* @param {boolean} subsequentRequestsFlag - Whether to include JS files from subsequent requests.
|
|
47
|
+
* @param {string} urlsFile - The JSON file containing additional URLs for subsequent requests.
|
|
48
|
+
* @returns {Promise<void>}
|
|
49
|
+
*/
|
|
50
|
+
const lazyLoad = (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile) => __awaiter(void 0, void 0, void 0, function* () {
|
|
51
|
+
console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
|
|
52
|
+
// if cache enabled, check if the cache file exists or not. If no, then create a new one
|
|
53
|
+
if (!globals.getDisableCache()) {
|
|
54
|
+
if (!fs.existsSync(globals.getRespCacheFile())) {
|
|
55
|
+
fs.writeFileSync(globals.getRespCacheFile(), "{}");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
let urls;
|
|
59
|
+
// check if the url is file or a URL
|
|
60
|
+
if (fs.existsSync(url)) {
|
|
61
|
+
urls = fs.readFileSync(url, "utf8").split("\n");
|
|
62
|
+
// remove the empty lines
|
|
63
|
+
urls = urls.filter((url) => url.trim() !== "");
|
|
64
|
+
}
|
|
65
|
+
else if (url.match(/https?:\/\/[a-zA-Z0-9\-_\.:]+/)) {
|
|
66
|
+
urls = [url];
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
console.log(chalk.red("[!] Invalid URL or file path"));
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
for (const url of urls) {
|
|
73
|
+
console.log(chalk.cyan(`[i] Processing ${url}`));
|
|
74
|
+
if (strictScope) {
|
|
75
|
+
lazyLoadGlobals.pushToScope(new URL(url).host);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
lazyLoadGlobals.setScope(inputScope);
|
|
79
|
+
}
|
|
80
|
+
lazyLoadGlobals.setMaxReqQueue(threads);
|
|
81
|
+
const tech = yield frameworkDetect(url);
|
|
82
|
+
globals.setTech(tech ? tech.name : undefined);
|
|
83
|
+
if (tech) {
|
|
84
|
+
if (tech.name === "next") {
|
|
85
|
+
console.log(chalk.green("[✓] Next.js detected"));
|
|
86
|
+
console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
|
|
87
|
+
// find the JS files from script of the webpage
|
|
88
|
+
const jsFilesFromScriptTag = yield next_getJSScript(url);
|
|
89
|
+
// get lazy resources
|
|
90
|
+
const lazyResourcesFromWebpack = yield next_getLazyResources(url);
|
|
91
|
+
let lazyResourcesFromSubsequentRequests;
|
|
92
|
+
if (subsequentRequestsFlag) {
|
|
93
|
+
// get JS files from subsequent requests
|
|
94
|
+
lazyResourcesFromSubsequentRequests =
|
|
95
|
+
yield subsequentRequests(url, urlsFile, threads, output, lazyLoadGlobals.getJsUrls() // Pass the global js_urls
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
// download the resources
|
|
99
|
+
// but combine them first
|
|
100
|
+
let jsFilesToDownload = [
|
|
101
|
+
...(jsFilesFromScriptTag || []),
|
|
102
|
+
...(lazyResourcesFromWebpack || []),
|
|
103
|
+
...(lazyResourcesFromSubsequentRequests || []),
|
|
104
|
+
];
|
|
105
|
+
// Ensure js_urls from globals are included if next_getJSScript or next_getLazyResources populated it.
|
|
106
|
+
// This is because those functions now push to the global js_urls via setters.
|
|
107
|
+
// The return values of next_getJSScript and next_getLazyResources might be the same array instance
|
|
108
|
+
// or a new one depending on their implementation, so explicitly get the global one here.
|
|
109
|
+
jsFilesToDownload.push(...lazyLoadGlobals.getJsUrls());
|
|
110
|
+
// also, download the JSON files, so push those as well into this list
|
|
111
|
+
jsFilesToDownload.push(...lazyLoadGlobals.getJsonUrls());
|
|
112
|
+
// dedupe the files
|
|
113
|
+
jsFilesToDownload = [...new Set(jsFilesToDownload)];
|
|
114
|
+
yield downloadFiles(jsFilesToDownload, output);
|
|
115
|
+
}
|
|
116
|
+
else if (tech.name === "vue") {
|
|
117
|
+
console.log(chalk.green("[✓] Vue.js detected"));
|
|
118
|
+
console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
|
|
119
|
+
}
|
|
120
|
+
else if (tech.name === "nuxt") {
|
|
121
|
+
console.log(chalk.green("[✓] Nuxt.js detected"));
|
|
122
|
+
console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
|
|
123
|
+
let jsFilesToDownload = [];
|
|
124
|
+
// find the files from the page source
|
|
125
|
+
const jsFilesFromPageSource = yield nuxt_getFromPageSource(url);
|
|
126
|
+
const jsFilesFromStringAnalysis = yield nuxt_stringAnalysisJSFiles(url);
|
|
127
|
+
jsFilesToDownload.push(...jsFilesFromPageSource);
|
|
128
|
+
jsFilesToDownload.push(...jsFilesFromStringAnalysis);
|
|
129
|
+
// dedupe the files
|
|
130
|
+
jsFilesToDownload = [...new Set(jsFilesToDownload)];
|
|
131
|
+
let jsFilesFromAST = [];
|
|
132
|
+
console.log(chalk.cyan("[i] Analyzing functions in the files found"));
|
|
133
|
+
for (const jsFile of jsFilesToDownload) {
|
|
134
|
+
jsFilesFromAST.push(...(yield nuxt_astParse(jsFile)));
|
|
135
|
+
}
|
|
136
|
+
jsFilesToDownload.push(...jsFilesFromAST);
|
|
137
|
+
jsFilesToDownload.push(...lazyLoadGlobals.getJsUrls());
|
|
138
|
+
// dedupe the files
|
|
139
|
+
jsFilesToDownload = [...new Set(jsFilesToDownload)];
|
|
140
|
+
yield downloadFiles(jsFilesToDownload, output);
|
|
141
|
+
}
|
|
142
|
+
else if (tech.name === "svelte") {
|
|
143
|
+
console.log(chalk.green("[✓] Svelte detected"));
|
|
144
|
+
console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
|
|
145
|
+
let jsFilesToDownload = [];
|
|
146
|
+
// find the files from the page source
|
|
147
|
+
const jsFilesFromPageSource = yield svelte_getFromPageSource(url);
|
|
148
|
+
jsFilesToDownload.push(...jsFilesFromPageSource);
|
|
149
|
+
// analyze the strings now
|
|
150
|
+
const jsFilesFromStringAnalysis = yield svelte_stringAnalysisJSFiles(url);
|
|
151
|
+
jsFilesToDownload.push(...jsFilesFromStringAnalysis);
|
|
152
|
+
// dedupe the files
|
|
153
|
+
jsFilesToDownload = [...new Set(jsFilesToDownload)];
|
|
154
|
+
yield downloadFiles(jsFilesToDownload, output);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
console.log(chalk.red("[!] Framework not detected :("));
|
|
159
|
+
console.log(chalk.magenta(CONFIG.notFoundMessage));
|
|
160
|
+
console.log(chalk.yellow("[i] Trying to download loaded JS files"));
|
|
161
|
+
const js_urls = yield downloadLoadedJs(url);
|
|
162
|
+
if (js_urls && js_urls.length > 0) {
|
|
163
|
+
console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
|
|
164
|
+
yield downloadFiles(js_urls, output);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
export default lazyLoad;
|
|
170
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/lazyLoad/index.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,eAAe,MAAM,wBAAwB,CAAC;AACrD,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,UAAU;AACV,OAAO,kBAAkB,MAAM,sCAAsC,CAAC;AACtE,OAAO,gBAAgB,MAAM,+BAA+B,CAAC;AAC7D,OAAO,qBAAqB,MAAM,oCAAoC,CAAC;AAEvE,UAAU;AACV,OAAO,sBAAsB,MAAM,qCAAqC,CAAC;AACzE,OAAO,0BAA0B,MAAM,yCAAyC,CAAC;AACjF,OAAO,aAAa,MAAM,4BAA4B,CAAC;AAEvD,SAAS;AACT,OAAO,wBAAwB,MAAM,sCAAsC,CAAC;AAC5E,OAAO,4BAA4B,MAAM,0CAA0C,CAAC;AAEpF,UAAU;AACV,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,gBAAgB,MAAM,2BAA2B,CAAC;AAEzD,qBAAqB;AACrB,OAAO,KAAK,eAAe,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AAEjD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,QAAQ,GAAG,CACb,GAAW,EACX,MAAc,EACd,WAAoB,EACpB,UAAc,EACd,OAAe,EACf,sBAA+B,EAC/B,QAAgB,EAClB,EAAE;IACA,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC,CAAC;IAE1D,wFAAwF;IACxF,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC;IAET,oCAAoC;IACpC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChD,yBAAyB;QACzB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACvD,OAAO;IACX,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC,CAAC;QAEjD,IAAI,WAAW,EAAE,CAAC;YACd,eAAe,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACJ,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAExC,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,+CAA+C;gBAC/C,MAAM,oBAAoB,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBAEzD,qBAAqB;gBACrB,MAAM,wBAAwB,GAC1B,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,mCAAmC,CAAC;gBAExC,IAAI,sBAAsB,EAAE,CAAC;oBACzB,wCAAwC;oBACxC,mCAAmC;wBAC/B,MAAM,kBAAkB,CACpB,GAAG,EACH,QAAQ,EACR,OAAO,EACP,MAAM,EACN,eAAe,CAAC,SAAS,EAAE,CAAC,0BAA0B;yBACzD,CAAC;gBACV,CAAC;gBAED,yBAAyB;gBACzB,yBAAyB;gBACzB,IAAI,iBAAiB,GAAmB;oBACpC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC;oBAC/B,GAAG,CAAC,wBAAwB,IAAI,EAAE,CAAC;oBACnC,GAAG,CAAC,mCAAmC,IAAI,EAAE,CAAC;iBACjD,CAAC;gBACF,sGAAsG;gBACtG,8EAA8E;gBAC9E,mGAAmG;gBACnG,yFAAyF;gBACzF,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvD,sEAAsE;gBACtE,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC,CAAC;gBAEzD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC5D,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,iBAAiB,GAAa,EAAE,CAAC;gBAErC,sCAAsC;gBACtC,MAAM,qBAAqB,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;gBAChE,MAAM,yBAAyB,GAC3B,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;gBAE1C,iBAAiB,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;gBACjD,iBAAiB,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAC;gBACrD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,IAAI,cAAc,GAAG,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAC3D,CAAC;gBACF,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;oBACrC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBAED,iBAAiB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;gBAE1C,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,iBAAiB,GAAG,EAAE,CAAC;gBAE3B,sCAAsC;gBACtC,MAAM,qBAAqB,GACvB,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;gBACxC,iBAAiB,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;gBAEjD,0BAA0B;gBAC1B,MAAM,yBAAyB,GAC3B,MAAM,4BAA4B,CAAC,GAAG,CAAC,CAAC;gBAC5C,iBAAiB,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAC;gBAErD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wCAAwC,CAAC,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CAAC,aAAa,OAAO,CAAC,MAAM,YAAY,CAAC,CACvD,CAAC;gBACF,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC,CAAA,CAAC;AAEF,eAAe,QAAQ,CAAC"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
// lazyLoad/nextGetJSScript.js
|
|
11
|
+
import chalk from "chalk";
|
|
12
|
+
import { URL } from "url";
|
|
13
|
+
import * as cheerio from "cheerio";
|
|
14
|
+
import makeRequest from "../../utility/makeReq.js";
|
|
15
|
+
import { getJsUrls, pushToJsUrls } from "../globals.js";
|
|
16
|
+
/**
|
|
17
|
+
* Asynchronously fetches the given URL and extracts JavaScript file URLs
|
|
18
|
+
* from script tags present in the HTML content.
|
|
19
|
+
*
|
|
20
|
+
* @param {string} url - The URL of the webpage to fetch and parse.
|
|
21
|
+
* @returns {Promise<string[]>} - A promise that resolves to an array of
|
|
22
|
+
* absolute URLs pointing to JavaScript files found in script tags.
|
|
23
|
+
*/
|
|
24
|
+
const next_getJSScript = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
25
|
+
// get the page source
|
|
26
|
+
const res = yield makeRequest(url, {});
|
|
27
|
+
const pageSource = yield res.text();
|
|
28
|
+
// cheerio to parse the page source
|
|
29
|
+
const $ = cheerio.load(pageSource);
|
|
30
|
+
// find all script tags
|
|
31
|
+
const scriptTags = $("script");
|
|
32
|
+
// iterate through script tags
|
|
33
|
+
for (const scriptTag of scriptTags) {
|
|
34
|
+
// get the src attribute
|
|
35
|
+
const src = $(scriptTag).attr("src");
|
|
36
|
+
// see if the src is a JS file
|
|
37
|
+
if (src !== undefined &&
|
|
38
|
+
src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)) {
|
|
39
|
+
// if the src starts with /, like `/static/js/a.js` find the absolute URL
|
|
40
|
+
if (src.startsWith("/")) {
|
|
41
|
+
const absoluteUrl = new URL(url).origin + src;
|
|
42
|
+
if (!getJsUrls().includes(absoluteUrl)) {
|
|
43
|
+
pushToJsUrls(absoluteUrl);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
else if (src.match(/^[^/]/)) {
|
|
47
|
+
// if the src is a relative URL, like `static/js/a.js` find the absolute URL
|
|
48
|
+
// Get directory URL (origin + path without filename)
|
|
49
|
+
const pathParts = new URL(url).pathname.split("/");
|
|
50
|
+
pathParts.pop(); // remove filename from last
|
|
51
|
+
const directory = new URL(url).origin + pathParts.join("/") + "/";
|
|
52
|
+
if (!getJsUrls().includes(directory + src)) {
|
|
53
|
+
pushToJsUrls(directory + src);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
if (!getJsUrls().includes(src)) {
|
|
58
|
+
pushToJsUrls(src);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
// if the script tag is inline, it could contain static JS URL
|
|
64
|
+
// to get these, simply regex from the JS script
|
|
65
|
+
const js_script = $(scriptTag).html();
|
|
66
|
+
const matches = js_script.match(/static\/chunks\/[a-zA-Z0-9_\-]+\.js/g);
|
|
67
|
+
if (matches) {
|
|
68
|
+
const uniqueMatches = [...new Set(matches)];
|
|
69
|
+
for (const match of uniqueMatches) {
|
|
70
|
+
// if it is using that static/chunks/ pattern, I can just get the filename
|
|
71
|
+
const filename = match.replace("static/chunks/", "");
|
|
72
|
+
// go through the already found URLs, coz they will have it (src attribute
|
|
73
|
+
// is there before inline things)
|
|
74
|
+
let js_path_dir;
|
|
75
|
+
for (const js_url of getJsUrls()) {
|
|
76
|
+
if (!js_path_dir &&
|
|
77
|
+
new URL(js_url).host === new URL(url).host &&
|
|
78
|
+
new URL(js_url).pathname.includes("static/chunks/")) {
|
|
79
|
+
js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (js_path_dir) {
|
|
83
|
+
// Ensure js_path_dir was found
|
|
84
|
+
pushToJsUrls(js_path_dir + "/" + filename);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
console.log(chalk.green(`[✓] Found ${getJsUrls().length} JS files from the script tags`));
|
|
91
|
+
return getJsUrls();
|
|
92
|
+
});
|
|
93
|
+
export default next_getJSScript;
|
|
94
|
+
//# sourceMappingURL=next_GetJSScript.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"next_GetJSScript.js","sourceRoot":"","sources":["../../../src/lazyLoad/next_js/next_GetJSScript.ts"],"names":[],"mappings":";;;;;;;;;AAAA,8BAA8B;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAExD;;;;;;;GAOG;AACH,MAAM,gBAAgB,GAAG,CAAO,GAAW,EAAqB,EAAE;IAC9D,sBAAsB;IACtB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAEpC,mCAAmC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEnC,uBAAuB;IACvB,MAAM,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE/B,8BAA8B;IAC9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,wBAAwB;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAErC,8BAA8B;QAC9B,IACI,GAAG,KAAK,SAAS;YACjB,GAAG,CAAC,KAAK,CACL,0DAA0D,CAC7D,EACH,CAAC;YACC,yEAAyE;YACzE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC9C,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBACrC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAC9B,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,4EAA4E;gBAC5E,qDAAqD;gBACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnD,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,4BAA4B;gBAC7C,MAAM,SAAS,GACX,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAEpD,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;oBACzC,YAAY,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC7B,YAAY,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;YACL,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,8DAA8D;YAC9D,gDAAgD;YAEhD,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAC3B,sCAAsC,CACzC,CAAC;YAEF,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC5C,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChC,0EAA0E;oBAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;oBAErD,0EAA0E;oBAC1E,iCAAiC;oBAEjC,IAAI,WAAW,CAAC;oBAEhB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,EAAE,CAAC;wBAC/B,IACI,CAAC,WAAW;4BACZ,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI;4BAC1C,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACrD,CAAC;4BACC,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;wBACxD,CAAC;oBACL,CAAC;oBACD,IAAI,WAAW,EAAE,CAAC;wBACd,+BAA+B;wBAC/B,YAAY,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC;oBAC/C,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CACP,aAAa,SAAS,EAAE,CAAC,MAAM,gCAAgC,CAClE,CACJ,CAAC;IAEF,OAAO,SAAS,EAAE,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF,eAAe,gBAAgB,CAAC"}
|