@shriyanss/js-recon 1.0.0 → 1.1.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.github/workflows/build-and-prettify.yaml +65 -0
  2. package/.github/workflows/npm-publish.yml +35 -0
  3. package/.prettierignore +2 -0
  4. package/.prettierrc +4 -0
  5. package/CHANGELOG.md +50 -0
  6. package/README.md +35 -139
  7. package/build/api_gateway/checkFeasibility.js +32 -0
  8. package/build/api_gateway/checkFeasibility.js.map +1 -0
  9. package/build/api_gateway/checkFireWallBlocking.js +24 -0
  10. package/build/api_gateway/checkFireWallBlocking.js.map +1 -0
  11. package/build/api_gateway/genReq.js +199 -0
  12. package/build/api_gateway/genReq.js.map +1 -0
  13. package/build/api_gateway/index.js +275 -0
  14. package/build/api_gateway/index.js.map +1 -0
  15. package/build/endpoints/gen_report/gen_json.js +22 -0
  16. package/build/endpoints/gen_report/gen_json.js.map +1 -0
  17. package/build/endpoints/gen_report/gen_markdown.js +66 -0
  18. package/build/endpoints/gen_report/gen_markdown.js.map +1 -0
  19. package/build/endpoints/gen_report/utility/iterate_n_store.js +46 -0
  20. package/build/endpoints/gen_report/utility/iterate_n_store.js.map +1 -0
  21. package/build/endpoints/index.js +89 -0
  22. package/build/endpoints/index.js.map +1 -0
  23. package/build/endpoints/next_js/client_jsFilesHref.js +91 -0
  24. package/build/endpoints/next_js/client_jsFilesHref.js.map +1 -0
  25. package/build/endpoints/next_js/client_jsonParse.js +78 -0
  26. package/build/endpoints/next_js/client_jsonParse.js.map +1 -0
  27. package/build/endpoints/next_js/client_subsequentRequests.js +199 -0
  28. package/build/endpoints/next_js/client_subsequentRequests.js.map +1 -0
  29. package/build/endpoints/next_js/getWebpacks.js +45 -0
  30. package/build/endpoints/next_js/getWebpacks.js.map +1 -0
  31. package/build/globalConfig.js +11 -0
  32. package/build/globalConfig.js.map +1 -0
  33. package/build/index.js +166 -0
  34. package/build/index.js.map +1 -0
  35. package/build/lazyLoad/downloadFilesUtil.js +128 -0
  36. package/build/lazyLoad/downloadFilesUtil.js.map +1 -0
  37. package/build/lazyLoad/downloadLoadedJsUtil.js +51 -0
  38. package/build/lazyLoad/downloadLoadedJsUtil.js.map +1 -0
  39. package/build/lazyLoad/globals.js +22 -0
  40. package/build/lazyLoad/globals.js.map +1 -0
  41. package/build/lazyLoad/index.js +170 -0
  42. package/build/lazyLoad/index.js.map +1 -0
  43. package/build/lazyLoad/next_js/next_GetJSScript.js +94 -0
  44. package/build/lazyLoad/next_js/next_GetJSScript.js.map +1 -0
  45. package/build/lazyLoad/next_js/next_GetLazyResources.js +202 -0
  46. package/build/lazyLoad/next_js/next_GetLazyResources.js.map +1 -0
  47. package/build/lazyLoad/next_js/next_SubsequentRequests.js +120 -0
  48. package/build/lazyLoad/next_js/next_SubsequentRequests.js.map +1 -0
  49. package/build/lazyLoad/nuxt_js/nuxt_astParse.js +188 -0
  50. package/build/lazyLoad/nuxt_js/nuxt_astParse.js.map +1 -0
  51. package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js +75 -0
  52. package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js.map +1 -0
  53. package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +94 -0
  54. package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js.map +1 -0
  55. package/build/lazyLoad/svelte/svelte_getFromPageSource.js +68 -0
  56. package/build/lazyLoad/svelte/svelte_getFromPageSource.js.map +1 -0
  57. package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js +95 -0
  58. package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js.map +1 -0
  59. package/build/map/index.js +58 -0
  60. package/build/map/index.js.map +1 -0
  61. package/build/map/next_js/getFetchInstances.js +108 -0
  62. package/build/map/next_js/getFetchInstances.js.map +1 -0
  63. package/build/map/next_js/getWebpackConnections.js +227 -0
  64. package/build/map/next_js/getWebpackConnections.js.map +1 -0
  65. package/build/map/next_js/interactive.js +32 -0
  66. package/build/map/next_js/interactive.js.map +1 -0
  67. package/build/map/next_js/interactive_helpers/commandHandler.js +190 -0
  68. package/build/map/next_js/interactive_helpers/commandHandler.js.map +1 -0
  69. package/build/map/next_js/interactive_helpers/commandHelpers.js +91 -0
  70. package/build/map/next_js/interactive_helpers/commandHelpers.js.map +1 -0
  71. package/build/map/next_js/interactive_helpers/helpMenu.js +11 -0
  72. package/build/map/next_js/interactive_helpers/helpMenu.js.map +1 -0
  73. package/build/map/next_js/interactive_helpers/keybindings.js +80 -0
  74. package/build/map/next_js/interactive_helpers/keybindings.js.map +1 -0
  75. package/build/map/next_js/interactive_helpers/printer.js +17 -0
  76. package/build/map/next_js/interactive_helpers/printer.js.map +1 -0
  77. package/build/map/next_js/interactive_helpers/ui.js +81 -0
  78. package/build/map/next_js/interactive_helpers/ui.js.map +1 -0
  79. package/build/map/next_js/resolveFetch.js +201 -0
  80. package/build/map/next_js/resolveFetch.js.map +1 -0
  81. package/build/run/index.js +62 -0
  82. package/build/run/index.js.map +1 -0
  83. package/build/strings/index.js +238 -0
  84. package/build/strings/index.js.map +1 -0
  85. package/build/strings/openapi.js +55 -0
  86. package/build/strings/openapi.js.map +1 -0
  87. package/build/strings/permutate.js +55 -0
  88. package/build/strings/permutate.js.map +1 -0
  89. package/build/strings/secrets.js +89 -0
  90. package/build/strings/secrets.js.map +1 -0
  91. package/build/techDetect/index.js +229 -0
  92. package/build/techDetect/index.js.map +1 -0
  93. package/build/utility/ai.js +69 -0
  94. package/build/utility/ai.js.map +1 -0
  95. package/build/utility/globals.js +84 -0
  96. package/build/utility/globals.js.map +1 -0
  97. package/build/utility/interfaces.js +2 -0
  98. package/build/utility/interfaces.js.map +1 -0
  99. package/build/utility/makeReq.js +265 -0
  100. package/build/utility/makeReq.js.map +1 -0
  101. package/build/utility/resolvePath.js +44 -0
  102. package/build/utility/resolvePath.js.map +1 -0
  103. package/{utility → build/utility}/runSandboxed.js +10 -13
  104. package/build/utility/runSandboxed.js.map +1 -0
  105. package/{utility → build/utility}/urlUtils.js +9 -11
  106. package/build/utility/urlUtils.js.map +1 -0
  107. package/docs/CNAME +1 -0
  108. package/docs/README.md +20 -0
  109. package/docs/api-gateway.md +68 -0
  110. package/docs/endpoints.md +49 -0
  111. package/docs/example-scenario.md +258 -0
  112. package/docs/interactive-mode.md +76 -0
  113. package/docs/lazyload.md +56 -0
  114. package/docs/map.md +53 -0
  115. package/docs/run.md +54 -0
  116. package/docs/strings.md +75 -0
  117. package/package.json +50 -38
  118. package/api_gateway/checkFeasibility.js +0 -25
  119. package/api_gateway/checkFireWallBlocking.js +0 -17
  120. package/api_gateway/genReq.js +0 -214
  121. package/api_gateway/index.js +0 -325
  122. package/endpoints/index.js +0 -7
  123. package/globalConfig.js +0 -12
  124. package/index.js +0 -69
  125. package/lazyLoad/downloadFilesUtil.js +0 -122
  126. package/lazyLoad/downloadLoadedJsUtil.js +0 -54
  127. package/lazyLoad/globals.js +0 -15
  128. package/lazyLoad/index.js +0 -167
  129. package/lazyLoad/next_js/next_GetJSScript.js +0 -99
  130. package/lazyLoad/next_js/next_GetLazyResources.js +0 -201
  131. package/lazyLoad/next_js/next_SubsequentRequests.js +0 -138
  132. package/lazyLoad/nuxt_js/nuxt_astParse.js +0 -194
  133. package/lazyLoad/nuxt_js/nuxt_getFromPageSource.js +0 -77
  134. package/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +0 -99
  135. package/research/firewall_bypass.md +0 -38
  136. package/research/next_js.md +0 -116
  137. package/research/nuxt_js.md +0 -125
  138. package/research/vue_js.md +0 -9
  139. package/strings/index.js +0 -145
  140. package/techDetect/index.js +0 -156
  141. package/utility/globals.js +0 -6
  142. package/utility/makeReq.js +0 -179
  143. package/utility/resolvePath.js +0 -43
@@ -0,0 +1,128 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import chalk from "chalk";
11
+ import path from "path";
12
+ import fs from "fs";
13
+ import prettier from "prettier";
14
+ import makeRequest from "../utility/makeReq.js";
15
+ import { getURLDirectory } from "../utility/urlUtils.js";
16
+ import { getScope, getMaxReqQueue } from "./globals.js"; // Import scope and max_req_queue functions
17
+ /**
18
+ * Downloads a list of URLs and saves them as files in the specified output directory.
19
+ * It creates the necessary subdirectories based on the URL's host and path.
20
+ * If the URL does not end with `.js`, it is skipped.
21
+ * The function logs the progress and any errors to the console.
22
+ * @param {string[]} urls - An array of URLs to be downloaded.
23
+ * @param {string} output - The directory where the downloaded files will be saved.
24
+ * @returns {Promise<void>}
25
+ */
26
+ const downloadFiles = (urls, output) => __awaiter(void 0, void 0, void 0, function* () {
27
+ console.log(chalk.cyan(`[i] Attempting to download ${urls.length} JS chunks`));
28
+ fs.mkdirSync(output, { recursive: true });
29
+ // to store ignored JS domain
30
+ let ignoredJSFiles = [];
31
+ let ignoredJSDomains = [];
32
+ let download_count = 0;
33
+ let queue = 0;
34
+ const downloadPromises = urls.map((url) => __awaiter(void 0, void 0, void 0, function* () {
35
+ try {
36
+ yield new Promise((resolve) => setTimeout(resolve, Math.random() * 4950 + 50));
37
+ if (url.match(/(\.js|\.json)/)) {
38
+ // get the directory of the url
39
+ const { host, directory } = getURLDirectory(url);
40
+ // check scope of file. Only if in scope, download it
41
+ if (!getScope().includes("*")) {
42
+ if (!getScope().includes(host)) {
43
+ ignoredJSFiles.push(url);
44
+ if (!ignoredJSDomains.includes(host)) {
45
+ ignoredJSDomains.push(host);
46
+ }
47
+ return;
48
+ }
49
+ }
50
+ // make the directory inside the output folder
51
+ const childDir = path.join(output, host, directory);
52
+ fs.mkdirSync(childDir, { recursive: true });
53
+ let res;
54
+ try {
55
+ // Wait until there is an available slot in the request queue
56
+ while (queue >= getMaxReqQueue()) {
57
+ yield new Promise((resolve) => setTimeout(resolve, Math.random() * 250 + 50));
58
+ }
59
+ queue++; // acquire a slot in the queue
60
+ res = yield makeRequest(url, {});
61
+ }
62
+ catch (err) {
63
+ console.error(chalk.red(`[!] Failed to download: ${url}`));
64
+ }
65
+ finally {
66
+ queue--;
67
+ }
68
+ // only push it if the file is a JS file
69
+ let file;
70
+ if (!url.match(/\.json/)) {
71
+ file = `// JS Source: ${url}\n${yield res.text()}`;
72
+ }
73
+ else {
74
+ file = yield res.text();
75
+ }
76
+ let filename;
77
+ try {
78
+ filename = url
79
+ .split("/")
80
+ .pop()
81
+ .match(/[a-zA-Z0-9\.\-_]+\.js(on)?/)[0];
82
+ }
83
+ catch (err) {
84
+ // split the URL into multiple chunks. then iterate
85
+ // through it, and find whatever matches with JS or JSON ext
86
+ const chunks = url.split("/");
87
+ for (const chunk of chunks) {
88
+ if (chunk.match(/\.js(on)?$/)) {
89
+ filename = chunk;
90
+ break;
91
+ }
92
+ }
93
+ }
94
+ if (!filename) {
95
+ // Handle cases where filename might not be found
96
+ console.warn(chalk.yellow(`[!] Could not determine filename for URL: ${url}. Skipping.`));
97
+ return;
98
+ }
99
+ const filePath = path.join(childDir, filename);
100
+ try {
101
+ // if it's json, then use different parser
102
+ if (url.match(/\.json/)) {
103
+ fs.writeFileSync(filePath, yield prettier.format(file, { parser: "json" }));
104
+ }
105
+ else {
106
+ fs.writeFileSync(filePath, yield prettier.format(file, { parser: "babel" }));
107
+ }
108
+ }
109
+ catch (err) {
110
+ console.error(chalk.red(`[!] Failed to write file: ${filePath}`));
111
+ }
112
+ download_count++;
113
+ }
114
+ }
115
+ catch (err) {
116
+ console.error(chalk.red(`[!] Failed to download: ${url} : ${err}`));
117
+ }
118
+ }));
119
+ yield Promise.all(downloadPromises);
120
+ if (ignoredJSFiles.length > 0) {
121
+ console.log(chalk.yellow(`[i] Ignored ${ignoredJSFiles.length} JS files across ${ignoredJSDomains.length} domain(s) - ${ignoredJSDomains.join(", ")}`));
122
+ }
123
+ if (download_count > 0) {
124
+ console.log(chalk.green(`[✓] Downloaded ${download_count} JS chunks to ${output} directory`));
125
+ }
126
+ });
127
+ export default downloadFiles;
128
+ //# sourceMappingURL=downloadFilesUtil.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"downloadFilesUtil.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadFilesUtil.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,WAAW,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC,CAAC,2CAA2C;AAEpG;;;;;;;;GAQG;AACH,MAAM,aAAa,GAAG,CAAO,IAAc,EAAE,MAAc,EAAE,EAAE;IAC3D,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,MAAM,YAAY,CAAC,CACpE,CAAC;IACF,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,6BAA6B;IAC7B,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAE1B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAO,GAAG,EAAE,EAAE;QAC5C,IAAI,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAE,CAAC,CACjD,CAAC;YACF,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC7B,+BAA+B;gBAC/B,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBAEjD,qDAAqD;gBACrD,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC7B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACzB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;4BACnC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAChC,CAAC;wBACD,OAAO;oBACX,CAAC;gBACL,CAAC;gBAED,8CAA8C;gBAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;gBACpD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE5C,IAAI,GAAG,CAAC;gBACR,IAAI,CAAC;oBACD,6DAA6D;oBAC7D,OAAO,KAAK,IAAI,cAAc,EAAE,EAAE,CAAC;wBAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,CAChD,CAAC;oBACN,CAAC;oBACD,KAAK,EAAE,CAAC,CAAC,8BAA8B;oBAEvC,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,CAAC;wBAAS,CAAC;oBACP,KAAK,EAAE,CAAC;gBACZ,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACvB,IAAI,GAAG,iBAAiB,GAAG,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvD,CAAC;qBAAM,CAAC;oBACJ,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5B,CAAC;gBACD,IAAI,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACD,QAAQ,GAAG,GAAG;yBACT,KAAK,CAAC,GAAG,CAAC;yBACV,GAAG,EAAE;yBACL,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,mDAAmD;oBACnD,4DAA4D;oBAC5D,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACzB,IAAI,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC5B,QAAQ,GAAG,KAAK,CAAC;4BACjB,MAAM;wBACV,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACZ,iDAAiD;oBACjD,OAAO,CAAC,IAAI,CACR,KAAK,CAAC,MAAM,CACR,6CAA6C,GAAG,aAAa,CAChE,CACJ,CAAC;oBACF,OAAO;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC/C,IAAI,CAAC;oBACD,0CAA0C;oBAC1C,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,EAAE,CAAC,aAAa,CACZ,QAAQ,EACR,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,EAAE,CAAC,aAAa,CACZ,QAAQ,EACR,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CACnD,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,KAAK,CAAC,GAAG,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CACrD,CAAC;gBACN,CAAC;gBACD,cAAc,EAAE,CAAC;YACrB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC;IACL,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAEpC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,MAAM,CACR,eAAe,cAAc,CAAC,MAAM,oBAAoB,gBAAgB,CAAC,MAAM,gBAAgB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/H,CACJ,CAAC;IACN,CAAC;IAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CACP,kBAAkB,cAAc,iBAAiB,MAAM,YAAY,CACtE,CACJ,CAAC;IACN,CAAC;AACL,CAAC,CAAA,CAAC;AAEF,eAAe,aAAa,CAAC"}
@@ -0,0 +1,51 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import chalk from "chalk";
11
+ import puppeteer from "puppeteer";
12
+ /**
13
+ * Downloads all the lazy loaded JS files from a given URL.
14
+ *
15
+ * It opens a headless browser instance, navigates to the given URL, and
16
+ * intercepts all the requests. It checks if the request is a JS file
17
+ * and if it is a GET request. If both conditions are satisfied, the URL
18
+ * is added to the array of URLs. Finally, it closes the browser instance
19
+ * and returns the array of URLs.
20
+ *
21
+ * @param {string} url - The URL of the webpage to fetch and parse.
22
+ * @returns {Promise<string[]|undefined>} - A promise that resolves to an array of
23
+ * absolute URLs pointing to JavaScript files found in the page, or undefined for invalid URL.
24
+ */
25
+ const downloadLoadedJs = (url) => __awaiter(void 0, void 0, void 0, function* () {
26
+ if (!url.match(/https?:\/\/[a-zA-Z0-9\._\-]+/)) {
27
+ console.log(chalk.red("[!] Invalid URL"));
28
+ return; // Return undefined as per JSDoc
29
+ }
30
+ const browser = yield puppeteer.launch({
31
+ headless: true,
32
+ });
33
+ const page = yield browser.newPage();
34
+ yield page.setRequestInterception(true);
35
+ let js_urls_local = []; // Use a local variable, not the global one
36
+ page.on("request", (request) => __awaiter(void 0, void 0, void 0, function* () {
37
+ // get the request url
38
+ const req_url = request.url(); // Renamed to avoid conflict with outer 'url'
39
+ // see if the request is a JS file, and is a get request
40
+ if (request.method() === "GET" &&
41
+ req_url.match(/https?:\/\/[a-z\._\-]+\/.+\.js\??.*/)) {
42
+ js_urls_local.push(req_url);
43
+ }
44
+ yield request.continue();
45
+ }));
46
+ yield page.goto(url);
47
+ yield browser.close();
48
+ return js_urls_local;
49
+ });
50
+ export default downloadLoadedJs;
51
+ //# sourceMappingURL=downloadLoadedJsUtil.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"downloadLoadedJsUtil.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadLoadedJsUtil.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,SAAS,MAAM,WAAW,CAAC;AAElC;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,CAAO,GAAG,EAAE,EAAE;IACnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,gCAAgC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;QACnC,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAExC,IAAI,aAAa,GAAG,EAAE,CAAC,CAAC,2CAA2C;IACnE,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAO,OAAO,EAAE,EAAE;QACjC,sBAAsB;QACtB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,6CAA6C;QAE5E,wDAAwD;QACxD,IACI,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK;YAC1B,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,EACtD,CAAC;YACC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC7B,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,aAAa,CAAC;AACzB,CAAC,CAAA,CAAC;AAEF,eAAe,gBAAgB,CAAC"}
@@ -0,0 +1,22 @@
1
+ let scope = [];
2
+ let js_urls = [];
3
+ let json_urls = [];
4
+ let max_req_queue;
5
+ export const getScope = () => scope;
6
+ export const setScope = (newScope) => {
7
+ scope = newScope;
8
+ };
9
+ export const pushToScope = (item) => {
10
+ scope.push(item);
11
+ };
12
+ export const getJsUrls = () => js_urls;
13
+ export const pushToJsUrls = (url) => {
14
+ js_urls.push(url);
15
+ };
16
+ export const getJsonUrls = () => json_urls;
17
+ export const pushToJsonUrls = (url) => json_urls.push(url);
18
+ export const getMaxReqQueue = () => max_req_queue;
19
+ export const setMaxReqQueue = (newMax) => {
20
+ max_req_queue = newMax;
21
+ };
22
+ //# sourceMappingURL=globals.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"globals.js","sourceRoot":"","sources":["../../src/lazyLoad/globals.ts"],"names":[],"mappings":"AAAA,IAAI,KAAK,GAAG,EAAE,CAAC;AACf,IAAI,OAAO,GAAa,EAAE,CAAC;AAC3B,IAAI,SAAS,GAAG,EAAE,CAAC;AACnB,IAAI,aAAa,CAAC;AAElB,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;AACpC,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,EAAE;IACjC,KAAK,GAAG,QAAQ,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE;IAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;AACvC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,EAAE;IAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC;AAC3C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEnE,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,aAAa,CAAC;AAClD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,EAAE;IACrC,aAAa,GAAG,MAAM,CAAC;AAC3B,CAAC,CAAC"}
@@ -0,0 +1,170 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import chalk from "chalk";
11
+ import fs from "fs";
12
+ import frameworkDetect from "../techDetect/index.js";
13
+ import CONFIG from "../globalConfig.js";
14
+ import _traverse from "@babel/traverse";
15
+ const traverse = _traverse.default;
16
+ import { URL } from "url";
17
+ // Next.js
18
+ import subsequentRequests from "./next_js/next_SubsequentRequests.js";
19
+ import next_getJSScript from "./next_js/next_GetJSScript.js";
20
+ import next_getLazyResources from "./next_js/next_GetLazyResources.js";
21
+ // Nuxt.js
22
+ import nuxt_getFromPageSource from "./nuxt_js/nuxt_getFromPageSource.js";
23
+ import nuxt_stringAnalysisJSFiles from "./nuxt_js/nuxt_stringAnalysisJSFiles.js";
24
+ import nuxt_astParse from "./nuxt_js/nuxt_astParse.js";
25
+ // Svelte
26
+ import svelte_getFromPageSource from "./svelte/svelte_getFromPageSource.js";
27
+ import svelte_stringAnalysisJSFiles from "./svelte/svelte_stringAnalysisJSFiles.js";
28
+ // generic
29
+ import downloadFiles from "./downloadFilesUtil.js";
30
+ import downloadLoadedJs from "./downloadLoadedJsUtil.js";
31
+ // import global vars
32
+ import * as lazyLoadGlobals from "./globals.js";
33
+ import * as globals from "../utility/globals.js";
34
+ /**
35
+ * Downloads all lazy-loaded JavaScript files from the specified URL or file containing URLs.
36
+ *
37
+ * The function detects the JavaScript framework used by the webpage (e.g., Next.js, Nuxt.js)
38
+ * and utilizes specific techniques to find and download lazy-loaded JS files.
39
+ * It supports subsequent requests for additional JS files if specified.
40
+ *
41
+ * @param {string} url - The URL or path to a file containing a list of URLs to process.
42
+ * @param {string} output - The directory where downloaded files will be saved.
43
+ * @param {boolean} strictScope - Whether to restrict downloads to the input URL domain.
44
+ * @param {string[]} inputScope - Specific domains to download JS files from.
45
+ * @param {number} threads - The number of threads to use for downloading files.
46
+ * @param {boolean} subsequentRequestsFlag - Whether to include JS files from subsequent requests.
47
+ * @param {string} urlsFile - The JSON file containing additional URLs for subsequent requests.
48
+ * @returns {Promise<void>}
49
+ */
50
+ const lazyLoad = (url, output, strictScope, inputScope, threads, subsequentRequestsFlag, urlsFile) => __awaiter(void 0, void 0, void 0, function* () {
51
+ console.log(chalk.cyan("[i] Loading 'Lazy Load' module"));
52
+ // if cache enabled, check if the cache file exists or not. If no, then create a new one
53
+ if (!globals.getDisableCache()) {
54
+ if (!fs.existsSync(globals.getRespCacheFile())) {
55
+ fs.writeFileSync(globals.getRespCacheFile(), "{}");
56
+ }
57
+ }
58
+ let urls;
59
+ // check if the url is file or a URL
60
+ if (fs.existsSync(url)) {
61
+ urls = fs.readFileSync(url, "utf8").split("\n");
62
+ // remove the empty lines
63
+ urls = urls.filter((url) => url.trim() !== "");
64
+ }
65
+ else if (url.match(/https?:\/\/[a-zA-Z0-9\-_\.:]+/)) {
66
+ urls = [url];
67
+ }
68
+ else {
69
+ console.log(chalk.red("[!] Invalid URL or file path"));
70
+ return;
71
+ }
72
+ for (const url of urls) {
73
+ console.log(chalk.cyan(`[i] Processing ${url}`));
74
+ if (strictScope) {
75
+ lazyLoadGlobals.pushToScope(new URL(url).host);
76
+ }
77
+ else {
78
+ lazyLoadGlobals.setScope(inputScope);
79
+ }
80
+ lazyLoadGlobals.setMaxReqQueue(threads);
81
+ const tech = yield frameworkDetect(url);
82
+ globals.setTech(tech ? tech.name : undefined);
83
+ if (tech) {
84
+ if (tech.name === "next") {
85
+ console.log(chalk.green("[✓] Next.js detected"));
86
+ console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
87
+ // find the JS files from script of the webpage
88
+ const jsFilesFromScriptTag = yield next_getJSScript(url);
89
+ // get lazy resources
90
+ const lazyResourcesFromWebpack = yield next_getLazyResources(url);
91
+ let lazyResourcesFromSubsequentRequests;
92
+ if (subsequentRequestsFlag) {
93
+ // get JS files from subsequent requests
94
+ lazyResourcesFromSubsequentRequests =
95
+ yield subsequentRequests(url, urlsFile, threads, output, lazyLoadGlobals.getJsUrls() // Pass the global js_urls
96
+ );
97
+ }
98
+ // download the resources
99
+ // but combine them first
100
+ let jsFilesToDownload = [
101
+ ...(jsFilesFromScriptTag || []),
102
+ ...(lazyResourcesFromWebpack || []),
103
+ ...(lazyResourcesFromSubsequentRequests || []),
104
+ ];
105
+ // Ensure js_urls from globals are included if next_getJSScript or next_getLazyResources populated it.
106
+ // This is because those functions now push to the global js_urls via setters.
107
+ // The return values of next_getJSScript and next_getLazyResources might be the same array instance
108
+ // or a new one depending on their implementation, so explicitly get the global one here.
109
+ jsFilesToDownload.push(...lazyLoadGlobals.getJsUrls());
110
+ // also, download the JSON files, so push those as well into this list
111
+ jsFilesToDownload.push(...lazyLoadGlobals.getJsonUrls());
112
+ // dedupe the files
113
+ jsFilesToDownload = [...new Set(jsFilesToDownload)];
114
+ yield downloadFiles(jsFilesToDownload, output);
115
+ }
116
+ else if (tech.name === "vue") {
117
+ console.log(chalk.green("[✓] Vue.js detected"));
118
+ console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
119
+ }
120
+ else if (tech.name === "nuxt") {
121
+ console.log(chalk.green("[✓] Nuxt.js detected"));
122
+ console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
123
+ let jsFilesToDownload = [];
124
+ // find the files from the page source
125
+ const jsFilesFromPageSource = yield nuxt_getFromPageSource(url);
126
+ const jsFilesFromStringAnalysis = yield nuxt_stringAnalysisJSFiles(url);
127
+ jsFilesToDownload.push(...jsFilesFromPageSource);
128
+ jsFilesToDownload.push(...jsFilesFromStringAnalysis);
129
+ // dedupe the files
130
+ jsFilesToDownload = [...new Set(jsFilesToDownload)];
131
+ let jsFilesFromAST = [];
132
+ console.log(chalk.cyan("[i] Analyzing functions in the files found"));
133
+ for (const jsFile of jsFilesToDownload) {
134
+ jsFilesFromAST.push(...(yield nuxt_astParse(jsFile)));
135
+ }
136
+ jsFilesToDownload.push(...jsFilesFromAST);
137
+ jsFilesToDownload.push(...lazyLoadGlobals.getJsUrls());
138
+ // dedupe the files
139
+ jsFilesToDownload = [...new Set(jsFilesToDownload)];
140
+ yield downloadFiles(jsFilesToDownload, output);
141
+ }
142
+ else if (tech.name === "svelte") {
143
+ console.log(chalk.green("[✓] Svelte detected"));
144
+ console.log(chalk.yellow(`Evidence: ${tech.evidence}`));
145
+ let jsFilesToDownload = [];
146
+ // find the files from the page source
147
+ const jsFilesFromPageSource = yield svelte_getFromPageSource(url);
148
+ jsFilesToDownload.push(...jsFilesFromPageSource);
149
+ // analyze the strings now
150
+ const jsFilesFromStringAnalysis = yield svelte_stringAnalysisJSFiles(url);
151
+ jsFilesToDownload.push(...jsFilesFromStringAnalysis);
152
+ // dedupe the files
153
+ jsFilesToDownload = [...new Set(jsFilesToDownload)];
154
+ yield downloadFiles(jsFilesToDownload, output);
155
+ }
156
+ }
157
+ else {
158
+ console.log(chalk.red("[!] Framework not detected :("));
159
+ console.log(chalk.magenta(CONFIG.notFoundMessage));
160
+ console.log(chalk.yellow("[i] Trying to download loaded JS files"));
161
+ const js_urls = yield downloadLoadedJs(url);
162
+ if (js_urls && js_urls.length > 0) {
163
+ console.log(chalk.green(`[✓] Found ${js_urls.length} JS chunks`));
164
+ yield downloadFiles(js_urls, output);
165
+ }
166
+ }
167
+ }
168
+ });
169
+ export default lazyLoad;
170
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/lazyLoad/index.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,eAAe,MAAM,wBAAwB,CAAC;AACrD,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,UAAU;AACV,OAAO,kBAAkB,MAAM,sCAAsC,CAAC;AACtE,OAAO,gBAAgB,MAAM,+BAA+B,CAAC;AAC7D,OAAO,qBAAqB,MAAM,oCAAoC,CAAC;AAEvE,UAAU;AACV,OAAO,sBAAsB,MAAM,qCAAqC,CAAC;AACzE,OAAO,0BAA0B,MAAM,yCAAyC,CAAC;AACjF,OAAO,aAAa,MAAM,4BAA4B,CAAC;AAEvD,SAAS;AACT,OAAO,wBAAwB,MAAM,sCAAsC,CAAC;AAC5E,OAAO,4BAA4B,MAAM,0CAA0C,CAAC;AAEpF,UAAU;AACV,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,gBAAgB,MAAM,2BAA2B,CAAC;AAEzD,qBAAqB;AACrB,OAAO,KAAK,eAAe,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AAEjD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,QAAQ,GAAG,CACb,GAAW,EACX,MAAc,EACd,WAAoB,EACpB,UAAc,EACd,OAAe,EACf,sBAA+B,EAC/B,QAAgB,EAClB,EAAE;IACA,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC,CAAC;IAE1D,wFAAwF;IACxF,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC;IAET,oCAAoC;IACpC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChD,yBAAyB;QACzB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACvD,OAAO;IACX,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC,CAAC;QAEjD,IAAI,WAAW,EAAE,CAAC;YACd,eAAe,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACJ,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAExC,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,+CAA+C;gBAC/C,MAAM,oBAAoB,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBAEzD,qBAAqB;gBACrB,MAAM,wBAAwB,GAC1B,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,mCAAmC,CAAC;gBAExC,IAAI,sBAAsB,EAAE,CAAC;oBACzB,wCAAwC;oBACxC,mCAAmC;wBAC/B,MAAM,kBAAkB,CACpB,GAAG,EACH,QAAQ,EACR,OAAO,EACP,MAAM,EACN,eAAe,CAAC,SAAS,EAAE,CAAC,0BAA0B;yBACzD,CAAC;gBACV,CAAC;gBAED,yBAAyB;gBACzB,yBAAyB;gBACzB,IAAI,iBAAiB,GAAmB;oBACpC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC;oBAC/B,GAAG,CAAC,wBAAwB,IAAI,EAAE,CAAC;oBACnC,GAAG,CAAC,mCAAmC,IAAI,EAAE,CAAC;iBACjD,CAAC;gBACF,sGAAsG;gBACtG,8EAA8E;gBAC9E,mGAAmG;gBACnG,yFAAyF;gBACzF,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvD,sEAAsE;gBACtE,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC,CAAC;gBAEzD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC5D,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,iBAAiB,GAAa,EAAE,CAAC;gBAErC,sCAAsC;gBACtC,MAAM,qBAAqB,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;gBAChE,MAAM,yBAAyB,GAC3B,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;gBAE1C,iBAAiB,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;gBACjD,iBAAiB,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAC;gBACrD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,IAAI,cAAc,GAAG,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAC3D,CAAC;gBACF,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;oBACrC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBAED,iBAAiB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;gBAE1C,iBAAiB,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,iBAAiB,GAAG,EAAE,CAAC;gBAE3B,sCAAsC;gBACtC,MAAM,qBAAqB,GACvB,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;gBACxC,iBAAiB,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;gBAEjD,0BAA0B;gBAC1B,MAAM,yBAAyB,GAC3B,MAAM,4BAA4B,CAAC,GAAG,CAAC,CAAC;gBAC5C,iBAAiB,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAC;gBAErD,mBAAmB;gBACnB,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAEpD,MAAM,aAAa,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,wCAAwC,CAAC,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CAAC,aAAa,OAAO,CAAC,MAAM,YAAY,CAAC,CACvD,CAAC;gBACF,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC,CAAA,CAAC;AAEF,eAAe,QAAQ,CAAC"}
@@ -0,0 +1,94 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ // lazyLoad/nextGetJSScript.js
11
+ import chalk from "chalk";
12
+ import { URL } from "url";
13
+ import * as cheerio from "cheerio";
14
+ import makeRequest from "../../utility/makeReq.js";
15
+ import { getJsUrls, pushToJsUrls } from "../globals.js";
16
+ /**
17
+ * Asynchronously fetches the given URL and extracts JavaScript file URLs
18
+ * from script tags present in the HTML content.
19
+ *
20
+ * @param {string} url - The URL of the webpage to fetch and parse.
21
+ * @returns {Promise<string[]>} - A promise that resolves to an array of
22
+ * absolute URLs pointing to JavaScript files found in script tags.
23
+ */
24
+ const next_getJSScript = (url) => __awaiter(void 0, void 0, void 0, function* () {
25
+ // get the page source
26
+ const res = yield makeRequest(url, {});
27
+ const pageSource = yield res.text();
28
+ // cheerio to parse the page source
29
+ const $ = cheerio.load(pageSource);
30
+ // find all script tags
31
+ const scriptTags = $("script");
32
+ // iterate through script tags
33
+ for (const scriptTag of scriptTags) {
34
+ // get the src attribute
35
+ const src = $(scriptTag).attr("src");
36
+ // see if the src is a JS file
37
+ if (src !== undefined &&
38
+ src.match(/(https:\/\/[a-zA-Z0-9_\_\.]+\/.+\.js\??.*|\/.+\.js\??.*)/)) {
39
+ // if the src starts with /, like `/static/js/a.js` find the absolute URL
40
+ if (src.startsWith("/")) {
41
+ const absoluteUrl = new URL(url).origin + src;
42
+ if (!getJsUrls().includes(absoluteUrl)) {
43
+ pushToJsUrls(absoluteUrl);
44
+ }
45
+ }
46
+ else if (src.match(/^[^/]/)) {
47
+ // if the src is a relative URL, like `static/js/a.js` find the absolute URL
48
+ // Get directory URL (origin + path without filename)
49
+ const pathParts = new URL(url).pathname.split("/");
50
+ pathParts.pop(); // remove filename from last
51
+ const directory = new URL(url).origin + pathParts.join("/") + "/";
52
+ if (!getJsUrls().includes(directory + src)) {
53
+ pushToJsUrls(directory + src);
54
+ }
55
+ }
56
+ else {
57
+ if (!getJsUrls().includes(src)) {
58
+ pushToJsUrls(src);
59
+ }
60
+ }
61
+ }
62
+ else {
63
+ // if the script tag is inline, it could contain static JS URL
64
+ // to get these, simply regex from the JS script
65
+ const js_script = $(scriptTag).html();
66
+ const matches = js_script.match(/static\/chunks\/[a-zA-Z0-9_\-]+\.js/g);
67
+ if (matches) {
68
+ const uniqueMatches = [...new Set(matches)];
69
+ for (const match of uniqueMatches) {
70
+ // if it is using that static/chunks/ pattern, I can just get the filename
71
+ const filename = match.replace("static/chunks/", "");
72
+ // go through the already found URLs, coz they will have it (src attribute
73
+ // is there before inline things)
74
+ let js_path_dir;
75
+ for (const js_url of getJsUrls()) {
76
+ if (!js_path_dir &&
77
+ new URL(js_url).host === new URL(url).host &&
78
+ new URL(js_url).pathname.includes("static/chunks/")) {
79
+ js_path_dir = js_url.replace(/\/[^\/]+\.js.*$/, "");
80
+ }
81
+ }
82
+ if (js_path_dir) {
83
+ // Ensure js_path_dir was found
84
+ pushToJsUrls(js_path_dir + "/" + filename);
85
+ }
86
+ }
87
+ }
88
+ }
89
+ }
90
+ console.log(chalk.green(`[✓] Found ${getJsUrls().length} JS files from the script tags`));
91
+ return getJsUrls();
92
+ });
93
+ export default next_getJSScript;
94
+ //# sourceMappingURL=next_GetJSScript.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"next_GetJSScript.js","sourceRoot":"","sources":["../../../src/lazyLoad/next_js/next_GetJSScript.ts"],"names":[],"mappings":";;;;;;;;;AAAA,8BAA8B;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAExD;;;;;;;GAOG;AACH,MAAM,gBAAgB,GAAG,CAAO,GAAW,EAAqB,EAAE;IAC9D,sBAAsB;IACtB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAEpC,mCAAmC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEnC,uBAAuB;IACvB,MAAM,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE/B,8BAA8B;IAC9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,wBAAwB;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAErC,8BAA8B;QAC9B,IACI,GAAG,KAAK,SAAS;YACjB,GAAG,CAAC,KAAK,CACL,0DAA0D,CAC7D,EACH,CAAC;YACC,yEAAyE;YACzE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC9C,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBACrC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAC9B,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,4EAA4E;gBAC5E,qDAAqD;gBACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnD,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,4BAA4B;gBAC7C,MAAM,SAAS,GACX,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAEpD,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;oBACzC,YAAY,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC7B,YAAY,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;YACL,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,8DAA8D;YAC9D,gDAAgD;YAEhD,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAC3B,sCAAsC,CACzC,CAAC;YAEF,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC5C,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChC,0EAA0E;oBAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;oBAErD,0EAA0E;oBAC1E,iCAAiC;oBAEjC,IAAI,WAAW,CAAC;oBAEhB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,EAAE,CAAC;wBAC/B,IACI,CAAC,WAAW;4BACZ,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI;4BAC1C,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACrD,CAAC;4BACC,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;wBACxD,CAAC;oBACL,CAAC;oBACD,IAAI,WAAW,EAAE,CAAC;wBACd,+BAA+B;wBAC/B,YAAY,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC;oBAC/C,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,CACP,KAAK,CAAC,KAAK,CACP,aAAa,SAAS,EAAE,CAAC,MAAM,gCAAgC,CAClE,CACJ,CAAC;IAEF,OAAO,SAAS,EAAE,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF,eAAe,gBAAgB,CAAC"}