@js-recon/js-recon 1.4.3 → 2.0.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-sync/sync.sh +1 -1
- package/.github/workflows/build-and-prettify.yaml +55 -25
- package/.github/workflows/codeql.yml +33 -0
- package/.github/workflows/exploit-smoke-test.yaml +160 -0
- package/.github/workflows/framework-detection-smoke-test.yaml +135 -0
- package/.github/workflows/plumber.yml +20 -0
- package/.github/workflows/promote-js-recon.yml +5 -3
- package/.github/workflows/publish-js-recon.yml +6 -1
- package/.github/workflows/rules-smoke-test.yaml +15 -7
- package/.nvmrc +1 -0
- package/.plumber.yaml +151 -0
- package/CHANGELOG.md +54 -0
- package/CONTRIBUTING.md +32 -0
- package/Dockerfile +1 -0
- package/README.md +55 -2
- package/SECURITY.md +1 -1
- package/build/analyze/engine/astEngine.js +10 -9
- package/build/analyze/engine/astEngine.js.map +1 -1
- package/build/analyze/engine/csMastSEngine.js +6 -6
- package/build/analyze/engine/csMastSEngine.js.map +1 -1
- package/build/analyze/engine/index.js +1 -1
- package/build/analyze/engine/index.js.map +1 -1
- package/build/analyze/engine/requestEngine.js +6 -6
- package/build/analyze/engine/requestEngine.js.map +1 -1
- package/build/analyze/helpers/determineCompatibleVersion.js +70 -0
- package/build/analyze/helpers/determineCompatibleVersion.js.map +1 -0
- package/build/analyze/helpers/engineHelpers/findMemberExpressionAssignment.js +9 -3
- package/build/analyze/helpers/engineHelpers/findMemberExpressionAssignment.js.map +1 -1
- package/build/analyze/helpers/initRules.js +42 -24
- package/build/analyze/helpers/initRules.js.map +1 -1
- package/build/analyze/helpers/outputHelper.js +3 -3
- package/build/analyze/helpers/outputHelper.js.map +1 -1
- package/build/analyze/helpers/schemas.js +1 -0
- package/build/analyze/helpers/schemas.js.map +1 -1
- package/build/analyze/helpers/validate.js +9 -5
- package/build/analyze/helpers/validate.js.map +1 -1
- package/build/analyze/index.js +26 -16
- package/build/analyze/index.js.map +1 -1
- package/build/cliProgram.js +710 -0
- package/build/cliProgram.js.map +1 -0
- package/build/completion/index.js +245 -339
- package/build/completion/index.js.map +1 -1
- package/build/completion/introspect.js +17 -0
- package/build/completion/introspect.js.map +1 -0
- package/build/config/applicationConfig.js +494 -0
- package/build/config/applicationConfig.js.map +1 -0
- package/build/cs_mast/index.js +18 -17
- package/build/cs_mast/index.js.map +1 -1
- package/build/endpoints/gen_report/gen_json.js +2 -2
- package/build/endpoints/gen_report/gen_json.js.map +1 -1
- package/build/endpoints/gen_report/utility/iterate_n_store.js +1 -1
- package/build/endpoints/gen_report/utility/iterate_n_store.js.map +1 -1
- package/build/endpoints/index.js +12 -12
- package/build/endpoints/index.js.map +1 -1
- package/build/endpoints/next_js/client_jsFilesHref.js +3 -3
- package/build/endpoints/next_js/client_jsFilesHref.js.map +1 -1
- package/build/endpoints/next_js/client_jsonParse.js +4 -4
- package/build/endpoints/next_js/client_jsonParse.js.map +1 -1
- package/build/endpoints/next_js/client_mappedJsonFile.js +6 -6
- package/build/endpoints/next_js/client_mappedJsonFile.js.map +1 -1
- package/build/endpoints/next_js/client_subsequentRequests.js +5 -5
- package/build/endpoints/next_js/client_subsequentRequests.js.map +1 -1
- package/build/exploit/angular/cve-2026-68945.js +61 -0
- package/build/exploit/angular/cve-2026-68945.js.map +1 -0
- package/build/exploit/index.js +60 -0
- package/build/exploit/index.js.map +1 -0
- package/build/exploit/next/cve-2025-29927.js +65 -0
- package/build/exploit/next/cve-2025-29927.js.map +1 -0
- package/build/exploit/next/cve-2026-64645.js +51 -0
- package/build/exploit/next/cve-2026-64645.js.map +1 -0
- package/build/exploit/nuxt/cve-2026-71315.js +47 -0
- package/build/exploit/nuxt/cve-2026-71315.js.map +1 -0
- package/build/exploit/nuxt/cve-2026-71320.js +114 -0
- package/build/exploit/nuxt/cve-2026-71320.js.map +1 -0
- package/build/exploit/react/cve-2025-55182.js +116 -0
- package/build/exploit/react/cve-2025-55182.js.map +1 -0
- package/build/exploit/registry/cveRegistry.js +115 -0
- package/build/exploit/registry/cveRegistry.js.map +1 -0
- package/build/exploit/types.js +2 -0
- package/build/exploit/types.js.map +1 -0
- package/build/exploit/utility/compareResponses.js +29 -0
- package/build/exploit/utility/compareResponses.js.map +1 -0
- package/build/exploit/utility/outputHelper.js +22 -0
- package/build/exploit/utility/outputHelper.js.map +1 -0
- package/build/exploit/utility/rawRequest.js +54 -0
- package/build/exploit/utility/rawRequest.js.map +1 -0
- package/build/exploit/utility/sandboxRce.js +49 -0
- package/build/exploit/utility/sandboxRce.js.map +1 -0
- package/build/exploit/utility/toEngineOutput.js +28 -0
- package/build/exploit/utility/toEngineOutput.js.map +1 -0
- package/build/exploit/utility/wsClient.js +78 -0
- package/build/exploit/utility/wsClient.js.map +1 -0
- package/build/exploit/vite/cve-2025-30208.js +44 -0
- package/build/exploit/vite/cve-2025-30208.js.map +1 -0
- package/build/exploit/vite/cve-2026-39363.js +60 -0
- package/build/exploit/vite/cve-2026-39363.js.map +1 -0
- package/build/fingerprint/index.js +11 -21
- package/build/fingerprint/index.js.map +1 -1
- package/build/globalConfig.js +2 -2
- package/build/globalConfig.js.map +1 -1
- package/build/index.js +16 -555
- package/build/index.js.map +1 -1
- package/build/lazyLoad/angular/angular_getFromPageSource.js +3 -3
- package/build/lazyLoad/angular/angular_getFromPageSource.js.map +1 -1
- package/build/lazyLoad/downloadFilesUtil.js +51 -40
- package/build/lazyLoad/downloadFilesUtil.js.map +1 -1
- package/build/lazyLoad/downloadQueue.js +154 -50
- package/build/lazyLoad/downloadQueue.js.map +1 -1
- package/build/lazyLoad/downloadResponse.js +113 -0
- package/build/lazyLoad/downloadResponse.js.map +1 -0
- package/build/lazyLoad/generic/generic_crawlPages.js +186 -0
- package/build/lazyLoad/generic/generic_crawlPages.js.map +1 -0
- package/build/lazyLoad/generic/generic_downloadFiles.js +81 -0
- package/build/lazyLoad/generic/generic_downloadFiles.js.map +1 -0
- package/build/lazyLoad/generic/generic_extractLinks.js +79 -0
- package/build/lazyLoad/generic/generic_extractLinks.js.map +1 -0
- package/build/lazyLoad/generic/generic_getScriptTags.js +152 -0
- package/build/lazyLoad/generic/generic_getScriptTags.js.map +1 -0
- package/build/lazyLoad/generic/generic_importMapDiscovery.js +87 -0
- package/build/lazyLoad/generic/generic_importMapDiscovery.js.map +1 -0
- package/build/lazyLoad/generic/generic_resolveScope.js +61 -0
- package/build/lazyLoad/generic/generic_resolveScope.js.map +1 -0
- package/build/lazyLoad/generic/generic_scanAttributesForJs.js +122 -0
- package/build/lazyLoad/generic/generic_scanAttributesForJs.js.map +1 -0
- package/build/lazyLoad/generic/generic_stringsDiscovery.js +103 -0
- package/build/lazyLoad/generic/generic_stringsDiscovery.js.map +1 -0
- package/build/lazyLoad/generic/generic_webpackChunkPaths.js +56 -0
- package/build/lazyLoad/generic/generic_webpackChunkPaths.js.map +1 -0
- package/build/lazyLoad/globals.js +35 -0
- package/build/lazyLoad/globals.js.map +1 -1
- package/build/lazyLoad/index.js +311 -175
- package/build/lazyLoad/index.js.map +1 -1
- package/build/lazyLoad/methodFilter.js +2 -0
- package/build/lazyLoad/methodFilter.js.map +1 -1
- package/build/lazyLoad/next_js/NextJsCrawler.js +39 -14
- package/build/lazyLoad/next_js/NextJsCrawler.js.map +1 -1
- package/build/lazyLoad/next_js/next_GetJSScript.js +3 -3
- package/build/lazyLoad/next_js/next_GetJSScript.js.map +1 -1
- package/build/lazyLoad/next_js/next_GetLazyResourcesBuildManifestJs.js +4 -4
- package/build/lazyLoad/next_js/next_GetLazyResourcesBuildManifestJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js +21 -20
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_SubsequentRequests.js +100 -99
- package/build/lazyLoad/next_js/next_SubsequentRequests.js.map +1 -1
- package/build/lazyLoad/next_js/next_bruteForceJsFiles.js +1 -1
- package/build/lazyLoad/next_js/next_bruteForceJsFiles.js.map +1 -1
- package/build/lazyLoad/next_js/next_buildId.js +2 -2
- package/build/lazyLoad/next_js/next_buildId.js.map +1 -1
- package/build/lazyLoad/next_js/next_getClientSidePaths.js +2 -2
- package/build/lazyLoad/next_js/next_getClientSidePaths.js.map +1 -1
- package/build/lazyLoad/next_js/next_parseLayoutJs.js +6 -6
- package/build/lazyLoad/next_js/next_parseLayoutJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_promiseResolve.js +1 -1
- package/build/lazyLoad/next_js/next_promiseResolve.js.map +1 -1
- package/build/lazyLoad/next_js/next_routerStateForge.js +341 -0
- package/build/lazyLoad/next_js/next_routerStateForge.js.map +1 -0
- package/build/lazyLoad/next_js/next_scriptTagsSubsequentRequests.js +1 -1
- package/build/lazyLoad/next_js/next_scriptTagsSubsequentRequests.js.map +1 -1
- package/build/lazyLoad/next_js/next_serverActionIdScan.js +77 -0
- package/build/lazyLoad/next_js/next_serverActionIdScan.js.map +1 -0
- package/build/lazyLoad/nuxt_js/nuxt_astParse.js +15 -13
- package/build/lazyLoad/nuxt_js/nuxt_astParse.js.map +1 -1
- package/build/lazyLoad/nuxt_js/nuxt_getBuildsManifest.js +2 -2
- package/build/lazyLoad/nuxt_js/nuxt_getBuildsManifest.js.map +1 -1
- package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js +3 -3
- package/build/lazyLoad/nuxt_js/nuxt_getFromPageSource.js.map +1 -1
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js +2 -2
- package/build/lazyLoad/nuxt_js/nuxt_stringAnalysisJSFiles.js.map +1 -1
- package/build/lazyLoad/outputPath.js +92 -0
- package/build/lazyLoad/outputPath.js.map +1 -0
- package/build/lazyLoad/react/react_followImports.js +15 -5
- package/build/lazyLoad/react/react_followImports.js.map +1 -1
- package/build/lazyLoad/react/react_getScriptTags.js +11 -6
- package/build/lazyLoad/react/react_getScriptTags.js.map +1 -1
- package/build/lazyLoad/react/react_sourcemapUrls.js +3 -2
- package/build/lazyLoad/react/react_sourcemapUrls.js.map +1 -1
- package/build/lazyLoad/react/react_webpackChunkPaths.js +12 -203
- package/build/lazyLoad/react/react_webpackChunkPaths.js.map +1 -1
- package/build/lazyLoad/researchUtils.js +15 -0
- package/build/lazyLoad/researchUtils.js.map +1 -0
- package/build/lazyLoad/shared/webpackChunkParsers.js +218 -0
- package/build/lazyLoad/shared/webpackChunkParsers.js.map +1 -0
- package/build/lazyLoad/stagnation/stagnationMonitor.js +63 -0
- package/build/lazyLoad/stagnation/stagnationMonitor.js.map +1 -0
- package/build/lazyLoad/svelte/svelte_discoverPagesFromJs.js +1 -0
- package/build/lazyLoad/svelte/svelte_discoverPagesFromJs.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_getFromPageSource.js +5 -5
- package/build/lazyLoad/svelte/svelte_getFromPageSource.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_getVersionJson.js +2 -2
- package/build/lazyLoad/svelte/svelte_getVersionJson.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_recursivePageCrawl.js +6 -6
- package/build/lazyLoad/svelte/svelte_recursivePageCrawl.js.map +1 -1
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js +2 -2
- package/build/lazyLoad/svelte/svelte_stringAnalysisJSFiles.js.map +1 -1
- package/build/lazyLoad/techDetect/checkAngularJS.js +1 -1
- package/build/lazyLoad/techDetect/checkAngularJS.js.map +1 -1
- package/build/lazyLoad/techDetect/checkNuxtJS.js +3 -3
- package/build/lazyLoad/techDetect/checkNuxtJS.js.map +1 -1
- package/build/lazyLoad/techDetect/checkSvelte.js +0 -1
- package/build/lazyLoad/techDetect/checkSvelte.js.map +1 -1
- package/build/lazyLoad/techDetect/checkVueJS.js +6 -6
- package/build/lazyLoad/techDetect/checkVueJS.js.map +1 -1
- package/build/lazyLoad/techDetect/index.js +106 -60
- package/build/lazyLoad/techDetect/index.js.map +1 -1
- package/build/lazyLoad/vue/vue_RuntimeJs.js +20 -15
- package/build/lazyLoad/vue/vue_RuntimeJs.js.map +1 -1
- package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js +10 -5
- package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js.map +1 -1
- package/build/lazyLoad/vue/vue_discoverJsFiles.js +20 -17
- package/build/lazyLoad/vue/vue_discoverJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_getClientSidePaths.js +70 -46
- package/build/lazyLoad/vue/vue_getClientSidePaths.js.map +1 -1
- package/build/lazyLoad/vue/vue_jsImports.js +3 -2
- package/build/lazyLoad/vue/vue_jsImports.js.map +1 -1
- package/build/lazyLoad/vue/vue_pageSrc.js +3 -2
- package/build/lazyLoad/vue/vue_pageSrc.js.map +1 -1
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js +6 -5
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js.map +1 -1
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js +10 -21
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js.map +1 -1
- package/build/lazyLoad/vue/vue_severalJsFilesHome.js +5 -4
- package/build/lazyLoad/vue/vue_severalJsFilesHome.js.map +1 -1
- package/build/lazyLoad/vue/vue_stringJsFiles.js +28 -25
- package/build/lazyLoad/vue/vue_stringJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_viteMapDeps.js +8 -2
- package/build/lazyLoad/vue/vue_viteMapDeps.js.map +1 -1
- package/build/load/index.js +8 -7
- package/build/load/index.js.map +1 -1
- package/build/map/angular_js/getAngularConnections.js +6 -6
- package/build/map/angular_js/getAngularConnections.js.map +1 -1
- package/build/map/graphql/resolveGraphql.js +6 -6
- package/build/map/graphql/resolveGraphql.js.map +1 -1
- package/build/map/index.js +17 -17
- package/build/map/index.js.map +1 -1
- package/build/map/interactive-mode/commandHandler.js +278 -0
- package/build/map/interactive-mode/commandHandler.js.map +1 -0
- package/build/map/interactive-mode/commandHelpers.js +189 -0
- package/build/map/interactive-mode/commandHelpers.js.map +1 -0
- package/build/map/interactive-mode/esqueryGen.js.map +1 -0
- package/build/map/interactive-mode/helpMenu.js +11 -0
- package/build/map/interactive-mode/helpMenu.js.map +1 -0
- package/build/map/interactive-mode/inputPatch.js.map +1 -0
- package/build/map/interactive-mode/interactive.js +71 -0
- package/build/map/interactive-mode/interactive.js.map +1 -0
- package/build/map/interactive-mode/keybindings.js.map +1 -0
- package/build/map/interactive-mode/printer.js.map +1 -0
- package/build/map/interactive-mode/ui.js.map +1 -0
- package/build/map/next_js/getAxiosInstances.js +5 -5
- package/build/map/next_js/getAxiosInstances.js.map +1 -1
- package/build/map/next_js/getExports.js +4 -4
- package/build/map/next_js/getExports.js.map +1 -1
- package/build/map/next_js/getFetchInstances.js +10 -10
- package/build/map/next_js/getFetchInstances.js.map +1 -1
- package/build/map/next_js/getTurbopackConnections.js +8 -8
- package/build/map/next_js/getTurbopackConnections.js.map +1 -1
- package/build/map/next_js/getWebpackConnections.js +27 -16
- package/build/map/next_js/getWebpackConnections.js.map +1 -1
- package/build/map/next_js/interactive.js +3 -65
- package/build/map/next_js/interactive.js.map +1 -1
- package/build/map/next_js/interactive_helpers/commandHandler.js +38 -290
- package/build/map/next_js/interactive_helpers/commandHandler.js.map +1 -1
- package/build/map/next_js/interactive_helpers/commandHelpers.js +0 -184
- package/build/map/next_js/interactive_helpers/commandHelpers.js.map +1 -1
- package/build/map/next_js/interactive_helpers/helpMenu.js +2 -10
- package/build/map/next_js/interactive_helpers/helpMenu.js.map +1 -1
- package/build/map/next_js/resolveAxios.js +6 -6
- package/build/map/next_js/resolveAxios.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/findAxiosClients.js +4 -4
- package/build/map/next_js/resolveAxiosHelpers/findAxiosClients.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/findAxiosInstance.js +2 -2
- package/build/map/next_js/resolveAxiosHelpers/findAxiosInstance.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/findCrossChunkParams.js +5 -5
- package/build/map/next_js/resolveAxiosHelpers/findCrossChunkParams.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/handleAxiosCreate.js +10 -10
- package/build/map/next_js/resolveAxiosHelpers/handleAxiosCreate.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/handleZDotCreate.js +13 -9
- package/build/map/next_js/resolveAxiosHelpers/handleZDotCreate.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/processAxiosCall.js +12 -8
- package/build/map/next_js/resolveAxiosHelpers/processAxiosCall.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/processDirectAxiosCall.js +12 -8
- package/build/map/next_js/resolveAxiosHelpers/processDirectAxiosCall.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/processExportedEndpoints.js +11 -11
- package/build/map/next_js/resolveAxiosHelpers/processExportedEndpoints.js.map +1 -1
- package/build/map/next_js/resolveAxiosHelpers/traceAxiosInstanceExports.js +16 -16
- package/build/map/next_js/resolveAxiosHelpers/traceAxiosInstanceExports.js.map +1 -1
- package/build/map/next_js/resolveFetch.js +23 -23
- package/build/map/next_js/resolveFetch.js.map +1 -1
- package/build/map/next_js/resolveNewRequest.js +10 -10
- package/build/map/next_js/resolveNewRequest.js.map +1 -1
- package/build/map/next_js/resolveServerActions.js +4 -4
- package/build/map/next_js/resolveServerActions.js.map +1 -1
- package/build/map/next_js/utils.js +8 -2
- package/build/map/next_js/utils.js.map +1 -1
- package/build/map/react_js/getReactConnections.js +21 -14
- package/build/map/react_js/getReactConnections.js.map +1 -1
- package/build/map/svelte_js/interactive.js +3 -54
- package/build/map/svelte_js/interactive.js.map +1 -1
- package/build/map/vue_js/getViteConnections.js +23 -16
- package/build/map/vue_js/getViteConnections.js.map +1 -1
- package/build/map/vue_js/interactive.js +3 -63
- package/build/map/vue_js/interactive.js.map +1 -1
- package/build/map/vue_js/interactive_helpers/commandHandler.js +34 -283
- package/build/map/vue_js/interactive_helpers/commandHandler.js.map +1 -1
- package/build/map/vue_js/interactive_helpers/helpMenu.js +2 -10
- package/build/map/vue_js/interactive_helpers/helpMenu.js.map +1 -1
- package/build/map/vue_js/taint_utils.js +1 -1
- package/build/map/vue_js/taint_utils.js.map +1 -1
- package/build/map/vue_js/vue_resolveFetch.js +12 -12
- package/build/map/vue_js/vue_resolveFetch.js.map +1 -1
- package/build/map/vue_js/vue_resolveHttpClient.js +18 -18
- package/build/map/vue_js/vue_resolveHttpClient.js.map +1 -1
- package/build/map/vue_js/vue_resolveXhr.js +11 -11
- package/build/map/vue_js/vue_resolveXhr.js.map +1 -1
- package/build/mcp/chatOneShot.js +7 -7
- package/build/mcp/chatOneShot.js.map +1 -1
- package/build/mcp/claudeCodeCreds.js +5 -5
- package/build/mcp/claudeCodeCreds.js.map +1 -1
- package/build/mcp/cli.js +36 -35
- package/build/mcp/cli.js.map +1 -1
- package/build/mcp/config.js +4 -4
- package/build/mcp/config.js.map +1 -1
- package/build/mcp/index.js +7 -7
- package/build/mcp/index.js.map +1 -1
- package/build/mcp/mcpServer.js +3 -2
- package/build/mcp/mcpServer.js.map +1 -1
- package/build/mcp/skills.js +1 -0
- package/build/mcp/skills.js.map +1 -1
- package/build/mcp/tools.js +5 -4
- package/build/mcp/tools.js.map +1 -1
- package/build/proxy/checkFeasibility.js +8 -4
- package/build/proxy/checkFeasibility.js.map +1 -1
- package/build/proxy/checkFireWallBlocking.js +23 -13
- package/build/proxy/checkFireWallBlocking.js.map +1 -1
- package/build/proxy/detectBlockedResponse.js +205 -0
- package/build/proxy/detectBlockedResponse.js.map +1 -0
- package/build/proxy/genReq.js +434 -127
- package/build/proxy/genReq.js.map +1 -1
- package/build/proxy/index.js +3 -3
- package/build/proxy/index.js.map +1 -1
- package/build/proxy/oxylabsFallback.js +110 -0
- package/build/proxy/oxylabsFallback.js.map +1 -0
- package/build/proxy/proxyFeasibilityRequest.js +11 -9
- package/build/proxy/proxyFeasibilityRequest.js.map +1 -1
- package/build/refactor/index.js +83 -79
- package/build/refactor/index.js.map +1 -1
- package/build/refactor/next/index.js +69 -26
- package/build/refactor/next/index.js.map +1 -1
- package/build/refactor/next/transform.js +11 -4
- package/build/refactor/next/transform.js.map +1 -1
- package/build/refactor/next/validator.js +7 -7
- package/build/refactor/next/validator.js.map +1 -1
- package/build/refactor/react/index.js +13 -13
- package/build/refactor/react/index.js.map +1 -1
- package/build/refactor/react/validator.js +7 -7
- package/build/refactor/react/validator.js.map +1 -1
- package/build/refactor/react-vite/index.js +3 -3
- package/build/refactor/react-vite/index.js.map +1 -1
- package/build/refactor/remote/hf-client.js +1 -0
- package/build/refactor/remote/hf-client.js.map +1 -1
- package/build/refactor/remote/version-detect.js +12 -12
- package/build/refactor/remote/version-detect.js.map +1 -1
- package/build/refactor/vue/index.js +6 -6
- package/build/refactor/vue/index.js.map +1 -1
- package/build/refactor/vue/vite.js +6 -6
- package/build/refactor/vue/vite.js.map +1 -1
- package/build/report/index.js +30 -8
- package/build/report/index.js.map +1 -1
- package/build/report/swaggerJacker.js +63 -0
- package/build/report/swaggerJacker.js.map +1 -0
- package/build/report/utility/genHtml.js +7 -7
- package/build/report/utility/genHtml.js.map +1 -1
- package/build/report/utility/populateDb/populateAnalysisFindings.js +2 -2
- package/build/report/utility/populateDb/populateAnalysisFindings.js.map +1 -1
- package/build/report/utility/populateDb/populateEndpoints.js +3 -3
- package/build/report/utility/populateDb/populateEndpoints.js.map +1 -1
- package/build/report/utility/populateDb/populateMappedJson.js +2 -2
- package/build/report/utility/populateDb/populateMappedJson.js.map +1 -1
- package/build/report/utility/populateDb/populateMappedOpenapi.js +3 -3
- package/build/report/utility/populateDb/populateMappedOpenapi.js.map +1 -1
- package/build/run/bundler-detect.js +45 -9
- package/build/run/bundler-detect.js.map +1 -1
- package/build/run/cancellableStep.js +27 -0
- package/build/run/cancellableStep.js.map +1 -0
- package/build/run/dashboard/dashboardHtml.js +150 -0
- package/build/run/dashboard/dashboardHtml.js.map +1 -0
- package/build/run/dashboard/fsBrowser.js +43 -0
- package/build/run/dashboard/fsBrowser.js.map +1 -0
- package/build/run/dashboard/server.js +123 -0
- package/build/run/dashboard/server.js.map +1 -0
- package/build/run/dashboard/state.js +97 -0
- package/build/run/dashboard/state.js.map +1 -0
- package/build/run/index.js +239 -204
- package/build/run/index.js.map +1 -1
- package/build/run/interruptHandler.js +18 -0
- package/build/run/interruptHandler.js.map +1 -1
- package/build/run/outputDirectory.js +377 -0
- package/build/run/outputDirectory.js.map +1 -0
- package/build/sourcemaps/index.js +116 -29
- package/build/sourcemaps/index.js.map +1 -1
- package/build/strings/index.js +38 -28
- package/build/strings/index.js.map +1 -1
- package/build/strings/openapi.js +4 -4
- package/build/strings/openapi.js.map +1 -1
- package/build/strings/permutate.js +4 -4
- package/build/strings/permutate.js.map +1 -1
- package/build/strings/secrets.js +1 -0
- package/build/strings/secrets.js.map +1 -1
- package/build/strings/trufflehog.js +66 -19
- package/build/strings/trufflehog.js.map +1 -1
- package/build/strings/trufflehogConfig.js +36 -0
- package/build/strings/trufflehogConfig.js.map +1 -0
- package/build/strings/trufflehogDownload.js +115 -0
- package/build/strings/trufflehogDownload.js.map +1 -0
- package/build/utility/ai.js +33 -4
- package/build/utility/ai.js.map +1 -1
- package/build/utility/banner.js +4 -3
- package/build/utility/banner.js.map +1 -1
- package/build/utility/configureSandbox.js +2 -2
- package/build/utility/configureSandbox.js.map +1 -1
- package/build/utility/downloadProgress.js +99 -0
- package/build/utility/downloadProgress.js.map +1 -0
- package/build/utility/globals.js +19 -11
- package/build/utility/globals.js.map +1 -1
- package/build/utility/heap.js +2 -1
- package/build/utility/heap.js.map +1 -1
- package/build/utility/makeReq.js +490 -257
- package/build/utility/makeReq.js.map +1 -1
- package/build/utility/openapiGenerator.js +2 -2
- package/build/utility/openapiGenerator.js.map +1 -1
- package/build/utility/printMsg.js +53 -0
- package/build/utility/printMsg.js.map +1 -0
- package/build/utility/progressLog.js +68 -7
- package/build/utility/progressLog.js.map +1 -1
- package/build/utility/resolvePath.js +2 -1
- package/build/utility/resolvePath.js.map +1 -1
- package/build/utility/ruleVersionMap.js +73 -0
- package/build/utility/ruleVersionMap.js.map +1 -0
- package/build/utility/targetInputs.js +177 -0
- package/build/utility/targetInputs.js.map +1 -0
- package/build/utility/urlUtils.js +37 -1
- package/build/utility/urlUtils.js.map +1 -1
- package/config.dist.yaml +251 -0
- package/contributing/testing.md +106 -0
- package/eslint.config.js +24 -0
- package/package.json +22 -5
- package/scripts/exploit-smoke-test.js +82 -0
- package/scripts/framework-detect-smoke-test.js +64 -0
- package/scripts/postinstall-completion.mjs +40 -0
- package/scripts/smoke-test.js +12 -0
- package/build/lazyLoad/downloadLoadedJsUtil.js +0 -98
- package/build/lazyLoad/downloadLoadedJsUtil.js.map +0 -1
- package/build/map/next_js/interactive_helpers/esqueryGen.js.map +0 -1
- package/build/map/next_js/interactive_helpers/inputPatch.js.map +0 -1
- package/build/map/next_js/interactive_helpers/keybindings.js.map +0 -1
- package/build/map/next_js/interactive_helpers/printer.js.map +0 -1
- package/build/map/next_js/interactive_helpers/ui.js.map +0 -1
- /package/build/map/{next_js/interactive_helpers → interactive-mode}/esqueryGen.js +0 -0
- /package/build/map/{next_js/interactive_helpers → interactive-mode}/inputPatch.js +0 -0
- /package/build/map/{next_js/interactive_helpers → interactive-mode}/keybindings.js +0 -0
- /package/build/map/{next_js/interactive_helpers → interactive-mode}/printer.js +0 -0
- /package/build/map/{next_js/interactive_helpers → interactive-mode}/ui.js +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloadQueue.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadQueue.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,WAAW,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"downloadQueue.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadQueue.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,WAAW,EAAE,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AAC5F,OAAO,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AACzF,OAAO,KAAK,WAAW,MAAM,uBAAuB,CAAC;AA6BrD,MAAM,oBAAoB,GAA0B,MAAM,CAAC,MAAM,CAAC;IAC9D,QAAQ,EAAE,CAAC;IACX,eAAe,EAAE,CAAC;IAClB,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;CACX,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG,GAAG,GAAG,IAAI,CAAC;AAEvC;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAa;IA6BtB,YAAY,MAAc,EAAE,WAAmB,EAAE,UAAgC,EAAE;;QAzBnF,mEAAmE;QAClD,SAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE1C,mDAAmD;QAClC,YAAO,GAAa,EAAE,CAAC;QAExC,kEAAkE;QAC1D,kBAAa,GAAG,CAAC,CAAC;QAE1B,gDAAgD;QACxC,mBAAc,GAAmB,EAAE,CAAC;QAE5C,YAAY;QACJ,kBAAa,GAAG,CAAC,CAAC;QAClB,mBAAc,GAAG,CAAC,CAAC;QACnB,kBAAa,GAA0B,oBAAoB,CAAC;QAC5D,iBAAY,GAAG,CAAC,CAAC;QACjB,iBAAY,GAAa,EAAE,CAAC;QAC5B,mBAAc,GAAa,EAAE,CAAC;QAQlC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,MAAA,OAAO,CAAC,aAAa,mCAAI,KAAK,CAAC;QACpD,IAAI,CAAC,sBAAsB,GAAG,MAAA,OAAO,CAAC,sBAAsB,mCAAI,KAAK,CAAC;QACtE,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC1D,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,IAAI,QAAQ;QACR,OAAO,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,IAAc;QACf,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACL,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,YAAY,EAAE,CAAC;IACxB,CAAC;IAED,+EAA+E;IAC/E,KAAK;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,wCAAwC;IACxC,YAAY;QACR,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;YACxE,MAAM,cAAc,GAAG,GAAG,QAAQ,cAAc,eAAe,sBAAsB,MAAM,YAAY,KAAK,QAAQ,CAAC;YACrH,MAAM,OAAO,GAAG,IAAI,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,eAAe,IAAI,CAAC,cAAc,eAAe,IAAI,CAAC,aAAa,gBAAgB,WAAW,YAAY,cAAc,MAAM,IAAI,CAAC,YAAY,UAAU,CAAC;YACzM,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBAClB,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACJ,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACtC,CAAC;YACD,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,WAAW,CACP,KAAK,CAAC,MAAM,CACR,eAAe,IAAI,CAAC,YAAY,CAAC,MAAM,iBAAiB,IAAI,CAAC,cAAc,CAAC,MAAM,gBAAgB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrI,CACJ,CAAC;QACN,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YACzB,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,aAAa,iBAAiB,IAAI,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC;QAC3G,CAAC;IACL,CAAC;IAED,wEAAwE;IAEhE,sBAAsB;QAC1B,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,SAAS,EAAE,IAAI,CAAC,cAAc;YAC9B,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;YACrB,UAAU,EAAE,IAAI,CAAC,aAAa;YAC9B,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE;YAC7B,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,mBAAM,IAAI,CAAC,aAAa,EAAG;SACrD,CAAC,CAAC;IACP,CAAC;IAEO,YAAY;QAChB,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,sBAAsB;YAAE,OAAO;QAE7D,IAAI,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,sBAAsB,GAAG,KAAK,CAAC;YACpC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,2DAA2D,GAAG,EAAE,CAAC,CAAC,CAAC;QACvG,CAAC;IACL,CAAC;IAEO,cAAc;QAClB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACxE,OAAO,QAAQ,GAAG,eAAe,GAAG,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IAEO,aAAa,CAAC,IAAyB;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,MAAM,iCAC3B,IAAI,CAAC,aAAa,KACrB,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IACtC,CAAC;IACP,CAAC;IAEO,WAAW,CAAC,OAAe;QAC/B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,aAAa,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAEO,aAAa,CAAC,OAAe;QACjC,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAEO,YAAY;QAChB,OAAO,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,EAAE,CAAC;QACrB,CAAC;IACL,CAAC;IAEa,SAAS;;YACnB,IAAI,CAAC;gBACD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAC;oBAClC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxD,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChD,KAAK,MAAM,EAAE,IAAI,SAAS;wBAAE,EAAE,EAAE,CAAC;gBACrC,CAAC;YACL,CAAC;QACL,CAAC;KAAA;IAEa,UAAU,CAAC,GAAW;;YAChC,IAAI,CAAC;gBACD,IAAI,kBAAkB,EAAE,EAAE,CAAC;oBACvB,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,OAAO;gBACX,CAAC;gBACD,IACI,CAAC,GAAG,CAAC,KAAK,CAAC,gDAAgD,CAAC;oBAC5D,GAAG,CAAC,KAAK,CAAC,iCAAiC,CAAC,EAC9C,CAAC;oBACC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,IAAI,CAAC,IAAI,CAAC,aAAa;wBAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,CAAC;oBACzE,OAAO;gBACX,CAAC;gBAED,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBAEzC,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;wBACzC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACtC,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,IAAI,GAAG,CAAC;gBACR,IAAI,CAAC;oBACD,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;gBACxE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;oBAC/B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;oBACvE,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,GAAG,EAAE,CAAC;oBACP,qEAAqE;oBACrE,oDAAoD;oBACpD,IAAI,kBAAkB,EAAE,EAAE,CAAC;wBACvB,IAAI,CAAC,YAAY,EAAE,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACJ,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;oBACnC,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,MAAM,cAAc,GAAG,MAAM,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACnE,IAAI,cAAc,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;oBAC9B,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;oBAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,KAAK,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,oBAAoB,CAAC;oBACvG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,KAAK,KAAK,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAC/E,OAAO;gBACX,CAAC;gBACD,IAAI,kBAAkB,EAAE,EAAE,CAAC;oBACvB,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,OAAO;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC;gBACpC,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;gBAC9C,iFAAiF;gBACjF,4DAA4D;gBAC5D,MAAM,IAAI,GAAG,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,GAAG,KAAK,OAAO,EAAE,CAAC;gBAEnF,MAAM,QAAQ,GAAG,yBAAyB,CAAC,GAAG,CAAC,CAAC;gBAEhD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACZ,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,6CAA6C,GAAG,aAAa,CAAC,CAAC,CAAC;oBAChG,OAAO;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;gBAC5F,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC/C,IAAI,SAAiB,CAAC;gBACtB,IAAI,CAAC;oBACD,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;wBACvB,SAAS;4BACL,IAAI,CAAC,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACpG,CAAC;yBAAM,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;wBAC7B,SAAS;4BACL,IAAI,CAAC,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACnG,CAAC;yBAAM,CAAC;wBACJ,SAAS;4BACL,IAAI,CAAC,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACrG,CAAC;gBACL,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBAC7B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,8BAA8B,QAAQ,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC;oBACrF,OAAO;gBACX,CAAC;gBAED,IAAI,kBAAkB,EAAE,EAAE,CAAC;oBACvB,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpB,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC;oBACD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC5C,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,QAAQ,EAAE,CAAC;oBAChB,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBAC5B,IAAI,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC;wBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;wBACjD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,QAAQ,MAAM,QAAQ,EAAE,CAAC,CAAC,CAAC;oBACvF,CAAC;oBACD,OAAO;gBACX,CAAC;gBACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;gBAC/B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3E,CAAC;oBAAS,CAAC;gBACP,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,EAAE,CAAC;YACxB,CAAC;QACL,CAAC;KAAA;CACJ"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
const JAVASCRIPT_CONTENT_TYPES = new Set([
|
|
11
|
+
"application/ecmascript",
|
|
12
|
+
"application/javascript",
|
|
13
|
+
"application/x-ecmascript",
|
|
14
|
+
"application/x-javascript",
|
|
15
|
+
"text/ecmascript",
|
|
16
|
+
"text/javascript",
|
|
17
|
+
"text/javascript1.0",
|
|
18
|
+
"text/javascript1.1",
|
|
19
|
+
"text/javascript1.2",
|
|
20
|
+
"text/javascript1.3",
|
|
21
|
+
"text/javascript1.4",
|
|
22
|
+
"text/javascript1.5",
|
|
23
|
+
"text/jscript",
|
|
24
|
+
"text/livescript",
|
|
25
|
+
"text/x-ecmascript",
|
|
26
|
+
"text/x-javascript",
|
|
27
|
+
]);
|
|
28
|
+
const GENERIC_CODE_CONTENT_TYPES = new Set(["application/octet-stream", "binary/octet-stream", "text/plain"]);
|
|
29
|
+
const VUE_CONTENT_TYPES = new Set(["application/x-vue", "text/x-vue"]);
|
|
30
|
+
const downloadPathname = (url) => {
|
|
31
|
+
try {
|
|
32
|
+
return new URL(url).pathname.toLowerCase();
|
|
33
|
+
}
|
|
34
|
+
catch (_a) {
|
|
35
|
+
return url.split(/[?#]/, 1)[0].toLowerCase();
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
export const getDownloadedAssetKind = (url) => {
|
|
39
|
+
const pathname = downloadPathname(url);
|
|
40
|
+
if (/(?:\.json|\.m?js\.map)$/.test(pathname))
|
|
41
|
+
return "json";
|
|
42
|
+
if (pathname.endsWith(".vue"))
|
|
43
|
+
return "vue";
|
|
44
|
+
return "javascript";
|
|
45
|
+
};
|
|
46
|
+
const hasCompatibleContentType = (url, rawContentType) => {
|
|
47
|
+
if (!rawContentType)
|
|
48
|
+
return true;
|
|
49
|
+
const contentType = rawContentType.split(";", 1)[0].trim().toLowerCase();
|
|
50
|
+
if (!contentType || GENERIC_CODE_CONTENT_TYPES.has(contentType))
|
|
51
|
+
return true;
|
|
52
|
+
const assetKind = getDownloadedAssetKind(url);
|
|
53
|
+
if (assetKind === "json") {
|
|
54
|
+
// eslint-disable-next-line security/detect-unsafe-regex -- anchored, no nested/ambiguous quantifiers — not exploitable
|
|
55
|
+
return contentType === "text/json" || /^application\/(?:[a-z0-9.+-]+\+)?json$/.test(contentType);
|
|
56
|
+
}
|
|
57
|
+
if (assetKind === "vue") {
|
|
58
|
+
// Some development servers serve SFC source as text/html. The body
|
|
59
|
+
// check below distinguishes an SFC from a complete HTML fallback.
|
|
60
|
+
return VUE_CONTENT_TYPES.has(contentType) || contentType === "text/html";
|
|
61
|
+
}
|
|
62
|
+
return JAVASCRIPT_CONTENT_TYPES.has(contentType);
|
|
63
|
+
};
|
|
64
|
+
const looksLikeHtmlDocument = (body) => {
|
|
65
|
+
const prefix = body
|
|
66
|
+
.replace(/^\uFEFF/, "")
|
|
67
|
+
.trimStart()
|
|
68
|
+
.slice(0, 1024);
|
|
69
|
+
return /^(?:<!doctype\s+html\b|<html(?:\s|>)|<head(?:\s|>)|<body(?:\s|>))/i.test(prefix);
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Validates a fetched code asset before it reaches Prettier. Missing and
|
|
73
|
+
* generic MIME types are accepted because many CDNs serve valid bundles with
|
|
74
|
+
* incomplete metadata; explicitly incompatible types and HTML fallbacks are
|
|
75
|
+
* rejected.
|
|
76
|
+
*/
|
|
77
|
+
export const readDownloadedAssetResponse = (url, response) => __awaiter(void 0, void 0, void 0, function* () {
|
|
78
|
+
if (!response.ok) {
|
|
79
|
+
return Object.freeze({
|
|
80
|
+
ok: false,
|
|
81
|
+
failure: "invalidResponse",
|
|
82
|
+
reason: `HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ""}`,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const contentType = response.headers.get("content-type");
|
|
86
|
+
if (!hasCompatibleContentType(url, contentType)) {
|
|
87
|
+
return Object.freeze({
|
|
88
|
+
ok: false,
|
|
89
|
+
failure: "invalidResponse",
|
|
90
|
+
reason: `unexpected content type ${contentType}`,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
let body;
|
|
94
|
+
try {
|
|
95
|
+
body = yield response.text();
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
return Object.freeze({
|
|
99
|
+
ok: false,
|
|
100
|
+
failure: "download",
|
|
101
|
+
reason: `could not read response body: ${err}`,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
if (looksLikeHtmlDocument(body)) {
|
|
105
|
+
return Object.freeze({
|
|
106
|
+
ok: false,
|
|
107
|
+
failure: "invalidResponse",
|
|
108
|
+
reason: "received an HTML document instead of a code asset",
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return Object.freeze({ ok: true, body });
|
|
112
|
+
});
|
|
113
|
+
//# sourceMappingURL=downloadResponse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"downloadResponse.js","sourceRoot":"","sources":["../../src/lazyLoad/downloadResponse.ts"],"names":[],"mappings":";;;;;;;;;AAMA,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,wBAAwB;IACxB,0BAA0B;IAC1B,0BAA0B;IAC1B,iBAAiB;IACjB,iBAAiB;IACjB,oBAAoB;IACpB,oBAAoB;IACpB,oBAAoB;IACpB,oBAAoB;IACpB,oBAAoB;IACpB,oBAAoB;IACpB,cAAc;IACd,iBAAiB;IACjB,mBAAmB;IACnB,mBAAmB;CACtB,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,0BAA0B,EAAE,qBAAqB,EAAE,YAAY,CAAC,CAAC,CAAC;AAC9G,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC,CAAC;AAEvE,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAU,EAAE;IAC7C,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC/C,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAuB,EAAE;IACvE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5D,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,YAAY,CAAC;AACxB,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,GAAW,EAAE,cAA6B,EAAW,EAAE;IACrF,IAAI,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC;IAEjC,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzE,IAAI,CAAC,WAAW,IAAI,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7E,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACvB,uHAAuH;QACvH,OAAO,WAAW,KAAK,WAAW,IAAI,wCAAwC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACtB,mEAAmE;QACnE,kEAAkE;QAClE,OAAO,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,WAAW,KAAK,WAAW,CAAC;IAC7E,CAAC;IACD,OAAO,wBAAwB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAW,EAAE;IACpD,MAAM,MAAM,GAAG,IAAI;SACd,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;SACtB,SAAS,EAAE;SACX,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACpB,OAAO,oEAAoE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC7F,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAO,GAAW,EAAE,QAAkB,EAAmC,EAAE;IAClH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,iBAAiB;YAC1B,MAAM,EAAE,QAAQ,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;SAC3F,CAAC,CAAC;IACP,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACzD,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,iBAAiB;YAC1B,MAAM,EAAE,2BAA2B,WAAW,EAAE;SACnD,CAAC,CAAC;IACP,CAAC;IAED,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACD,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,iCAAiC,GAAG,EAAE;SACjD,CAAC,CAAC;IACP,CAAC;IAED,IAAI,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,iBAAiB;YAC1B,MAAM,EAAE,mDAAmD;SAC9D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC,CAAA,CAAC"}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import generic_getScriptTags from "./generic_getScriptTags.js";
|
|
11
|
+
import generic_scanAttributesForJs from "./generic_scanAttributesForJs.js";
|
|
12
|
+
import generic_downloadFiles from "./generic_downloadFiles.js";
|
|
13
|
+
import generic_stringsDiscovery from "./generic_stringsDiscovery.js";
|
|
14
|
+
import generic_importMapDiscovery from "./generic_importMapDiscovery.js";
|
|
15
|
+
import generic_webpackChunkPaths from "./generic_webpackChunkPaths.js";
|
|
16
|
+
import { extractPageLinks } from "./generic_extractLinks.js";
|
|
17
|
+
import { resolveJsPathCandidate, confirmJsContentType } from "./generic_scanAttributesForJs.js";
|
|
18
|
+
import { accumulateTechnique } from "../researchUtils.js";
|
|
19
|
+
import * as lazyLoadGlobals from "../globals.js";
|
|
20
|
+
import { StagnationMonitor } from "../stagnation/stagnationMonitor.js";
|
|
21
|
+
import { printMsg, MSG } from "../../utility/printMsg.js";
|
|
22
|
+
// internal#74 / internal#75: import-map manifests and webpack chunk hash-maps are
|
|
23
|
+
// finite, already-known structures (unlike open-ended strings discovery) — a handful of
|
|
24
|
+
// rounds is enough to reach a fixed point in practice, but the loop stays bounded rather
|
|
25
|
+
// than unconditional in case a pathological manifest chain never converges.
|
|
26
|
+
const STRUCTURAL_DISCOVERY_MAX_ITERATIONS = 5;
|
|
27
|
+
/**
|
|
28
|
+
* Whether a link's host is within the active scope list. "*" allows any host —
|
|
29
|
+
* matches the sentinel convention already used by downloadFilesUtil.ts / downloadQueue.ts.
|
|
30
|
+
*/
|
|
31
|
+
export const isInScope = (url, scope) => {
|
|
32
|
+
if (scope.includes("*"))
|
|
33
|
+
return true;
|
|
34
|
+
try {
|
|
35
|
+
return scope.includes(new URL(url).host);
|
|
36
|
+
}
|
|
37
|
+
catch (_a) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Recursively crawls in-scope HTML pages reachable via <a href> from seedUrl,
|
|
43
|
+
* downloading JS discovered on each page (via generic_getScriptTags /
|
|
44
|
+
* generic_scanAttributesForJs) and following newly discovered links until the
|
|
45
|
+
* page pool is exhausted or maxPageVisits is reached. Implements internal#66:
|
|
46
|
+
* following non-JS <a href> links recursively is what lets the generic tech
|
|
47
|
+
* reach JS referenced only from pages the seed page doesn't link to directly.
|
|
48
|
+
*
|
|
49
|
+
* JS files are downloaded incrementally, page by page, rather than batched
|
|
50
|
+
* until the whole crawl finishes — a multi-hundred-page crawl can take
|
|
51
|
+
* minutes, and deferring every download to the end makes the tool look stuck.
|
|
52
|
+
*
|
|
53
|
+
* When stringsEnabled is set, once the page crawl is exhausted a second
|
|
54
|
+
* discovery mode runs: generic_stringsDiscovery.ts scans every downloaded file
|
|
55
|
+
* (inline scripts, decoded data: URIs, and external JS) for string-literal JS
|
|
56
|
+
* paths using the existing `strings` module's AST-based extraction, resolves
|
|
57
|
+
* each one against the URL the file it came from was downloaded from, and
|
|
58
|
+
* downloads any new confirmed JS files. This repeats — new downloads feed the
|
|
59
|
+
* next strings pass — until a pass finds nothing new or stringsMaxIterations
|
|
60
|
+
* is reached, the same "loop until nothing new" shape as react_followImports.
|
|
61
|
+
*
|
|
62
|
+
* Regardless of stringsEnabled, once the page crawl is exhausted a structural
|
|
63
|
+
* discovery pass runs unconditionally (internal#74, internal#75): generic_importMapDiscovery.ts
|
|
64
|
+
* parses already-downloaded files for Module Federation "import map" manifests and seeds
|
|
65
|
+
* every listed remote-entry.js URL, and generic_webpackChunkPaths.ts applies the same
|
|
66
|
+
* webpack chunk-hash-map patterns the React crawler resolves (shared via
|
|
67
|
+
* ../shared/webpackChunkParsers.ts) to statically enumerate a downloaded entry chunk's
|
|
68
|
+
* own async-chunk set. Both repeat — newly downloaded files feed the next pass — until a
|
|
69
|
+
* round finds nothing new, since a newly-discovered remote-entry.js can itself carry
|
|
70
|
+
* either pattern.
|
|
71
|
+
*
|
|
72
|
+
* When stagnationTimeinMs > 0, a StagnationMonitor tracks JS content hashes (recorded
|
|
73
|
+
* globally by generic_downloadFiles.ts / generic_getScriptTags.ts as files are discovered)
|
|
74
|
+
* and, once armed, stops the crawl early if one content hash comes to dominate the
|
|
75
|
+
* discovered set with no genuinely new content appearing — catches infinite/near-infinite
|
|
76
|
+
* sites (blogs, news feeds) whose pages keep referencing cache-busted-but-identical JS.
|
|
77
|
+
*
|
|
78
|
+
* techDetectInterceptedUrls carries every request Puppeteer's tech-detection pass
|
|
79
|
+
* already intercepted for this target (see techDetect/index.ts's getLastInterceptedUrls)
|
|
80
|
+
* — this includes anything requested only because a runtime-injected script asked for
|
|
81
|
+
* it (e.g. Cloudflare's own bot-challenge script self-injecting via
|
|
82
|
+
* `element.innerHTML = "...;a.src='...';..."`), which a plain-fetch crawl has no way
|
|
83
|
+
* to see on its own since the reference only exists once the page's JS actually runs.
|
|
84
|
+
*/
|
|
85
|
+
const generic_crawlPages = (seedUrl_1, maxJsSizeMb_1, outputDir_1, maxPageVisits_1, threads_1, researchMap_1, ...args_1) => __awaiter(void 0, [seedUrl_1, maxJsSizeMb_1, outputDir_1, maxPageVisits_1, threads_1, researchMap_1, ...args_1], void 0, function* (seedUrl, maxJsSizeMb, outputDir, maxPageVisits, threads, researchMap, stringsEnabled = false, stringsMaxIterations = 5, techDetectInterceptedUrls = [], stagnationTimeinMs = 0, stagnationPercentage = 80, stagnationMonitorMs = 60000) {
|
|
86
|
+
const visitedPages = new Set();
|
|
87
|
+
const queued = new Set([seedUrl]);
|
|
88
|
+
const queue = [seedUrl];
|
|
89
|
+
const downloadedJsUrls = new Set();
|
|
90
|
+
const stagnationMonitor = stagnationTimeinMs > 0
|
|
91
|
+
? new StagnationMonitor(stagnationTimeinMs, stagnationPercentage, stagnationMonitorMs)
|
|
92
|
+
: null;
|
|
93
|
+
if (techDetectInterceptedUrls.length > 0) {
|
|
94
|
+
const candidates = [
|
|
95
|
+
...new Set(techDetectInterceptedUrls
|
|
96
|
+
.map((u) => resolveJsPathCandidate(u, u))
|
|
97
|
+
.filter((u) => u !== null)),
|
|
98
|
+
];
|
|
99
|
+
if (candidates.length > 0) {
|
|
100
|
+
const confirmed = yield confirmJsContentType(candidates);
|
|
101
|
+
if (researchMap)
|
|
102
|
+
accumulateTechnique(researchMap, "techDetect_interceptedUrls", confirmed);
|
|
103
|
+
if (confirmed.length > 0) {
|
|
104
|
+
confirmed.forEach((u) => downloadedJsUrls.add(u));
|
|
105
|
+
yield generic_downloadFiles(confirmed, outputDir, threads);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
while (queue.length > 0) {
|
|
110
|
+
if (maxPageVisits > 0 && visitedPages.size >= maxPageVisits) {
|
|
111
|
+
printMsg(MSG.Warn, `[i] Reached max page visits (${maxPageVisits}); stopping generic crawl`);
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
if ((stagnationMonitor === null || stagnationMonitor === void 0 ? void 0 : stagnationMonitor.shouldEvaluate()) && stagnationMonitor.evaluate()) {
|
|
115
|
+
printMsg(MSG.Warn, `[i] Detected content stagnation (dominant JS content ≥ ${stagnationPercentage}% with no new content); stopping generic crawl`);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
const pageUrl = queue.shift();
|
|
119
|
+
if (visitedPages.has(pageUrl))
|
|
120
|
+
continue;
|
|
121
|
+
visitedPages.add(pageUrl);
|
|
122
|
+
printMsg(MSG.Header, `[i] Crawling page (${visitedPages.size}): ${pageUrl}`);
|
|
123
|
+
const { urls: scriptUrls, pageSource } = yield generic_getScriptTags(pageUrl, maxJsSizeMb, outputDir);
|
|
124
|
+
if (researchMap)
|
|
125
|
+
accumulateTechnique(researchMap, "generic_getScriptTags", scriptUrls);
|
|
126
|
+
let newJsUrls = scriptUrls.filter((u) => !downloadedJsUrls.has(u));
|
|
127
|
+
if (pageSource) {
|
|
128
|
+
const attrCandidates = yield generic_scanAttributesForJs(pageSource, pageUrl, downloadedJsUrls);
|
|
129
|
+
if (researchMap)
|
|
130
|
+
accumulateTechnique(researchMap, "generic_scanAttributesForJs", attrCandidates);
|
|
131
|
+
newJsUrls = [...new Set([...newJsUrls, ...attrCandidates.filter((u) => !downloadedJsUrls.has(u))])];
|
|
132
|
+
}
|
|
133
|
+
if (newJsUrls.length > 0) {
|
|
134
|
+
newJsUrls.forEach((u) => downloadedJsUrls.add(u));
|
|
135
|
+
yield generic_downloadFiles(newJsUrls, outputDir, threads);
|
|
136
|
+
}
|
|
137
|
+
if (!pageSource)
|
|
138
|
+
continue;
|
|
139
|
+
const scope = lazyLoadGlobals.getScope();
|
|
140
|
+
const links = extractPageLinks(pageSource, pageUrl);
|
|
141
|
+
const newLinks = links.filter((l) => isInScope(l, scope) && !queued.has(l));
|
|
142
|
+
newLinks.forEach((l) => queued.add(l));
|
|
143
|
+
if (researchMap)
|
|
144
|
+
accumulateTechnique(researchMap, "generic_crawlPages", newLinks);
|
|
145
|
+
queue.push(...newLinks);
|
|
146
|
+
}
|
|
147
|
+
for (let iteration = 0; iteration < STRUCTURAL_DISCOVERY_MAX_ITERATIONS; iteration++) {
|
|
148
|
+
const [importMapUrls, chunkPathUrls] = yield Promise.all([
|
|
149
|
+
generic_importMapDiscovery(outputDir, downloadedJsUrls, threads),
|
|
150
|
+
generic_webpackChunkPaths(outputDir, downloadedJsUrls, threads),
|
|
151
|
+
]);
|
|
152
|
+
const freshUrls = [...new Set([...importMapUrls, ...chunkPathUrls])].filter((u) => !downloadedJsUrls.has(u));
|
|
153
|
+
if (researchMap)
|
|
154
|
+
accumulateTechnique(researchMap, "generic_importMapDiscovery", importMapUrls);
|
|
155
|
+
if (researchMap)
|
|
156
|
+
accumulateTechnique(researchMap, "generic_webpackChunkPaths", chunkPathUrls);
|
|
157
|
+
if (freshUrls.length === 0) {
|
|
158
|
+
if (iteration === 0) {
|
|
159
|
+
printMsg(MSG.Warn, "[i] No import-map manifests or webpack chunk hash-maps found");
|
|
160
|
+
}
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
printMsg(MSG.Run, `[✓] Found ${freshUrls.length} new JS file(s) via structural discovery`);
|
|
164
|
+
freshUrls.forEach((u) => downloadedJsUrls.add(u));
|
|
165
|
+
yield generic_downloadFiles(freshUrls, outputDir, threads);
|
|
166
|
+
}
|
|
167
|
+
if (stringsEnabled) {
|
|
168
|
+
for (let iteration = 0; stringsMaxIterations <= 0 || iteration < stringsMaxIterations; iteration++) {
|
|
169
|
+
printMsg(MSG.Header, `[i] Running strings-based discovery pass ${iteration + 1}`);
|
|
170
|
+
const candidates = yield generic_stringsDiscovery(outputDir, downloadedJsUrls);
|
|
171
|
+
const freshUrls = candidates.filter((u) => !downloadedJsUrls.has(u));
|
|
172
|
+
if (researchMap)
|
|
173
|
+
accumulateTechnique(researchMap, "generic_stringsDiscovery", freshUrls);
|
|
174
|
+
if (freshUrls.length === 0) {
|
|
175
|
+
printMsg(MSG.Warn, "[i] No new JS files found via strings; stopping strings discovery");
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
printMsg(MSG.Run, `[✓] Found ${freshUrls.length} new JS file(s) via strings`);
|
|
179
|
+
freshUrls.forEach((u) => downloadedJsUrls.add(u));
|
|
180
|
+
yield generic_downloadFiles(freshUrls, outputDir, threads);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return [...downloadedJsUrls];
|
|
184
|
+
});
|
|
185
|
+
export default generic_crawlPages;
|
|
186
|
+
//# sourceMappingURL=generic_crawlPages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generic_crawlPages.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_crawlPages.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,qBAAqB,MAAM,4BAA4B,CAAC;AAC/D,OAAO,2BAA2B,MAAM,kCAAkC,CAAC;AAC3E,OAAO,qBAAqB,MAAM,4BAA4B,CAAC;AAC/D,OAAO,wBAAwB,MAAM,+BAA+B,CAAC;AACrE,OAAO,0BAA0B,MAAM,iCAAiC,CAAC;AACzE,OAAO,yBAAyB,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AAChG,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,eAAe,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAE1D,kFAAkF;AAClF,wFAAwF;AACxF,yFAAyF;AACzF,4EAA4E;AAC5E,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,KAAe,EAAW,EAAE;IAC/D,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,CAAC;QACD,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,MAAM,kBAAkB,GAAG,8FAaN,EAAE,qIAZnB,OAAe,EACf,WAAmB,EACnB,SAAiB,EACjB,aAAqB,EACrB,OAAe,EACf,WAAsC,EACtC,iBAA0B,KAAK,EAC/B,uBAA+B,CAAC,EAChC,4BAAsC,EAAE,EACxC,qBAA6B,CAAC,EAC9B,uBAA+B,EAAE,EACjC,sBAA8B,KAAK;IAEnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,MAAM,iBAAiB,GACnB,kBAAkB,GAAG,CAAC;QAClB,CAAC,CAAC,IAAI,iBAAiB,CAAC,kBAAkB,EAAE,oBAAoB,EAAE,mBAAmB,CAAC;QACtF,CAAC,CAAC,IAAI,CAAC;IAEf,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG;YACf,GAAG,IAAI,GAAG,CACN,yBAAyB;iBACpB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBACxC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAC9C;SACJ,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,UAAU,CAAC,CAAC;YACzD,IAAI,WAAW;gBAAE,mBAAmB,CAAC,WAAW,EAAE,4BAA4B,EAAE,SAAS,CAAC,CAAC;YAC3F,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,qBAAqB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAC/D,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,IAAI,aAAa,EAAE,CAAC;YAC1D,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,gCAAgC,aAAa,2BAA2B,CAAC,CAAC;YAC7F,MAAM;QACV,CAAC;QAED,IAAI,CAAA,iBAAiB,aAAjB,iBAAiB,uBAAjB,iBAAiB,CAAE,cAAc,EAAE,KAAI,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC;YACtE,QAAQ,CACJ,GAAG,CAAC,IAAI,EACR,0DAA0D,oBAAoB,gDAAgD,CACjI,CAAC;YACF,MAAM;QACV,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAY,CAAC;QACxC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QACxC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE1B,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,sBAAsB,YAAY,CAAC,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;QAE7E,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QACtG,IAAI,WAAW;YAAE,mBAAmB,CAAC,WAAW,EAAE,uBAAuB,EAAE,UAAU,CAAC,CAAC;QAEvF,IAAI,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,IAAI,UAAU,EAAE,CAAC;YACb,MAAM,cAAc,GAAG,MAAM,2BAA2B,CAAC,UAAU,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;YAChG,IAAI,WAAW;gBAAE,mBAAmB,CAAC,WAAW,EAAE,6BAA6B,EAAE,cAAc,CAAC,CAAC;YACjG,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxG,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,qBAAqB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,WAAW;YAAE,mBAAmB,CAAC,WAAW,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,mCAAmC,EAAE,SAAS,EAAE,EAAE,CAAC;QACnF,MAAM,CAAC,aAAa,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACrD,0BAA0B,CAAC,SAAS,EAAE,gBAAgB,EAAE,OAAO,CAAC;YAChE,yBAAyB,CAAC,SAAS,EAAE,gBAAgB,EAAE,OAAO,CAAC;SAClE,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7G,IAAI,WAAW;YAAE,mBAAmB,CAAC,WAAW,EAAE,4BAA4B,EAAE,aAAa,CAAC,CAAC;QAC/F,IAAI,WAAW;YAAE,mBAAmB,CAAC,WAAW,EAAE,2BAA2B,EAAE,aAAa,CAAC,CAAC;QAE9F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBAClB,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,8DAA8D,CAAC,CAAC;YACvF,CAAC;YACD,MAAM;QACV,CAAC;QAED,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,SAAS,CAAC,MAAM,0CAA0C,CAAC,CAAC;QAC3F,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,qBAAqB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,cAAc,EAAE,CAAC;QACjB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,oBAAoB,IAAI,CAAC,IAAI,SAAS,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,CAAC;YACjG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,4CAA4C,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAC/E,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAErE,IAAI,WAAW;gBAAE,mBAAmB,CAAC,WAAW,EAAE,0BAA0B,EAAE,SAAS,CAAC,CAAC;YAEzF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,mEAAmE,CAAC,CAAC;gBACxF,MAAM;YACV,CAAC;YAED,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,SAAS,CAAC,MAAM,6BAA6B,CAAC,CAAC;YAC9E,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,qBAAqB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,gBAAgB,CAAC,CAAC;AACjC,CAAC,CAAA,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import path from "path";
|
|
11
|
+
import fs from "fs";
|
|
12
|
+
import crypto from "crypto";
|
|
13
|
+
import makeRequest from "../../utility/makeReq.js";
|
|
14
|
+
import { getURLDirectory } from "../../utility/urlUtils.js";
|
|
15
|
+
import * as lazyLoadGlobals from "../globals.js";
|
|
16
|
+
import { printMsg, MSG } from "../../utility/printMsg.js";
|
|
17
|
+
/**
|
|
18
|
+
* Synthesizes an on-disk filename for a generic-tech JS URL. downloadFilesUtil.ts's
|
|
19
|
+
* regex expects the LAST path segment to already be a clean ".js"/".mjs" filename,
|
|
20
|
+
* which breaks for content-type-confirmed assets like ".../beacon.min.js/v124/token"
|
|
21
|
+
* (the ".js" segment isn't the last one) — hence a dedicated synthesis step here.
|
|
22
|
+
*/
|
|
23
|
+
export const synthesizeFilename = (url) => {
|
|
24
|
+
var _a;
|
|
25
|
+
const hash = crypto.createHash("sha1").update(url).digest("hex").slice(0, 8);
|
|
26
|
+
const pathname = new URL(url).pathname;
|
|
27
|
+
const segments = pathname.split("/").filter(Boolean);
|
|
28
|
+
const lastSegment = (_a = segments[segments.length - 1]) !== null && _a !== void 0 ? _a : "";
|
|
29
|
+
if (/^[a-zA-Z0-9.\-_]+\.(mjs|js)$/.test(lastSegment)) {
|
|
30
|
+
return lastSegment;
|
|
31
|
+
}
|
|
32
|
+
for (let i = segments.length - 1; i >= 0; i--) {
|
|
33
|
+
if (segments[i].toLowerCase().endsWith(".js")) {
|
|
34
|
+
return `${segments[i].replace(/\.js$/i, "")}-${hash}.js`;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return `${hash}.js`;
|
|
38
|
+
};
|
|
39
|
+
const generic_downloadFiles = (urls, output, threads) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
+
if (urls.length === 0)
|
|
41
|
+
return;
|
|
42
|
+
printMsg(MSG.Header, `[i] Attempting to download ${urls.length} generic JS file(s)`);
|
|
43
|
+
let downloadCount = 0;
|
|
44
|
+
let cursor = 0;
|
|
45
|
+
const concurrency = Math.max(1, threads);
|
|
46
|
+
const processOne = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
47
|
+
try {
|
|
48
|
+
const { host, directory } = getURLDirectory(url);
|
|
49
|
+
const childDir = path.join(output, host, directory);
|
|
50
|
+
fs.mkdirSync(childDir, { recursive: true });
|
|
51
|
+
const res = yield makeRequest(url, {});
|
|
52
|
+
if (!res) {
|
|
53
|
+
printMsg(MSG.Err, `[!] Failed to download: ${url}`);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const rawText = yield res.text();
|
|
57
|
+
const filename = synthesizeFilename(url);
|
|
58
|
+
const filePath = path.join(childDir, filename);
|
|
59
|
+
fs.writeFileSync(filePath, `// File Source: ${url}\n${rawText}`);
|
|
60
|
+
lazyLoadGlobals.recordJsFileHash(crypto.createHash("sha256").update(rawText).digest("hex"));
|
|
61
|
+
downloadCount++;
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
printMsg(MSG.Err, `[!] Failed to download: ${url} : ${err}`);
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
const worker = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
68
|
+
while (true) {
|
|
69
|
+
const idx = cursor++;
|
|
70
|
+
if (idx >= urls.length)
|
|
71
|
+
break;
|
|
72
|
+
yield processOne(urls[idx]);
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
yield Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
76
|
+
if (downloadCount > 0) {
|
|
77
|
+
printMsg(MSG.Run, `[✓] Downloaded ${downloadCount} generic JS file(s) to ${output} directory`);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
export default generic_downloadFiles;
|
|
81
|
+
//# sourceMappingURL=generic_downloadFiles.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generic_downloadFiles.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_downloadFiles.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,KAAK,eAAe,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,2BAA2B,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAAU,EAAE;;IACtD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,MAAA,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,mCAAI,EAAE,CAAC;IAExD,IAAI,8BAA8B,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QACnD,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC;QAC7D,CAAC;IACL,CAAC;IAED,OAAO,GAAG,IAAI,KAAK,CAAC;AACxB,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAO,IAAc,EAAE,MAAc,EAAE,OAAe,EAAiB,EAAE;IACnG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAE9B,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,8BAA8B,IAAI,CAAC,MAAM,qBAAqB,CAAC,CAAC;IAErF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAEzC,MAAM,UAAU,GAAG,CAAO,GAAW,EAAE,EAAE;QACrC,IAAI,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;YACpD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE5C,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,2BAA2B,GAAG,EAAE,CAAC,CAAC;gBACpD,OAAO;YACX,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC/C,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,mBAAmB,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC;YACjE,eAAe,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,aAAa,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,2BAA2B,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC;QACjE,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,MAAM,GAAG,GAAS,EAAE;QACtB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM;gBAAE,MAAM;YAC9B,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEvE,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACpB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,kBAAkB,aAAa,0BAA0B,MAAM,YAAY,CAAC,CAAC;IACnG,CAAC;AACL,CAAC,CAAA,CAAC;AAEF,eAAe,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import * as cheerio from "cheerio";
|
|
2
|
+
const resolvePageUrl = (rawValue, baseUrl) => {
|
|
3
|
+
if (!rawValue)
|
|
4
|
+
return null;
|
|
5
|
+
const trimmed = rawValue.trim();
|
|
6
|
+
if (!trimmed || trimmed.startsWith("#"))
|
|
7
|
+
return null;
|
|
8
|
+
if (/^(mailto|tel|javascript|data|blob):/i.test(trimmed))
|
|
9
|
+
return null;
|
|
10
|
+
let resolved;
|
|
11
|
+
try {
|
|
12
|
+
resolved = new URL(trimmed, baseUrl);
|
|
13
|
+
}
|
|
14
|
+
catch (_a) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
if (resolved.protocol !== "http:" && resolved.protocol !== "https:")
|
|
18
|
+
return null;
|
|
19
|
+
resolved.hash = "";
|
|
20
|
+
return resolved.href;
|
|
21
|
+
};
|
|
22
|
+
// Matches an absolute http(s) URL embedded anywhere in a string — e.g. inside an
|
|
23
|
+
// onclick handler's JS: onclick="return powerpress_pinw('https://site/?pinw=123')".
|
|
24
|
+
// Deliberately not tied to a specific function name (window.open, a plugin helper,
|
|
25
|
+
// etc.) — popup/widget links are triggered through all sorts of call shapes, but the
|
|
26
|
+
// URL argument itself is always a plain quoted absolute URL, which this catches
|
|
27
|
+
// regardless of what function it's passed to.
|
|
28
|
+
const EMBEDDED_URL_RE = /https?:\/\/[^\s'"<>)]+/g;
|
|
29
|
+
/**
|
|
30
|
+
* Extracts every absolute http(s) URL substring embedded in a string value (typically
|
|
31
|
+
* an onclick handler or other inline JS attribute). See EMBEDDED_URL_RE for why this
|
|
32
|
+
* isn't tied to a specific JS call pattern like window.open(...).
|
|
33
|
+
*/
|
|
34
|
+
export const extractEmbeddedUrls = (attrValue) => [...attrValue.matchAll(EMBEDDED_URL_RE)].map((m) => m[0]);
|
|
35
|
+
/**
|
|
36
|
+
* Extracts navigable page URLs from <a href>/<iframe src> tags and absolute URLs
|
|
37
|
+
* embedded in any other attribute value (typically an onclick handler) — not JS
|
|
38
|
+
* files. Used to recursively discover more HTML pages to scan for JS, per
|
|
39
|
+
* internal#66. <iframe src> and onclick-embedded URLs are included alongside
|
|
40
|
+
* <a href> because embedded widgets (a podcast player, a survey, a booking form) are
|
|
41
|
+
* commonly reachable only via an iframe or a JS-driven popup, never a plain link —
|
|
42
|
+
* WordPress's PowerPress podcast plugin is one real example: its "pop out player"
|
|
43
|
+
* link is `onclick="return powerpress_pinw('https://site/?powerpress_pinw=123-podcast')"`,
|
|
44
|
+
* invisible to href-only or iframe-only crawling. Filters out fragment-only links,
|
|
45
|
+
* non-http(s) schemes (mailto/tel/javascript/data/blob), and strips the hash so
|
|
46
|
+
* #section variants of the same page dedupe to one entry.
|
|
47
|
+
*/
|
|
48
|
+
export const extractPageLinks = (html, baseUrl) => {
|
|
49
|
+
const $ = cheerio.load(html);
|
|
50
|
+
const links = new Set();
|
|
51
|
+
$("a[href], iframe[src]").each((_, elem) => {
|
|
52
|
+
var _a;
|
|
53
|
+
const rawValue = (_a = $(elem).attr("href")) !== null && _a !== void 0 ? _a : $(elem).attr("src");
|
|
54
|
+
const resolved = resolvePageUrl(rawValue, baseUrl);
|
|
55
|
+
if (resolved)
|
|
56
|
+
links.add(resolved);
|
|
57
|
+
});
|
|
58
|
+
$("*").each((_, elem) => {
|
|
59
|
+
const attribs = elem.attribs;
|
|
60
|
+
if (!attribs)
|
|
61
|
+
return;
|
|
62
|
+
for (const [attrName, value] of Object.entries(attribs)) {
|
|
63
|
+
// href/src already handled above via cheerio's typed accessors; re-scanning
|
|
64
|
+
// them here would just re-add the same URL, which the Set dedupes anyway,
|
|
65
|
+
// but skipping is cheaper and avoids false "embedded URL" matches inside a
|
|
66
|
+
// long querystring value that itself contains another URL as a parameter.
|
|
67
|
+
if (attrName === "href" || attrName === "src" || !value)
|
|
68
|
+
continue;
|
|
69
|
+
for (const embeddedUrl of extractEmbeddedUrls(value)) {
|
|
70
|
+
const resolved = resolvePageUrl(embeddedUrl, baseUrl);
|
|
71
|
+
if (resolved)
|
|
72
|
+
links.add(resolved);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
return [...links];
|
|
77
|
+
};
|
|
78
|
+
export default extractPageLinks;
|
|
79
|
+
//# sourceMappingURL=generic_extractLinks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generic_extractLinks.js","sourceRoot":"","sources":["../../../src/lazyLoad/generic/generic_extractLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG,CAAC,QAA4B,EAAE,OAAe,EAAiB,EAAE;IACpF,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,sCAAsC,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtE,IAAI,QAAa,CAAC;IAClB,IAAI,CAAC;QACD,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjF,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,CAAC;AACzB,CAAC,CAAC;AAEF,iFAAiF;AACjF,oFAAoF;AACpF,mFAAmF;AACnF,qFAAqF;AACrF,gFAAgF;AAChF,8CAA8C;AAC9C,MAAM,eAAe,GAAG,yBAAyB,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,SAAiB,EAAY,EAAE,CAC/D,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE9D;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAE,OAAe,EAAY,EAAE;IACxE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,CAAC,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;;QACvC,MAAM,QAAQ,GAAG,MAAA,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,QAAQ;YAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,OAAO,GAAI,IAAY,CAAC,OAA6C,CAAC;QAC5E,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACtD,4EAA4E;YAC5E,0EAA0E;YAC1E,2EAA2E;YAC3E,0EAA0E;YAC1E,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,KAAK;gBAAE,SAAS;YAClE,KAAK,MAAM,WAAW,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBACtD,IAAI,QAAQ;oBAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC,CAAC;AAEF,eAAe,gBAAgB,CAAC"}
|