@js-recon/js-recon 1.4.2 → 1.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/build-and-prettify.yaml +6 -6
- package/.github/workflows/rules-smoke-test.yaml +2 -2
- package/CHANGELOG.md +30 -0
- package/Dockerfile +13 -2
- package/Dockerfile.release +12 -2
- package/Dockerfile.w-rules +0 -1
- package/README.md +52 -0
- package/build/analyze/helpers/initRules.js +34 -16
- package/build/analyze/helpers/initRules.js.map +1 -1
- package/build/completion/index.js +44 -3
- package/build/completion/index.js.map +1 -1
- package/build/config/applicationConfig.js +493 -0
- package/build/config/applicationConfig.js.map +1 -0
- package/build/fingerprint/index.js +8 -19
- package/build/fingerprint/index.js.map +1 -1
- package/build/globalConfig.js +1 -1
- package/build/index.js +62 -9
- package/build/index.js.map +1 -1
- package/build/lazyLoad/downloadFilesUtil.js +40 -32
- package/build/lazyLoad/downloadFilesUtil.js.map +1 -1
- package/build/lazyLoad/downloadQueue.js +150 -50
- package/build/lazyLoad/downloadQueue.js.map +1 -1
- package/build/lazyLoad/downloadResponse.js +112 -0
- package/build/lazyLoad/downloadResponse.js.map +1 -0
- package/build/lazyLoad/globals.js +4 -0
- package/build/lazyLoad/globals.js.map +1 -1
- package/build/lazyLoad/index.js +188 -115
- package/build/lazyLoad/index.js.map +1 -1
- package/build/lazyLoad/next_js/NextJsCrawler.js +3 -2
- package/build/lazyLoad/next_js/NextJsCrawler.js.map +1 -1
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js +8 -9
- package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js.map +1 -1
- package/build/lazyLoad/next_js/next_SubsequentRequests.js +93 -93
- package/build/lazyLoad/next_js/next_SubsequentRequests.js.map +1 -1
- package/build/lazyLoad/outputPath.js +92 -0
- package/build/lazyLoad/outputPath.js.map +1 -0
- package/build/lazyLoad/react/react_followImports.js +15 -5
- package/build/lazyLoad/react/react_followImports.js.map +1 -1
- package/build/lazyLoad/react/react_getScriptTags.js +11 -6
- package/build/lazyLoad/react/react_getScriptTags.js.map +1 -1
- package/build/lazyLoad/react/react_sourcemapUrls.js +3 -2
- package/build/lazyLoad/react/react_sourcemapUrls.js.map +1 -1
- package/build/lazyLoad/react/react_webpackChunkPaths.js +2 -1
- package/build/lazyLoad/react/react_webpackChunkPaths.js.map +1 -1
- package/build/lazyLoad/techDetect/index.js +93 -59
- package/build/lazyLoad/techDetect/index.js.map +1 -1
- package/build/lazyLoad/vue/vue_RuntimeJs.js +20 -15
- package/build/lazyLoad/vue/vue_RuntimeJs.js.map +1 -1
- package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js +8 -3
- package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js.map +1 -1
- package/build/lazyLoad/vue/vue_discoverJsFiles.js +10 -9
- package/build/lazyLoad/vue/vue_discoverJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_getClientSidePaths.js +69 -45
- package/build/lazyLoad/vue/vue_getClientSidePaths.js.map +1 -1
- package/build/lazyLoad/vue/vue_jsImports.js +3 -2
- package/build/lazyLoad/vue/vue_jsImports.js.map +1 -1
- package/build/lazyLoad/vue/vue_pageSrc.js +2 -1
- package/build/lazyLoad/vue/vue_pageSrc.js.map +1 -1
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js +4 -3
- package/build/lazyLoad/vue/vue_reconstructSourceMaps.js.map +1 -1
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js +9 -20
- package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js.map +1 -1
- package/build/lazyLoad/vue/vue_severalJsFilesHome.js +2 -1
- package/build/lazyLoad/vue/vue_severalJsFilesHome.js.map +1 -1
- package/build/lazyLoad/vue/vue_stringJsFiles.js +28 -25
- package/build/lazyLoad/vue/vue_stringJsFiles.js.map +1 -1
- package/build/lazyLoad/vue/vue_viteMapDeps.js +8 -2
- package/build/lazyLoad/vue/vue_viteMapDeps.js.map +1 -1
- package/build/proxy/checkFeasibility.js +8 -4
- package/build/proxy/checkFeasibility.js.map +1 -1
- package/build/proxy/checkFireWallBlocking.js +23 -13
- package/build/proxy/checkFireWallBlocking.js.map +1 -1
- package/build/proxy/detectBlockedResponse.js +203 -0
- package/build/proxy/detectBlockedResponse.js.map +1 -0
- package/build/proxy/genReq.js +434 -127
- package/build/proxy/genReq.js.map +1 -1
- package/build/proxy/oxylabsFallback.js +110 -0
- package/build/proxy/oxylabsFallback.js.map +1 -0
- package/build/proxy/proxyFeasibilityRequest.js +11 -9
- package/build/proxy/proxyFeasibilityRequest.js.map +1 -1
- package/build/run/cancellableStep.js +27 -0
- package/build/run/cancellableStep.js.map +1 -0
- package/build/run/index.js +80 -85
- package/build/run/index.js.map +1 -1
- package/build/run/outputDirectory.js +377 -0
- package/build/run/outputDirectory.js.map +1 -0
- package/build/sourcemaps/index.js +109 -23
- package/build/sourcemaps/index.js.map +1 -1
- package/build/strings/index.js +12 -5
- package/build/strings/index.js.map +1 -1
- package/build/utility/downloadProgress.js +99 -0
- package/build/utility/downloadProgress.js.map +1 -0
- package/build/utility/makeReq.js +490 -257
- package/build/utility/makeReq.js.map +1 -1
- package/build/utility/progressLog.js +64 -4
- package/build/utility/progressLog.js.map +1 -1
- package/build/utility/targetInputs.js +177 -0
- package/build/utility/targetInputs.js.map +1 -0
- package/build/utility/urlUtils.js +37 -1
- package/build/utility/urlUtils.js.map +1 -1
- package/config.dist.yaml +251 -0
- package/docker-entrypoint.sh +16 -0
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"genReq.js","sourceRoot":"","sources":["../../src/proxy/genReq.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACH,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,gBAAgB,EAChB,qBAAqB;AACrB,6BAA6B;AAC7B,wBAAwB;AACxB,6BAA6B,EAC7B,wBAAwB,EACxB,uBAAuB,EACvB,qBAAqB,GACxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAE/F;;;;;GAKG;AACH,MAAM,GAAG,GAAG,mBAAuD,EAAE,0DAAlD,GAAW,EAAE,UAAc,EAAE;IAC5C,sDAAsD;IACtD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACD,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,8CAA8C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,8BAA8B;IAC9B,IAAI,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7F,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;QAChC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM;QACjC,WAAW,EAAE;YACT,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU;YAC1C,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU;SACjD;KACJ,CAAC,CAAC;IAEH,2BAA2B;IAC3B,MAAM,kBAAkB,GAAG,IAAI,mBAAmB,CAAC;QAC/C,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,KAAK,EAAE,SAAS;KACnB,CAAC,CAAC;IACH,MAAM,mBAAmB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAClE,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAEjB,mEAAmE;IACnE,MAAM,cAAc,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI;IACjD,oDAAoD;IACpD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,GAAG,CAAC,CACvC,CAAC;IAEF,IAAI,mBAAmB,CAAC;IACxB,IAAI,cAAc,EAAE,CAAC;QACjB,8DAA8D;QAC9D,mBAAmB,GAAG;YAClB,EAAE,EAAE,cAAc,CAAC,EAAE;SACxB,CAAC;IACN,CAAC;SAAM,CAAC;QACJ,wBAAwB;QACxB,IAAI,MAAM,CAAC;QACX,IAAI,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9D,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QAC5E,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QACD,MAAM,kBAAkB,GAAG,IAAI,qBAAqB,CAAC;YACjD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,QAAQ,EAAE,MAAM;YAChB,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,iBAAiB;SACxC,CAAC,CAAC;QACH,mBAAmB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,mBAAmB;QACnB,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC1C,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,UAAU,EAAE,KAAK;YACjB,iBAAiB,EAAE,MAAM;YACzB,iBAAiB,EAAE;gBACf,2BAA2B,EAAE,KAAK;gBAClC,kCAAkC,EAAE,KAAK;gBACzC,+BAA+B,EAAE,KAAK;gBACtC,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;gBAC3C,sCAAsC,EAAE,KAAK;gBAC7C,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,wCAAwC,EAAE,KAAK;gBAC/C,4BAA4B,EAAE,KAAK;gBACnC,yCAAyC,EAAE,KAAK;gBAChD,wCAAwC,EAAE,KAAK;gBAC/C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;aAChD;SACJ,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,yBAAyB;QACzB,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;YACpD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,UAAU,EAAE,KAAK;YACjB,qBAAqB,EAAE,KAAK;YAC5B,IAAI,EAAE,MAAM;YACZ,eAAe,EAAE,KAAK;YACtB,GAAG,EAAE,GAAG;SACX,CAAC,CAAC;QACH,MAAM,sBAAsB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACxE,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,+BAA+B;QAC/B,MAAM,wBAAwB,GAAG,IAAI,wBAAwB,CAAC;YAC1D,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,KAAK;SACpB,CAAC,CAAC;QACH,MAAM,yBAAyB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC9E,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,2BAA2B;QAC3B,MAAM,6BAA6B,GAAG,IAAI,6BAA6B,CAAC;YACpE,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,KAAK;SACpB,CAAC,CAAC;QACH,MAAM,8BAA8B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACxF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,8BAA8B;IAC9B,iGAAiG;IACjG,kFAAkF;IAElF,sBAAsB;IACtB,kEAAkE;IAClE,wCAAwC;IACxC,mCAAmC;IACnC,wGAAwG;IACxG,QAAQ;IACR,2EAA2E;IAC3E,gFAAgF;IAChF,sBAAsB;IAEtB,MAAM,qBAAqB,GAAG,IAAI,uBAAuB,CAAC;QACtD,UAAU,EAAE,KAAK;QACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;QAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,OAAO,EAAE,OAAO,IAAI,EAAE;KACzB,CAAC,CAAC;IACH,MAAM,wBAAwB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC1E,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,IAAI,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;IAEjD,sBAAsB;IACtB,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;QACpD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;KACrC,CAAC,CAAC;IACH,IAAI,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,0DAA0D,GAAG,EAAE,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,OAAO,IAAI,CAAC;IAEZ,qBAAqB;IACrB,2FAA2F;IAC3F,qDAAqD;IACrD,wCAAwC;IACxC,6DAA6D;IAC7D,6EAA6E;IAC7E,kCAAkC;IAClC,2EAA2E;IAC3E,wBAAwB;IACxB,UAAU;IACV,2BAA2B;IAC3B,4GAA4G;IAC5G,mCAAmC;IACnC,mCAAmC;IACnC,WAAW;IACX,SAAS;IACT,QAAQ;IACR,iEAAiE;IACjE,sBAAsB;IACtB,mCAAmC;AACvC,CAAC,CAAA,CAAC;AAEF,OAAO,EAAE,GAAG,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"genReq.js","sourceRoot":"","sources":["../../src/proxy/genReq.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACH,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,gBAAgB,EAChB,qBAAqB;AACrB,6BAA6B;AAC7B,wBAAwB;AACxB,6BAA6B,EAC7B,wBAAwB,EACxB,uBAAuB,EACvB,qBAAqB,GACxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,4BAA4B,GAAG,KAAM,CAAC;AAC5C,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAM,CAAC;AACxC,MAAM,wBAAwB,GAAG,CAAC,GAAG,KAAM,CAAC;AAC5C,MAAM,yBAAyB,GAAG,KAAM,CAAC;AACzC,MAAM,wBAAwB,GAAG,EAAE,GAAG,KAAM,CAAC;AAC7C,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAyB,CAAC;AAiB5D,MAAM,eAAe,GAAG,CAAC,KAAc,EAA6B,EAAE;IAClE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,SAAS,GAAG,KAAyD,CAAC;IAC5E,OAAO,CACH,SAAS,CAAC,OAAO,KAAK,CAAC;QACvB,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ;QACtC,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC3C,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ;QACxC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACjD,OAAO,SAAS,CAAC,cAAc,KAAK,QAAQ;QAC5C,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACrD,OAAO,SAAS,CAAC,WAAW,KAAK,QAAQ;QACzC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ;QACxC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC;QAC1C,SAAS,CAAC,UAAU,IAAI,CAAC;QACzB,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ;QACvC,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ;QACpC,OAAO,SAAS,CAAC,gBAAgB,KAAK,QAAQ;QAC9C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC;QAClD,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS;YAC/B,CAAC,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ,IAAI,wBAAwB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CACzG,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,6BAA6B,GAAG,CAAO,GAAW,EAAE,IAAyB,EAAiB,EAAE;IAClG,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,EAAE,CAAC;IACvB,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC;QACD,MAAM,OAAO,CAAC;IAClB,CAAC;YAAS,CAAC;QACP,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,OAAO;YAAE,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChF,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;;GAKG;AACH,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,MAAoB,EAAiB,EAAE,CAC9D,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IAC5B,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,EAAE,CAAC;QAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtB,OAAO;IACX,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,EAAE;QAChB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7C,OAAO,EAAE,CAAC;IACd,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,GAAG,EAAE;QAChB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAQP;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,mBAIO,EAAE,0DAH7B,GAAW,EACX,UAAkC,EAAE,EACpC,MAAoB;;IAEpB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO;QAAE,MAAM,MAAM,CAAC,MAAM,CAAC;IAEzC,sDAAsD;IACtD,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACjB,IAAI,CAAC;YACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8CAA8C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;IACL,CAAC,CAAC,EAAE,CAAC;IACL,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE;YACT,WAAW,EAAE,OAAO,CAAC,UAAU;YAC/B,eAAe,EAAE,OAAO,CAAC,UAAU;SACtC;KACJ,CAAC,CAAC;IACH,MAAM,mBAAmB,GAAG,IAAI,eAAe,EAAE,CAAC;IAClD,MAAM,gBAAgB,GAAG,UAAU,CAC/B,GAAG,EAAE,CACD,mBAAmB,CAAC,KAAK,CACrB,IAAI,YAAY,CAAC,+DAA+D,EAAE,cAAc,CAAC,CACpG,EACL,wBAAwB,CAC3B,CAAC;IACF,MAAA,gBAAgB,CAAC,KAAK,gEAAI,CAAC;IAC3B,MAAM,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC;IACpH,MAAM,WAAW,GAAG,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC;IACrD,MAAM,WAAW,GAAG,UAAU,EAAE,CAAC;IACjC,8EAA8E;IAC9E,+EAA+E;IAC/E,kFAAkF;IAClF,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,gBAAgB,GAAG,GAAG,OAAO,CAAC,kBAAkB,EAAE,UAAU,CAAC;IACnE,MAAM,iBAAiB,GAAG,CAAC,QAAgB,EAAU,EAAE,CAAC,GAAG,gBAAgB,IAAI,QAAQ,OAAO,CAAC;IAE/F,MAAM,oBAAoB,GAAG,CAAC,MAAwB,EAAW,EAAE;QAC/D,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,GAAG,UAAU,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAC;QACzE,IAAI,CAAC;YACD,EAAE,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACjE,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACvG,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,yDAAyD,GAAG,EAAE,CAAC,CAAC,CAAC;YACzF,OAAO,KAAK,CAAC;QACjB,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC;gBACD,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;oBAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;YACnE,CAAC;YAAC,WAAM,CAAC;gBACL,wEAAwE;YAC5E,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,mBAAmB,GAAG,CAAC,MAAwB,EAAQ,EAAE;QAC3D,IAAI,CAAC;YACD,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5F,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACpB,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,kEAAkE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,GAAqB,EAAE;QACjD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAqB,MAAM,CAAC,MAAM,CAAC;YAC3C,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,WAAW;YACrB,UAAU,EAAE,UAAU,CAAC,WAAW,EAAE;YACpC,cAAc,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,oBAAoB,CAAC,CAAC,WAAW,EAAE;YACnF,WAAW,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,oBAAoB,CAAC,CAAC,WAAW,EAAE;YAChF,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;QACxG,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,GAAgC,EAAE;QACzD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC;YAAE,OAAO,EAAE,CAAC;QAChD,IAAI,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,CAChB,EAAE;iBACG,WAAW,CAAC,gBAAgB,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBACtD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;iBACjE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC;oBACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAC9B,EAAE,CAAC,YAAY,CAAC,GAAG,gBAAgB,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAC/D,CAAC;oBACF,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,mBAAM,MAAM,EAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzE,CAAC;gBAAC,WAAM,CAAC;oBACL,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,CAAC,CACT,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,+DAA+D,GAAG,EAAE,CAAC,CAAC,CAAC;YAC/F,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,MAAwB,EAAQ,EAAE;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACvB,yBAAyB,GAAG,SAAA,CAAC,EAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA,EAChE,wBAAwB,CAC3B,CAAC;QACF,oBAAoB,CAChB,MAAM,CAAC,MAAM,iCACN,MAAM,KACT,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,EACjC,WAAW,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE,IAC9D,CACL,CAAC;IACN,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,CAAC,GAAY,EAAW,EAAE;;QAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC1D,MAAM,SAAS,GAAG,GAGjB,CAAC;QACF,OAAO,SAAS,CAAC,IAAI,KAAK,mBAAmB,IAAI,CAAA,MAAA,SAAS,CAAC,SAAS,0CAAE,cAAc,MAAK,GAAG,CAAC;IACjG,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,0BAIH,EAAE,iEAHlB,UAAkB,EAClB,gBAAkC,MAAM,EACxC,YAAoB,OAAO,CAAC,EAAE;QAE9B,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;YACpD,SAAS;YACT,UAAU;SACb,CAAC,CAAC;QACH,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,sEAAsE;gBACtE,wEAAwE;gBACxE,MAAM,aAAa,CAAC,IAAI,CAAC,qBAAqB,EAAE,cAAc,EAAE,CAAC,CAAC;gBAClE,OAAO,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,eAAe,CAAC,GAAG,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACtC,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;oBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC;QACD,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,+DAA+D,SAAS,EAAE,CAAC,CAAC,CAAC;QACrG,OAAO,KAAK,CAAC;IACjB,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,YAU3B,EAAE,mDATA,gBAAkC,MAAM,EACxC,YAAoB,OAAO,CAAC,EAAE;;QAS9B,MAAM,KAAK,GAKL,EAAE,CAAC;QACT,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,IAAI,QAA4B,CAAC;QACjC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,qBAAqB,EAAE,IAAI,EAAE,EAAE,CAAC;YACtD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CACtC,IAAI,mBAAmB,iBAAG,SAAS,EAAE,KAAK,EAAE,GAAG,IAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAG,EACrF,cAAc,EAAE,CACnB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,SAAS,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,CAAC,QAAQ;gBAAE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrD,IAAI,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;YACrF,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACtC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAClC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,oCAAoC,qBAAqB,iBAAiB,CAAC,CAAC;IAChG,CAAC,CAAA,CAAC;IAEF,MAAM,iBAAiB,GAAG,CACtB,gBAAwB,EACxB,kBAA2B,EACG,EAAE;;QAChC,IAAI,SAAkB,CAAC;QACvB,IAAI,uBAAuB,GAAG,KAAK,CAAC;QACpC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,MAAM,SAAS,GAAG,MAAM,oBAAoB,EAAE,CAAC;gBAC/C,uBAAuB,GAAG,IAAI,CAAC;gBAC/B,MAAM,UAAU,GAAG,MAAA,SAAS,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CACL,IAAI,CAAC,QAAQ,KAAK,gBAAgB;oBAClC,CAAC,kBAAkB,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,kBAAkB,CAAC,CAC3E,0CAAE,EAAE,CAAC;gBACN,IAAI,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;gBACzE,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,SAAS,GAAG,GAAG,CAAC;YACpB,CAAC;YACD,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;gBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,IAAI,CAAC,uBAAuB,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACtD,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,kEAAkE,SAAS,EAAE,CAAC,CAAC,CAAC;YACxG,OAAO,YAAY,CAAC;QACxB,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,CACzB,OAAoC,EACpC,aAA6B,EAC7B,YAA8B,EACjB,EAAE;;QACf,IAAI,SAAS,GAA2E,IAAI,CAAC;QAC7F,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,SAAS,GAAG,MAAM,oBAAoB,CAAC,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACV,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;oBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,aAAa,CACT,KAAK,CAAC,GAAG,CACL,gEAAgE,aAAa,CAAC,EAAE,KAAK,SAAS,EAAE,CACnG,CACJ,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAA,SAAS,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CACL,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,gBAAgB;gBACzC,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,UAAU,CAAC,CACzE,0CAAE,EAAE,CAAC;YACN,IAAI,UAAU,EAAE,CAAC;gBACb,IAAI,MAAM,cAAc,CAAC,UAAU,EAAE,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;oBACnE,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACJ,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC/B,CAAC;gBACD,SAAS;YACb,CAAC;YACD,MAAM,yBAAyB,GAC3B,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,4BAA4B,CAAC;YAC/E,IAAI,yBAAyB;gBAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,GAAwB,EAAE;QACnD,OAAA,6BAA6B,CAAC,gBAAgB,EAAE,GAAS,EAAE;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,kBAAkB,EAAE,CAAC,MAAM,CAC9C,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,GAAG,CAChG,CAAC;YACF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAExC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;YAC3C,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChD,MAAM,UAAU,GAAG,GAAG,aAAa,CAAC,MAAM,KAAK,aAAa,CAAC,EAAE,EAAE,CAAC;gBAClE,IAAI,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAC/C,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACjC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CACjC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,KAAK,aAAa,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,CAC9F,CAAC;gBACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEnC,MAAM,gBAAgB,GAAG,aAAa,KAAK,OAAO,CAAC;gBACnD,MAAM,YAAY,GAAG,gBAAgB;oBACjC,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,IAAI,gBAAgB,CAAC;wBACjB,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,WAAW,EAAE;4BACT,WAAW,EAAE,aAAa,CAAC,UAAU;4BACrC,eAAe,EAAE,aAAa,CAAC,UAAU;yBAC5C;qBACJ,CAAC,CAAC;gBACT,IAAI,CAAC;oBACD,MAAM,oBAAoB,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;gBACrE,CAAC;wBAAS,CAAC;oBACP,IAAI,CAAC,gBAAgB;wBAAE,YAAY,CAAC,OAAO,EAAE,CAAC;gBAClD,CAAC;YACL,CAAC;QACL,CAAC,CAAA,CAAC,CAAA;MAAA,CAAC;IAEP,MAAM,0BAA0B,GAAG,CAAO,MAAwB,EAAiB,EAAE;QACjF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACjE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;YAClC,aAAa,CACT,KAAK,CAAC,GAAG,CACL,8DAA8D,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CACrG,CACJ,CAAC;QACN,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,iBAAiB,GAAG,CAAO,UAAkB,EAAiB,EAAE;QAClE,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,gBAAgB,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU;YACV,UAAU,EAAE,KAAK;YACjB,iBAAiB,EAAE,MAAM;YACzB,iBAAiB,EAAE;gBACf,2BAA2B,EAAE,KAAK;gBAClC,kCAAkC,EAAE,KAAK;gBACzC,+BAA+B,EAAE,KAAK;gBACtC,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;gBAC3C,sCAAsC,EAAE,KAAK;gBAC7C,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,wCAAwC,EAAE,KAAK;gBAC/C,4BAA4B,EAAE,KAAK;gBACnC,yCAAyC,EAAE,KAAK;gBAChD,wCAAwC,EAAE,KAAK;gBAC/C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;aAChD;SACJ,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,qBAAqB,CAAC;YACtB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU;YACV,UAAU,EAAE,KAAK;YACjB,qBAAqB,EAAE,KAAK;YAC5B,IAAI,EAAE,MAAM;YACZ,eAAe,EAAE,KAAK;YACtB,GAAG,EAAE,GAAG;SACX,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,wBAAwB,CAAC;YACzB,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU,EAAE,KAAK;SACpB,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,6BAA6B,CAAC;YAC9B,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU,EAAE,KAAK;SACpB,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IACtC,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CACnB,QAAgB,EACkD,EAAE;QACpE,MAAM,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;QACrD,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC7B,IAAI,qBAAqB,CAAC;gBACtB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,QAAQ;gBACR,QAAQ,EAAE,gBAAgB;aAC7B,CAAC,EACF,WAAW,CACd,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YACjF,UAAU,GAAG,OAAO,CAAC,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,iEAAiE;YACjE,mEAAmE;YACnE,kDAAkD;YAClD,MAAM,0BAA0B,CAAC,oBAAoB,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC;QAChB,CAAC;QAED,MAAM,aAAa,GAAqB,MAAM,CAAC,MAAM,iCAC9C,oBAAoB,KACvB,UAAU,IACZ,CAAC;QACH,qEAAqE;QACrE,mDAAmD;QACnD,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAEpC,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAClC,MAAM,iBAAiB,CAAC,UAAU,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,MAAM,cAAc,CAAC,UAAU,CAAC;gBAAE,mBAAmB,CAAC,aAAa,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,IAAI,CAAC;QACD,MAAM,oBAAoB,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,oBAAoB,EAAE,CAAC;QAC/C,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAClC,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,MAAA,MAAA,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,0CAAE,EAAE,mCAAI,MAAA,SAAS,CAAC,CAAC,CAAC,0CAAE,QAAQ,CAAC;QACzF,IAAI,CAAC,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE,CAAA,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,QAAQ,GAAG,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE;YACjC,CAAC,CAAC,EAAE,EAAE,EAAE,gBAAgB,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,cAAc,CAAC,MAAO,CAAC,CAAC;QAEpC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC7B,IAAI,uBAAuB,CAAC;gBACxB,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,OAAO;aACV,CAAC,EACF,WAAW,CACd,CAAC;YACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,MAAA,OAAO,CAAC,IAAI,mCAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACP,IAAI,QAAQ,CAAC,aAAa,IAAI,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChE,mBAAmB,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,EAAE,CAAC;IACrB,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,GAAG,GAAG,mBAAiG,EAAE,0DAA5F,GAAW,EAAE,UAAkC,EAAE,EAAE,MAAoB,IACtF,OAAA,CAAC,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA,GAAA,CAAC;AAEvD,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
export class OxylabsFallbackConfigurationError extends Error {
|
|
2
|
+
constructor(message) {
|
|
3
|
+
super(message);
|
|
4
|
+
this.name = "OxylabsFallbackConfigurationError";
|
|
5
|
+
}
|
|
6
|
+
}
|
|
7
|
+
let fallbackConfiguration = Object.freeze({ enabled: false });
|
|
8
|
+
let requestsByOrigin = new Map();
|
|
9
|
+
let totalRequests = 0;
|
|
10
|
+
let fallbackOrigins = new Set();
|
|
11
|
+
const nonEmpty = (value) => {
|
|
12
|
+
const normalized = value === null || value === void 0 ? void 0 : value.trim();
|
|
13
|
+
return normalized ? normalized : undefined;
|
|
14
|
+
};
|
|
15
|
+
const nonEmptySecret = (value) => value && value.trim().length > 0 ? value : undefined;
|
|
16
|
+
export const resolveOxylabsFallback = (input) => {
|
|
17
|
+
var _a, _b, _c, _d, _e, _f;
|
|
18
|
+
if (!input.enabled)
|
|
19
|
+
return Object.freeze({ enabled: false });
|
|
20
|
+
if (!((_a = input.loadedConfig) === null || _a === void 0 ? void 0 : _a.explicitlySupplied)) {
|
|
21
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs WAF fallback requires a config selected with explicit --config or JS_RECON_CONFIG");
|
|
22
|
+
}
|
|
23
|
+
const yamlProxy = input.loadedConfig.data.oxylabs;
|
|
24
|
+
const yamlUsername = nonEmpty(yamlProxy.username);
|
|
25
|
+
const yamlPassword = nonEmptySecret(yamlProxy.password);
|
|
26
|
+
if (!yamlUsername || !yamlPassword) {
|
|
27
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs WAF fallback requires non-empty oxylabs.username and oxylabs.password in the selected YAML config");
|
|
28
|
+
}
|
|
29
|
+
if (yamlProxy.city || yamlProxy.sessionId) {
|
|
30
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs datacenter fallback does not support city or sessionId targeting; remove those values");
|
|
31
|
+
}
|
|
32
|
+
const perOriginBudget = Number(input.maxRequestsPerOrigin);
|
|
33
|
+
const totalBudget = Number((_b = input.maxTotalRequests) !== null && _b !== void 0 ? _b : 100);
|
|
34
|
+
const originBudget = Number((_c = input.maxOrigins) !== null && _c !== void 0 ? _c : 25);
|
|
35
|
+
if (!Number.isSafeInteger(perOriginBudget) || perOriginBudget < 1 || perOriginBudget > 10000) {
|
|
36
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs fallback max requests per origin must be a positive integer no greater than 10000");
|
|
37
|
+
}
|
|
38
|
+
if (!Number.isSafeInteger(totalBudget) || totalBudget < 1 || totalBudget > 100000) {
|
|
39
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs fallback max total requests must be a positive integer no greater than 100000");
|
|
40
|
+
}
|
|
41
|
+
if (!Number.isSafeInteger(originBudget) || originBudget < 1 || originBudget > 10000) {
|
|
42
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs fallback max origins must be a positive integer no greater than 10000");
|
|
43
|
+
}
|
|
44
|
+
const readEnvironment = (key, secret = false) => {
|
|
45
|
+
if (input.ignoreEnvironment || input.env[key] === undefined)
|
|
46
|
+
return undefined;
|
|
47
|
+
const value = secret ? nonEmptySecret(input.env[key]) : nonEmpty(input.env[key]);
|
|
48
|
+
if (!value) {
|
|
49
|
+
throw new OxylabsFallbackConfigurationError(`${key} cannot be empty when it is set`);
|
|
50
|
+
}
|
|
51
|
+
return value;
|
|
52
|
+
};
|
|
53
|
+
const country = (_d = readEnvironment("JS_RECON_OXYLABS_COUNTRY")) !== null && _d !== void 0 ? _d : nonEmpty(yamlProxy.country);
|
|
54
|
+
if (country && !/^[a-z]{2}$/i.test(country)) {
|
|
55
|
+
throw new OxylabsFallbackConfigurationError("Oxylabs country must be a two-letter country code");
|
|
56
|
+
}
|
|
57
|
+
const proxy = Object.freeze(Object.assign({ username: (_e = readEnvironment("JS_RECON_OXYLABS_USERNAME")) !== null && _e !== void 0 ? _e : yamlUsername, password: (_f = readEnvironment("JS_RECON_OXYLABS_PASSWORD", true)) !== null && _f !== void 0 ? _f : yamlPassword }, (country ? { country: country.toLowerCase() } : {})));
|
|
58
|
+
return Object.freeze({
|
|
59
|
+
enabled: true,
|
|
60
|
+
proxy,
|
|
61
|
+
maxRequestsPerOrigin: perOriginBudget,
|
|
62
|
+
maxTotalRequests: totalBudget,
|
|
63
|
+
maxOrigins: originBudget,
|
|
64
|
+
});
|
|
65
|
+
};
|
|
66
|
+
export const configureOxylabsFallback = (configuration) => {
|
|
67
|
+
var _a, _b;
|
|
68
|
+
fallbackConfiguration = configuration.enabled
|
|
69
|
+
? Object.freeze({
|
|
70
|
+
enabled: true,
|
|
71
|
+
proxy: Object.freeze(Object.assign({}, configuration.proxy)),
|
|
72
|
+
maxRequestsPerOrigin: configuration.maxRequestsPerOrigin,
|
|
73
|
+
maxTotalRequests: (_a = configuration.maxTotalRequests) !== null && _a !== void 0 ? _a : 100,
|
|
74
|
+
maxOrigins: (_b = configuration.maxOrigins) !== null && _b !== void 0 ? _b : 25,
|
|
75
|
+
})
|
|
76
|
+
: Object.freeze({ enabled: false });
|
|
77
|
+
requestsByOrigin = new Map();
|
|
78
|
+
totalRequests = 0;
|
|
79
|
+
fallbackOrigins = new Set();
|
|
80
|
+
};
|
|
81
|
+
export const resetOxylabsFallback = () => configureOxylabsFallback({ enabled: false });
|
|
82
|
+
export const getOxylabsFallbackConfiguration = () => fallbackConfiguration;
|
|
83
|
+
/**
|
|
84
|
+
* Atomically reserves one paid fallback request for the URL's origin.
|
|
85
|
+
* Returns a copied proxy config when the request is allowed, otherwise null.
|
|
86
|
+
*/
|
|
87
|
+
export const claimOxylabsFallback = (url) => {
|
|
88
|
+
var _a, _b, _c;
|
|
89
|
+
if (!fallbackConfiguration.enabled)
|
|
90
|
+
return null;
|
|
91
|
+
let origin;
|
|
92
|
+
try {
|
|
93
|
+
origin = new URL(url).origin;
|
|
94
|
+
}
|
|
95
|
+
catch (_d) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
const used = (_a = requestsByOrigin.get(origin)) !== null && _a !== void 0 ? _a : 0;
|
|
99
|
+
if (used >= fallbackConfiguration.maxRequestsPerOrigin)
|
|
100
|
+
return null;
|
|
101
|
+
if (totalRequests >= ((_b = fallbackConfiguration.maxTotalRequests) !== null && _b !== void 0 ? _b : 100))
|
|
102
|
+
return null;
|
|
103
|
+
if (!fallbackOrigins.has(origin) && fallbackOrigins.size >= ((_c = fallbackConfiguration.maxOrigins) !== null && _c !== void 0 ? _c : 25))
|
|
104
|
+
return null;
|
|
105
|
+
requestsByOrigin = new Map(requestsByOrigin).set(origin, used + 1);
|
|
106
|
+
fallbackOrigins = new Set(fallbackOrigins).add(origin);
|
|
107
|
+
totalRequests += 1;
|
|
108
|
+
return Object.assign({}, fallbackConfiguration.proxy);
|
|
109
|
+
};
|
|
110
|
+
//# sourceMappingURL=oxylabsFallback.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oxylabsFallback.js","sourceRoot":"","sources":["../../src/proxy/oxylabsFallback.ts"],"names":[],"mappings":"AA2BA,MAAM,OAAO,iCAAkC,SAAQ,KAAK;IACxD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mCAAmC,CAAC;IACpD,CAAC;CACJ;AAED,IAAI,qBAAqB,GAAiC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAC5F,IAAI,gBAAgB,GAAgC,IAAI,GAAG,EAAE,CAAC;AAC9D,IAAI,aAAa,GAAG,CAAC,CAAC;AACtB,IAAI,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;AAErD,MAAM,QAAQ,GAAG,CAAC,KAAgC,EAAsB,EAAE;IACtE,MAAM,UAAU,GAAG,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,EAAE,CAAC;IACjC,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAgC,EAAsB,EAAE,CAC5E,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAEzD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,KAAkC,EAAgC,EAAE;;IACvG,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC,CAAA,MAAA,KAAK,CAAC,YAAY,0CAAE,kBAAkB,CAAA,EAAE,CAAC;QAC1C,MAAM,IAAI,iCAAiC,CACvC,2FAA2F,CAC9F,CAAC;IACN,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,cAAc,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,iCAAiC,CACvC,2GAA2G,CAC9G,CAAC;IACN,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,IAAI,iCAAiC,CACvC,+FAA+F,CAClG,CAAC;IACN,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAA,KAAK,CAAC,gBAAgB,mCAAI,GAAG,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,CAAC,MAAA,KAAK,CAAC,UAAU,mCAAI,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,KAAM,EAAE,CAAC;QAC5F,MAAM,IAAI,iCAAiC,CACvC,2FAA2F,CAC9F,CAAC;IACN,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,MAAO,EAAE,CAAC;QACjF,MAAM,IAAI,iCAAiC,CACvC,uFAAuF,CAC1F,CAAC;IACN,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,KAAM,EAAE,CAAC;QACnF,MAAM,IAAI,iCAAiC,CACvC,+EAA+E,CAClF,CAAC;IACN,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,KAAK,EAAsB,EAAE;QACxE,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC9E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,iCAAiC,CAAC,GAAG,GAAG,iCAAiC,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,MAAA,eAAe,CAAC,0BAA0B,CAAC,mCAAI,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3F,IAAI,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,iCAAiC,CAAC,mDAAmD,CAAC,CAAC;IACrG,CAAC;IACD,MAAM,KAAK,GAAkB,MAAM,CAAC,MAAM,iBACtC,QAAQ,EAAE,MAAA,eAAe,CAAC,2BAA2B,CAAC,mCAAI,YAAY,EACtE,QAAQ,EAAE,MAAA,eAAe,CAAC,2BAA2B,EAAE,IAAI,CAAC,mCAAI,YAAY,IACzE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACxD,CAAC;IAEH,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,OAAO,EAAE,IAAI;QACb,KAAK;QACL,oBAAoB,EAAE,eAAe;QACrC,gBAAgB,EAAE,WAAW;QAC7B,UAAU,EAAE,YAAY;KAC3B,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,aAA2C,EAAQ,EAAE;;IAC1F,qBAAqB,GAAG,aAAa,CAAC,OAAO;QACzC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YACV,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,MAAM,CAAC,MAAM,mBAAM,aAAa,CAAC,KAAK,EAAG;YAChD,oBAAoB,EAAE,aAAa,CAAC,oBAAoB;YACxD,gBAAgB,EAAE,MAAA,aAAa,CAAC,gBAAgB,mCAAI,GAAG;YACvD,UAAU,EAAE,MAAA,aAAa,CAAC,UAAU,mCAAI,EAAE;SAC7C,CAAC;QACJ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACxC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;IAC7B,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,GAAG,IAAI,GAAG,EAAE,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAS,EAAE,CAAC,wBAAwB,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAE7F,MAAM,CAAC,MAAM,+BAA+B,GAAG,GAAiC,EAAE,CAAC,qBAAqB,CAAC;AAEzG;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAwB,EAAE;;IACtE,IAAI,CAAC,qBAAqB,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACD,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,IAAI,GAAG,MAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,mCAAI,CAAC,CAAC;IAC/C,IAAI,IAAI,IAAI,qBAAqB,CAAC,oBAAoB;QAAE,OAAO,IAAI,CAAC;IACpE,IAAI,aAAa,IAAI,CAAC,MAAA,qBAAqB,CAAC,gBAAgB,mCAAI,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAClF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,eAAe,CAAC,IAAI,IAAI,CAAC,MAAA,qBAAqB,CAAC,UAAU,mCAAI,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAClH,gBAAgB,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;IACnE,eAAe,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvD,aAAa,IAAI,CAAC,CAAC;IACnB,yBAAY,qBAAqB,CAAC,KAAK,EAAG;AAC9C,CAAC,CAAC"}
|
|
@@ -7,24 +7,26 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
11
|
-
import { buildUndiciDispatcher } from "../utility/makeReq.js";
|
|
10
|
+
import { getWithMetadata } from "./genReq.js";
|
|
11
|
+
import { buildUndiciDispatcher, withManagedDispatcher } from "../utility/makeReq.js";
|
|
12
12
|
/**
|
|
13
13
|
* Makes a single request to `url` through the given resolved proxy method and returns the
|
|
14
|
-
* response body
|
|
15
|
-
* feasibility flow doesn't special-case AWS vs undici-dispatched methods itself.
|
|
14
|
+
* response body, status, and headers. Shared by `checkFeasibility` across all four proxy
|
|
15
|
+
* methods so the feasibility flow doesn't special-case AWS vs undici-dispatched methods itself.
|
|
16
16
|
*
|
|
17
17
|
* @param resolved - The resolved proxy config to route the request through.
|
|
18
18
|
* @param url - The target URL to request.
|
|
19
|
-
* @returns The response body
|
|
19
|
+
* @returns The response body, status, and headers.
|
|
20
20
|
*/
|
|
21
21
|
export const proxyFeasibilityRequest = (resolved, url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
22
|
if (resolved.method === "aws") {
|
|
23
|
-
return yield
|
|
23
|
+
return yield getWithMetadata(url);
|
|
24
24
|
}
|
|
25
25
|
const dispatcher = buildUndiciDispatcher(resolved);
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
return yield withManagedDispatcher(dispatcher, (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
27
|
+
const response = yield fetch(url, options);
|
|
28
|
+
const body = yield response.text();
|
|
29
|
+
return { body, status: response.status, headers: response.headers };
|
|
30
|
+
}));
|
|
29
31
|
});
|
|
30
32
|
//# sourceMappingURL=proxyFeasibilityRequest.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxyFeasibilityRequest.js","sourceRoot":"","sources":["../../src/proxy/proxyFeasibilityRequest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"proxyFeasibilityRequest.js","sourceRoot":"","sources":["../../src/proxy/proxyFeasibilityRequest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AASrF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACnC,QAA6B,EAC7B,GAAW,EACsB,EAAE;IACnC,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,OAAO,MAAM,qBAAqB,CAAC,UAAU,EAAE,CAAO,OAAO,EAAE,EAAE;QAC7D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;IACxE,CAAC,CAAA,CAAC,CAAC;AACP,CAAC,CAAA,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Runs an interruptible step without abandoning its work promise. If a skip
|
|
12
|
+
* request wins the race, the step receives an abort signal and this function
|
|
13
|
+
* still waits for its cleanup boundary before returning to the pipeline.
|
|
14
|
+
*/
|
|
15
|
+
export const awaitCancellableStep = (work, cancellationRequested) => __awaiter(void 0, void 0, void 0, function* () {
|
|
16
|
+
const controller = new AbortController();
|
|
17
|
+
let cancellationWon = false;
|
|
18
|
+
const workPromise = work(controller.signal);
|
|
19
|
+
const cancellationPromise = cancellationRequested.then(() => {
|
|
20
|
+
cancellationWon = true;
|
|
21
|
+
controller.abort();
|
|
22
|
+
});
|
|
23
|
+
yield Promise.race([workPromise, cancellationPromise]);
|
|
24
|
+
if (cancellationWon)
|
|
25
|
+
yield workPromise;
|
|
26
|
+
});
|
|
27
|
+
//# sourceMappingURL=cancellableStep.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cancellableStep.js","sourceRoot":"","sources":["../../src/run/cancellableStep.ts"],"names":[],"mappings":";;;;;;;;;AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,IAA4C,EAC5C,qBAAoC,EACvB,EAAE;IACf,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,GAAG,EAAE;QACxD,eAAe,GAAG,IAAI,CAAC;QACvB,UAAU,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvD,IAAI,eAAe;QAAE,MAAM,WAAW,CAAC;AAC3C,CAAC,CAAA,CAAC"}
|
package/build/run/index.js
CHANGED
|
@@ -14,7 +14,6 @@ import * as globalsUtil from "../utility/globals.js";
|
|
|
14
14
|
import * as fs from "fs";
|
|
15
15
|
import lazyLoad from "../lazyLoad/index.js";
|
|
16
16
|
import chalk from "chalk";
|
|
17
|
-
import CONFIG from "../globalConfig.js";
|
|
18
17
|
import analyze from "../analyze/index.js";
|
|
19
18
|
import report from "../report/index.js";
|
|
20
19
|
import refactor from "../refactor/index.js";
|
|
@@ -25,8 +24,13 @@ import path from "path";
|
|
|
25
24
|
import { installSigintHandler, removeSigintHandler, getSkipStepPromise, resetSkipStep, shouldSkipTarget, resetSkipTarget, waitForPendingInterrupt, } from "./interruptHandler.js";
|
|
26
25
|
import { detectBundler } from "./bundler-detect.js";
|
|
27
26
|
import configureProxy from "../utility/configureProxy.js";
|
|
27
|
+
import { getOxylabsFallbackConfiguration } from "../proxy/oxylabsFallback.js";
|
|
28
28
|
import { probeFeasibility } from "../proxy/checkFeasibility.js";
|
|
29
29
|
import { getResolvedProxyConfigFromGlobals } from "../utility/makeReq.js";
|
|
30
|
+
import { resolveHostOutputDirectory } from "../lazyLoad/outputPath.js";
|
|
31
|
+
import { awaitCancellableStep } from "./cancellableStep.js";
|
|
32
|
+
import { resolveTargetInputs } from "../utility/targetInputs.js";
|
|
33
|
+
import { clearRunOutputDirectory, getBatchTargetDirectoryName, reserveRunOutputDirectory } from "./outputDirectory.js";
|
|
30
34
|
/**
|
|
31
35
|
* Determines the directory for a Content Delivery Network (CDN) if used by the target.
|
|
32
36
|
*
|
|
@@ -37,17 +41,16 @@ import { getResolvedProxyConfigFromGlobals } from "../utility/makeReq.js";
|
|
|
37
41
|
* @param outputDir - The base output directory
|
|
38
42
|
* @returns Promise that resolves to the path of the CDN directory or undefined if no CDN is detected
|
|
39
43
|
*/
|
|
40
|
-
const getCdnDir = (host, outputDir) => __awaiter(void 0, void 0, void 0, function* () {
|
|
44
|
+
const getCdnDir = (host, outputDir, isBatch) => __awaiter(void 0, void 0, void 0, function* () {
|
|
41
45
|
// get the JS URLs
|
|
42
46
|
let cdnDir;
|
|
43
47
|
for (const url of getJsUrls()) {
|
|
44
48
|
if (url.includes("_next/static/chunks")) {
|
|
45
49
|
// check if the host and url.host match
|
|
46
|
-
const urlHostDir = new URL(url).host.replace(":", "_"); // e.g. example.com_8443
|
|
47
50
|
const urlHost = new URL(url).host; // e.g. example.com:8443
|
|
48
51
|
const initialHost = new URL(host).host; // e.g. example.com:443
|
|
49
52
|
if (urlHost !== initialHost) {
|
|
50
|
-
cdnDir =
|
|
53
|
+
cdnDir = resolveHostOutputDirectory(outputDir, urlHost, isBatch);
|
|
51
54
|
break;
|
|
52
55
|
}
|
|
53
56
|
}
|
|
@@ -74,7 +77,7 @@ const getCdnDir = (host, outputDir) => __awaiter(void 0, void 0, void 0, functio
|
|
|
74
77
|
*/
|
|
75
78
|
const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_1) => __awaiter(void 0, [url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_1], void 0, function* (url, outputDir, workingDir, cmd, isBatch, includeMethods = [], excludeMethods = []) {
|
|
76
79
|
var _a, _b, _c, _d;
|
|
77
|
-
const targetHost = new URL(url).host
|
|
80
|
+
const targetHost = new URL(url).host;
|
|
78
81
|
console.log(chalk.bgGreenBright(`[+] Starting analysis for ${url}...`));
|
|
79
82
|
if (isBatch) {
|
|
80
83
|
clearJsUrls();
|
|
@@ -109,10 +112,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
109
112
|
}
|
|
110
113
|
console.log(chalk.bgCyan("[1/8] Running lazyload to download JavaScript files..."));
|
|
111
114
|
resetSkipStep();
|
|
112
|
-
yield
|
|
113
|
-
lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
|
|
114
|
-
getSkipStepPromise(),
|
|
115
|
-
]);
|
|
115
|
+
yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
|
|
116
116
|
console.log(chalk.bgGreen("[+] Lazyload complete."));
|
|
117
117
|
if (shouldSkipTarget())
|
|
118
118
|
return;
|
|
@@ -194,7 +194,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
194
194
|
// of always assuming targetHost, mirroring getCdnDir's approach for the map step.
|
|
195
195
|
let reactAssetsHostDir = targetHost;
|
|
196
196
|
for (const jsUrl of getJsUrls()) {
|
|
197
|
-
const jsUrlHost = new URL(jsUrl).host
|
|
197
|
+
const jsUrlHost = new URL(jsUrl).host;
|
|
198
198
|
if (jsUrlHost !== targetHost) {
|
|
199
199
|
reactAssetsHostDir = jsUrlHost;
|
|
200
200
|
break;
|
|
@@ -202,7 +202,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
202
202
|
}
|
|
203
203
|
resetSkipStep();
|
|
204
204
|
yield Promise.race([
|
|
205
|
-
refactor(mappedJsonFileReact, refactorOutputDirReact, detectedBundlerTechReact, false, undefined, undefined,
|
|
205
|
+
refactor(mappedJsonFileReact, refactorOutputDirReact, detectedBundlerTechReact, false, undefined, undefined, path.join(resolveHostOutputDirectory(outputDir, reactAssetsHostDir, isBatch), "assets")),
|
|
206
206
|
getSkipStepPromise(),
|
|
207
207
|
]);
|
|
208
208
|
if (shouldSkipTarget())
|
|
@@ -437,7 +437,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
437
437
|
const reportDbFile = isBatch ? `${workingDir}/js-recon.db` : "js-recon.db";
|
|
438
438
|
const reportFile = isBatch ? `${workingDir}/report` : "report";
|
|
439
439
|
const endpointsFile = isBatch ? `${workingDir}/endpoints` : "endpoints";
|
|
440
|
-
const angularHostDir =
|
|
440
|
+
const angularHostDir = resolveHostOutputDirectory(outputDir, targetHost, isBatch);
|
|
441
441
|
console.log(chalk.bgCyan("[2/4] Running map to find functions and API calls..."));
|
|
442
442
|
globalsUtil.setOpenapi(true);
|
|
443
443
|
if (isBatch) {
|
|
@@ -492,8 +492,8 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
492
492
|
// if the target is using a CDN, then just passing the outputDir/host won't work, and would throw an error.
|
|
493
493
|
// So, if the target was found to be using a CDN, scan the CDN directory rather than the outputDir/host
|
|
494
494
|
// One IMPORTANT thing: this is only meant for modules that rely on just the code (map)
|
|
495
|
-
const cdnDir = yield getCdnDir(url, outputDir);
|
|
496
|
-
const cdnOutputDir = cdnDir ? cdnDir : outputDir
|
|
495
|
+
const cdnDir = yield getCdnDir(url, outputDir, isBatch);
|
|
496
|
+
const cdnOutputDir = cdnDir ? cdnDir : resolveHostOutputDirectory(outputDir, targetHost, isBatch);
|
|
497
497
|
console.log(chalk.bgCyan("[2/8] Running strings to extract endpoints..."));
|
|
498
498
|
resetSkipStep();
|
|
499
499
|
yield Promise.race([
|
|
@@ -505,10 +505,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
505
505
|
return;
|
|
506
506
|
console.log(chalk.bgCyan("[3/8] Running lazyload with subsequent requests to download JavaScript files..."));
|
|
507
507
|
resetSkipStep();
|
|
508
|
-
yield
|
|
509
|
-
lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, true, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
|
|
510
|
-
getSkipStepPromise(),
|
|
511
|
-
]);
|
|
508
|
+
yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, true, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
|
|
512
509
|
console.log(chalk.bgGreen("[+] Lazyload with subsequent requests complete."));
|
|
513
510
|
if (shouldSkipTarget())
|
|
514
511
|
return;
|
|
@@ -527,10 +524,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
527
524
|
// updated paths picks up chunks for those dynamic routes (e.g. the post page).
|
|
528
525
|
console.log(chalk.bgCyan("[4.5/8] Re-running lazyload with subsequent requests for newly discovered paths..."));
|
|
529
526
|
resetSkipStep();
|
|
530
|
-
yield
|
|
531
|
-
lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
|
|
532
|
-
getSkipStepPromise(),
|
|
533
|
-
]);
|
|
527
|
+
yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
|
|
534
528
|
console.log(chalk.bgGreen("[+] Lazyload re-pass complete."));
|
|
535
529
|
if (shouldSkipTarget())
|
|
536
530
|
return;
|
|
@@ -566,9 +560,10 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
566
560
|
return;
|
|
567
561
|
console.log(chalk.bgCyan("[6/8] Running endpoints to extract endpoints..."));
|
|
568
562
|
resetSkipStep();
|
|
569
|
-
|
|
563
|
+
const targetOutputDir = resolveHostOutputDirectory(outputDir, targetHost, isBatch);
|
|
564
|
+
if (fs.existsSync(path.join(targetOutputDir, "___subsequent_requests"))) {
|
|
570
565
|
yield Promise.race([
|
|
571
|
-
endpoints(url,
|
|
566
|
+
endpoints(url, targetOutputDir, endpointsFile, ["json"], "next", false, mappedJsonFile),
|
|
572
567
|
getSkipStepPromise(),
|
|
573
568
|
]);
|
|
574
569
|
}
|
|
@@ -636,93 +631,93 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
|
|
|
636
631
|
* @returns Promise that resolves when all URL processing is complete
|
|
637
632
|
*/
|
|
638
633
|
export default (cmd) => __awaiter(void 0, void 0, void 0, function* () {
|
|
639
|
-
var _a, _b, _c, _d, _e;
|
|
634
|
+
var _a, _b, _c, _d, _e, _f;
|
|
635
|
+
const resolvedTargets = resolveTargetInputs(cmd.url);
|
|
640
636
|
configureProxy(cmd);
|
|
637
|
+
if (getOxylabsFallbackConfiguration().enabled)
|
|
638
|
+
globalsUtil.setUseProxy(false);
|
|
641
639
|
// Snapshot the originally-configured proxy state so `processUrl`'s --proxy-waf-fallback check
|
|
642
640
|
// can reset it before each target in batch mode, regardless of what a previous target left it as.
|
|
643
|
-
cmd._proxyConfigured = globalsUtil.getUseProxy();
|
|
644
641
|
globalsUtil.setDisableCache(cmd.disableCache);
|
|
645
642
|
globalsUtil.setRespCacheFile(cmd.cacheFile);
|
|
646
643
|
globalsUtil.setYes(cmd.yes);
|
|
647
|
-
const
|
|
644
|
+
const outputOverwrite = cmd.outputOverwrite === true;
|
|
645
|
+
const outputReservation = reserveRunOutputDirectory(cmd.output, outputOverwrite);
|
|
646
|
+
const isBatch = resolvedTargets.isBatch;
|
|
647
|
+
const runCommand = Object.freeze(Object.assign(Object.assign({}, cmd), { output: outputReservation.path, _proxyConfigured: globalsUtil.getUseProxy() }));
|
|
648
|
+
if (outputReservation.redirected) {
|
|
649
|
+
console.log(chalk.yellow(`[i] Output directory ${cmd.output} is already occupied; using ${outputReservation.path} instead.`));
|
|
650
|
+
}
|
|
651
|
+
if (outputOverwrite && outputReservation.occupied) {
|
|
652
|
+
console.log(chalk.yellow(`[!] Output directory ${runCommand.output} already exists. Overwriting it.`));
|
|
653
|
+
clearRunOutputDirectory(runCommand.output);
|
|
654
|
+
}
|
|
648
655
|
installSigintHandler(isBatch);
|
|
649
656
|
try {
|
|
650
|
-
// check if the given URL is a file
|
|
651
657
|
if (!isBatch) {
|
|
652
|
-
|
|
653
|
-
// if not done, it might conflict this process
|
|
654
|
-
// for devs: run `npm run cleanup` to prepare this directory
|
|
655
|
-
if (fs.existsSync(cmd.output)) {
|
|
656
|
-
console.error(chalk.red(`[!] Output directory ${cmd.output} already exists. Please switch to other directory or it might conflict with this process.`));
|
|
657
|
-
console.log(chalk.yellow(`[i] For advanced users: use the individual modules separately. See docs at ${CONFIG.modulesDocs}`));
|
|
658
|
-
process.exit(11);
|
|
659
|
-
}
|
|
660
|
-
try {
|
|
661
|
-
new URL(cmd.url);
|
|
662
|
-
}
|
|
663
|
-
catch (e) {
|
|
664
|
-
console.error(chalk.red(`[!] Invalid URL: ${cmd.url}`));
|
|
665
|
-
process.exit(12);
|
|
666
|
-
}
|
|
667
|
-
yield processUrl(cmd.url, cmd.output, ".", cmd, false, (_a = cmd._includeMethods) !== null && _a !== void 0 ? _a : [], (_b = cmd._excludeMethods) !== null && _b !== void 0 ? _b : []);
|
|
658
|
+
yield processUrl(resolvedTargets.targets[0], runCommand.output, ".", runCommand, false, (_a = runCommand._includeMethods) !== null && _a !== void 0 ? _a : [], (_b = runCommand._excludeMethods) !== null && _b !== void 0 ? _b : []);
|
|
668
659
|
}
|
|
669
660
|
else {
|
|
670
|
-
|
|
671
|
-
const urls = fs
|
|
672
|
-
.readFileSync(cmd.url, "utf-8")
|
|
673
|
-
.split("\n")
|
|
674
|
-
.filter((url) => url !== "");
|
|
675
|
-
if (!fs.existsSync(cmd.output)) {
|
|
676
|
-
fs.mkdirSync(cmd.output, { recursive: true });
|
|
677
|
-
}
|
|
678
|
-
const globalDbPath = `${cmd.output}/js-recon.db`;
|
|
661
|
+
const globalDbPath = `${runCommand.output}/js-recon.db`;
|
|
679
662
|
yield initGlobalReportDb(globalDbPath);
|
|
680
|
-
|
|
663
|
+
const targetHosts = resolvedTargets.targets.map((target) => new URL(target).host);
|
|
664
|
+
const hostFrequencies = targetHosts.reduce((frequencies, host) => { var _a; return new Map(frequencies).set(host, ((_a = frequencies.get(host)) !== null && _a !== void 0 ? _a : 0) + 1); }, new Map());
|
|
665
|
+
for (const url of resolvedTargets.targets) {
|
|
681
666
|
resetSkipTarget();
|
|
682
|
-
|
|
683
|
-
let urlObj;
|
|
684
|
-
|
|
685
|
-
|
|
667
|
+
const urlObj = new URL(url);
|
|
668
|
+
let hostDir = getBatchTargetDirectoryName(url, ((_c = hostFrequencies.get(urlObj.host)) !== null && _c !== void 0 ? _c : 0) > 1);
|
|
669
|
+
const requestedTargetDir = path.join(runCommand.output, hostDir);
|
|
670
|
+
const targetReservation = reserveRunOutputDirectory(requestedTargetDir, outputOverwrite);
|
|
671
|
+
const thisTargetDir = targetReservation.path;
|
|
672
|
+
hostDir = path.basename(thisTargetDir);
|
|
673
|
+
if (outputOverwrite && targetReservation.occupied) {
|
|
674
|
+
console.log(chalk.yellow(`[!] Output directory ${thisTargetDir} already exists. Overwriting it.`));
|
|
675
|
+
clearRunOutputDirectory(thisTargetDir);
|
|
686
676
|
}
|
|
687
|
-
|
|
688
|
-
console.
|
|
689
|
-
continue;
|
|
677
|
+
else if (targetReservation.redirected) {
|
|
678
|
+
console.log(chalk.yellow(`[i] Target output already exists; using ${thisTargetDir} for ${url}.`));
|
|
690
679
|
}
|
|
691
|
-
const hostDir = urlObj.host.replace(":", "_");
|
|
692
|
-
const thisTargetDir = `${cmd.output}/${hostDir}`;
|
|
693
|
-
if (fs.existsSync(thisTargetDir)) {
|
|
694
|
-
console.error(chalk.red(`[!] Output directory ${thisTargetDir} already exists. Skipping ${url}.`));
|
|
695
|
-
console.log(chalk.yellow(`[i] For advanced users: use the individual modules separately. See docs at ${CONFIG.modulesDocs}`));
|
|
696
|
-
continue;
|
|
697
|
-
}
|
|
698
|
-
fs.mkdirSync(thisTargetDir, { recursive: true });
|
|
699
680
|
try {
|
|
700
|
-
yield processUrl(url, thisTargetDir, thisTargetDir, cmd, true, (_c = cmd._includeMethods) !== null && _c !== void 0 ? _c : [], (_d = cmd._excludeMethods) !== null && _d !== void 0 ? _d : []);
|
|
701
|
-
}
|
|
702
|
-
catch (error) {
|
|
703
|
-
console.error(chalk.bgRed(`[!] Unhandled error while processing ${url}: ${error}`));
|
|
704
|
-
process.exitCode = 1;
|
|
705
|
-
continue;
|
|
706
|
-
}
|
|
707
|
-
const domainDbPath = `${thisTargetDir}/js-recon.db`;
|
|
708
|
-
if (fs.existsSync(domainDbPath)) {
|
|
709
681
|
try {
|
|
710
|
-
|
|
682
|
+
yield processUrl(url, thisTargetDir, thisTargetDir, runCommand, true, (_d = runCommand._includeMethods) !== null && _d !== void 0 ? _d : [], (_e = runCommand._excludeMethods) !== null && _e !== void 0 ? _e : []);
|
|
711
683
|
}
|
|
712
684
|
catch (error) {
|
|
713
|
-
console.error(chalk.
|
|
685
|
+
console.error(chalk.bgRed(`[!] Unhandled error while processing ${url}: ${error}`));
|
|
686
|
+
process.exitCode = 1;
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
const domainDbPath = `${thisTargetDir}/js-recon.db`;
|
|
690
|
+
if (fs.existsSync(domainDbPath)) {
|
|
691
|
+
try {
|
|
692
|
+
mergeDomainIntoGlobalDb(globalDbPath, domainDbPath, hostDir);
|
|
693
|
+
}
|
|
694
|
+
catch (error) {
|
|
695
|
+
console.error(chalk.yellow(`[!] Failed to merge ${hostDir} into the global js-recon.db: ${error}`));
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
else {
|
|
699
|
+
console.log(chalk.yellow(`[i] No js-recon.db found for ${hostDir}, skipping merge into the global database.`));
|
|
714
700
|
}
|
|
715
701
|
}
|
|
716
|
-
|
|
717
|
-
|
|
702
|
+
finally {
|
|
703
|
+
targetReservation.release();
|
|
718
704
|
}
|
|
719
705
|
}
|
|
720
706
|
}
|
|
721
707
|
}
|
|
708
|
+
catch (error) {
|
|
709
|
+
console.error(chalk.bgRed(`[!] Unhandled error: ${error}`));
|
|
710
|
+
process.exitCode = 1;
|
|
711
|
+
}
|
|
722
712
|
finally {
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
713
|
+
try {
|
|
714
|
+
yield waitForPendingInterrupt();
|
|
715
|
+
}
|
|
716
|
+
finally {
|
|
717
|
+
removeSigintHandler();
|
|
718
|
+
outputReservation.release();
|
|
719
|
+
}
|
|
720
|
+
process.exit((_f = process.exitCode) !== null && _f !== void 0 ? _f : 0);
|
|
726
721
|
}
|
|
727
722
|
});
|
|
728
723
|
//# sourceMappingURL=index.js.map
|