@js-recon/js-recon 1.4.2 → 1.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/.github/workflows/build-and-prettify.yaml +6 -6
  2. package/.github/workflows/rules-smoke-test.yaml +2 -2
  3. package/CHANGELOG.md +30 -0
  4. package/Dockerfile +13 -2
  5. package/Dockerfile.release +12 -2
  6. package/Dockerfile.w-rules +0 -1
  7. package/README.md +52 -0
  8. package/build/analyze/helpers/initRules.js +34 -16
  9. package/build/analyze/helpers/initRules.js.map +1 -1
  10. package/build/completion/index.js +44 -3
  11. package/build/completion/index.js.map +1 -1
  12. package/build/config/applicationConfig.js +493 -0
  13. package/build/config/applicationConfig.js.map +1 -0
  14. package/build/fingerprint/index.js +8 -19
  15. package/build/fingerprint/index.js.map +1 -1
  16. package/build/globalConfig.js +1 -1
  17. package/build/index.js +62 -9
  18. package/build/index.js.map +1 -1
  19. package/build/lazyLoad/downloadFilesUtil.js +40 -32
  20. package/build/lazyLoad/downloadFilesUtil.js.map +1 -1
  21. package/build/lazyLoad/downloadQueue.js +150 -50
  22. package/build/lazyLoad/downloadQueue.js.map +1 -1
  23. package/build/lazyLoad/downloadResponse.js +112 -0
  24. package/build/lazyLoad/downloadResponse.js.map +1 -0
  25. package/build/lazyLoad/globals.js +4 -0
  26. package/build/lazyLoad/globals.js.map +1 -1
  27. package/build/lazyLoad/index.js +188 -115
  28. package/build/lazyLoad/index.js.map +1 -1
  29. package/build/lazyLoad/next_js/NextJsCrawler.js +3 -2
  30. package/build/lazyLoad/next_js/NextJsCrawler.js.map +1 -1
  31. package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js +8 -9
  32. package/build/lazyLoad/next_js/next_GetLazyResourcesWebpackJs.js.map +1 -1
  33. package/build/lazyLoad/next_js/next_SubsequentRequests.js +93 -93
  34. package/build/lazyLoad/next_js/next_SubsequentRequests.js.map +1 -1
  35. package/build/lazyLoad/outputPath.js +92 -0
  36. package/build/lazyLoad/outputPath.js.map +1 -0
  37. package/build/lazyLoad/react/react_followImports.js +15 -5
  38. package/build/lazyLoad/react/react_followImports.js.map +1 -1
  39. package/build/lazyLoad/react/react_getScriptTags.js +11 -6
  40. package/build/lazyLoad/react/react_getScriptTags.js.map +1 -1
  41. package/build/lazyLoad/react/react_sourcemapUrls.js +3 -2
  42. package/build/lazyLoad/react/react_sourcemapUrls.js.map +1 -1
  43. package/build/lazyLoad/react/react_webpackChunkPaths.js +2 -1
  44. package/build/lazyLoad/react/react_webpackChunkPaths.js.map +1 -1
  45. package/build/lazyLoad/techDetect/index.js +93 -59
  46. package/build/lazyLoad/techDetect/index.js.map +1 -1
  47. package/build/lazyLoad/vue/vue_RuntimeJs.js +20 -15
  48. package/build/lazyLoad/vue/vue_RuntimeJs.js.map +1 -1
  49. package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js +8 -3
  50. package/build/lazyLoad/vue/vue_SingleJsFileOnHome.js.map +1 -1
  51. package/build/lazyLoad/vue/vue_discoverJsFiles.js +10 -9
  52. package/build/lazyLoad/vue/vue_discoverJsFiles.js.map +1 -1
  53. package/build/lazyLoad/vue/vue_getClientSidePaths.js +69 -45
  54. package/build/lazyLoad/vue/vue_getClientSidePaths.js.map +1 -1
  55. package/build/lazyLoad/vue/vue_jsImports.js +3 -2
  56. package/build/lazyLoad/vue/vue_jsImports.js.map +1 -1
  57. package/build/lazyLoad/vue/vue_pageSrc.js +2 -1
  58. package/build/lazyLoad/vue/vue_pageSrc.js.map +1 -1
  59. package/build/lazyLoad/vue/vue_reconstructSourceMaps.js +4 -3
  60. package/build/lazyLoad/vue/vue_reconstructSourceMaps.js.map +1 -1
  61. package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js +9 -20
  62. package/build/lazyLoad/vue/vue_recursiveClientSidePathDownload.js.map +1 -1
  63. package/build/lazyLoad/vue/vue_severalJsFilesHome.js +2 -1
  64. package/build/lazyLoad/vue/vue_severalJsFilesHome.js.map +1 -1
  65. package/build/lazyLoad/vue/vue_stringJsFiles.js +28 -25
  66. package/build/lazyLoad/vue/vue_stringJsFiles.js.map +1 -1
  67. package/build/lazyLoad/vue/vue_viteMapDeps.js +8 -2
  68. package/build/lazyLoad/vue/vue_viteMapDeps.js.map +1 -1
  69. package/build/proxy/checkFeasibility.js +8 -4
  70. package/build/proxy/checkFeasibility.js.map +1 -1
  71. package/build/proxy/checkFireWallBlocking.js +23 -13
  72. package/build/proxy/checkFireWallBlocking.js.map +1 -1
  73. package/build/proxy/detectBlockedResponse.js +203 -0
  74. package/build/proxy/detectBlockedResponse.js.map +1 -0
  75. package/build/proxy/genReq.js +434 -127
  76. package/build/proxy/genReq.js.map +1 -1
  77. package/build/proxy/oxylabsFallback.js +110 -0
  78. package/build/proxy/oxylabsFallback.js.map +1 -0
  79. package/build/proxy/proxyFeasibilityRequest.js +11 -9
  80. package/build/proxy/proxyFeasibilityRequest.js.map +1 -1
  81. package/build/run/cancellableStep.js +27 -0
  82. package/build/run/cancellableStep.js.map +1 -0
  83. package/build/run/index.js +80 -85
  84. package/build/run/index.js.map +1 -1
  85. package/build/run/outputDirectory.js +377 -0
  86. package/build/run/outputDirectory.js.map +1 -0
  87. package/build/sourcemaps/index.js +109 -23
  88. package/build/sourcemaps/index.js.map +1 -1
  89. package/build/strings/index.js +12 -5
  90. package/build/strings/index.js.map +1 -1
  91. package/build/utility/downloadProgress.js +99 -0
  92. package/build/utility/downloadProgress.js.map +1 -0
  93. package/build/utility/makeReq.js +490 -257
  94. package/build/utility/makeReq.js.map +1 -1
  95. package/build/utility/progressLog.js +64 -4
  96. package/build/utility/progressLog.js.map +1 -1
  97. package/build/utility/targetInputs.js +177 -0
  98. package/build/utility/targetInputs.js.map +1 -0
  99. package/build/utility/urlUtils.js +37 -1
  100. package/build/utility/urlUtils.js.map +1 -1
  101. package/config.dist.yaml +251 -0
  102. package/docker-entrypoint.sh +16 -0
  103. package/package.json +2 -2
@@ -1 +1 @@
1
- {"version":3,"file":"genReq.js","sourceRoot":"","sources":["../../src/proxy/genReq.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACH,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,gBAAgB,EAChB,qBAAqB;AACrB,6BAA6B;AAC7B,wBAAwB;AACxB,6BAA6B,EAC7B,wBAAwB,EACxB,uBAAuB,EACvB,qBAAqB,GACxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAE/F;;;;;GAKG;AACH,MAAM,GAAG,GAAG,mBAAuD,EAAE,0DAAlD,GAAW,EAAE,UAAc,EAAE;IAC5C,sDAAsD;IACtD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACD,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,8CAA8C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,8BAA8B;IAC9B,IAAI,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7F,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;QAChC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM;QACjC,WAAW,EAAE;YACT,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU;YAC1C,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU;SACjD;KACJ,CAAC,CAAC;IAEH,2BAA2B;IAC3B,MAAM,kBAAkB,GAAG,IAAI,mBAAmB,CAAC;QAC/C,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,KAAK,EAAE,SAAS;KACnB,CAAC,CAAC;IACH,MAAM,mBAAmB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAClE,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAEjB,mEAAmE;IACnE,MAAM,cAAc,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI;IACjD,oDAAoD;IACpD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,GAAG,CAAC,GAAG,CAAC,CACvC,CAAC;IAEF,IAAI,mBAAmB,CAAC;IACxB,IAAI,cAAc,EAAE,CAAC;QACjB,8DAA8D;QAC9D,mBAAmB,GAAG;YAClB,EAAE,EAAE,cAAc,CAAC,EAAE;SACxB,CAAC;IACN,CAAC;SAAM,CAAC;QACJ,wBAAwB;QACxB,IAAI,MAAM,CAAC;QACX,IAAI,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9D,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QAC5E,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QACD,MAAM,kBAAkB,GAAG,IAAI,qBAAqB,CAAC;YACjD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,QAAQ,EAAE,MAAM;YAChB,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,iBAAiB;SACxC,CAAC,CAAC;QACH,mBAAmB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC5D,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,mBAAmB;QACnB,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC1C,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,UAAU,EAAE,KAAK;YACjB,iBAAiB,EAAE,MAAM;YACzB,iBAAiB,EAAE;gBACf,2BAA2B,EAAE,KAAK;gBAClC,kCAAkC,EAAE,KAAK;gBACzC,+BAA+B,EAAE,KAAK;gBACtC,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;gBAC3C,sCAAsC,EAAE,KAAK;gBAC7C,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,wCAAwC,EAAE,KAAK;gBAC/C,4BAA4B,EAAE,KAAK;gBACnC,yCAAyC,EAAE,KAAK;gBAChD,wCAAwC,EAAE,KAAK;gBAC/C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;aAChD;SACJ,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,yBAAyB;QACzB,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;YACpD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,UAAU,EAAE,KAAK;YACjB,qBAAqB,EAAE,KAAK;YAC5B,IAAI,EAAE,MAAM;YACZ,eAAe,EAAE,KAAK;YACtB,GAAG,EAAE,GAAG;SACX,CAAC,CAAC;QACH,MAAM,sBAAsB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACxE,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,+BAA+B;QAC/B,MAAM,wBAAwB,GAAG,IAAI,wBAAwB,CAAC;YAC1D,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,KAAK;SACpB,CAAC,CAAC;QACH,MAAM,yBAAyB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC9E,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjB,2BAA2B;QAC3B,MAAM,6BAA6B,GAAG,IAAI,6BAA6B,CAAC;YACpE,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;YAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;YAChC,UAAU,EAAE,KAAK;SACpB,CAAC,CAAC;QACH,MAAM,8BAA8B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACxF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,8BAA8B;IAC9B,iGAAiG;IACjG,kFAAkF;IAElF,sBAAsB;IACtB,kEAAkE;IAClE,wCAAwC;IACxC,mCAAmC;IACnC,wGAAwG;IACxG,QAAQ;IACR,2EAA2E;IAC3E,gFAAgF;IAChF,sBAAsB;IAEtB,MAAM,qBAAqB,GAAG,IAAI,uBAAuB,CAAC;QACtD,UAAU,EAAE,KAAK;QACjB,UAAU,EAAE,mBAAmB,CAAC,EAAE;QAClC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,OAAO,EAAE,OAAO,IAAI,EAAE;KACzB,CAAC,CAAC;IACH,MAAM,wBAAwB,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC1E,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,IAAI,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;IAEjD,sBAAsB;IACtB,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;QACpD,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE;QAChC,UAAU,EAAE,mBAAmB,CAAC,EAAE;KACrC,CAAC,CAAC;IACH,IAAI,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,0DAA0D,GAAG,EAAE,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,OAAO,IAAI,CAAC;IAEZ,qBAAqB;IACrB,2FAA2F;IAC3F,qDAAqD;IACrD,wCAAwC;IACxC,6DAA6D;IAC7D,6EAA6E;IAC7E,kCAAkC;IAClC,2EAA2E;IAC3E,wBAAwB;IACxB,UAAU;IACV,2BAA2B;IAC3B,4GAA4G;IAC5G,mCAAmC;IACnC,mCAAmC;IACnC,WAAW;IACX,SAAS;IACT,QAAQ;IACR,iEAAiE;IACjE,sBAAsB;IACtB,mCAAmC;AACvC,CAAC,CAAA,CAAC;AAEF,OAAO,EAAE,GAAG,EAAE,CAAC"}
1
+ {"version":3,"file":"genReq.js","sourceRoot":"","sources":["../../src/proxy/genReq.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EACH,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,gBAAgB,EAChB,qBAAqB;AACrB,6BAA6B;AAC7B,wBAAwB;AACxB,6BAA6B,EAC7B,wBAAwB,EACxB,uBAAuB,EACvB,qBAAqB,GACxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,MAAM,sBAAsB,GAAG,IAAI,CAAC;AACpC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,4BAA4B,GAAG,KAAM,CAAC;AAC5C,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAM,CAAC;AACxC,MAAM,wBAAwB,GAAG,CAAC,GAAG,KAAM,CAAC;AAC5C,MAAM,yBAAyB,GAAG,KAAM,CAAC;AACzC,MAAM,wBAAwB,GAAG,EAAE,GAAG,KAAM,CAAC;AAC7C,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAyB,CAAC;AAiB5D,MAAM,eAAe,GAAG,CAAC,KAAc,EAA6B,EAAE;IAClE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,SAAS,GAAG,KAAyD,CAAC;IAC5E,OAAO,CACH,SAAS,CAAC,OAAO,KAAK,CAAC;QACvB,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ;QACtC,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC3C,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ;QACxC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACjD,OAAO,SAAS,CAAC,cAAc,KAAK,QAAQ;QAC5C,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACrD,OAAO,SAAS,CAAC,WAAW,KAAK,QAAQ;QACzC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ;QACxC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC;QAC1C,SAAS,CAAC,UAAU,IAAI,CAAC;QACzB,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ;QACvC,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ;QACpC,OAAO,SAAS,CAAC,gBAAgB,KAAK,QAAQ;QAC9C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC;QAClD,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS;YAC/B,CAAC,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ,IAAI,wBAAwB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CACzG,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,6BAA6B,GAAG,CAAO,GAAW,EAAE,IAAyB,EAAiB,EAAE;IAClG,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,EAAE,CAAC;IACvB,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC;QACD,MAAM,OAAO,CAAC;IAClB,CAAC;YAAS,CAAC;QACP,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,OAAO;YAAE,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChF,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;;GAKG;AACH,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,MAAoB,EAAiB,EAAE,CAC9D,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IAC5B,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,EAAE,CAAC;QAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtB,OAAO;IACX,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,EAAE;QAChB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7C,OAAO,EAAE,CAAC;IACd,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,GAAG,EAAE;QAChB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAQP;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,mBAIO,EAAE,0DAH7B,GAAW,EACX,UAAkC,EAAE,EACpC,MAAoB;;IAEpB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO;QAAE,MAAM,MAAM,CAAC,MAAM,CAAC;IAEzC,sDAAsD;IACtD,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACjB,IAAI,CAAC;YACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8CAA8C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;IACL,CAAC,CAAC,EAAE,CAAC;IACL,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE;YACT,WAAW,EAAE,OAAO,CAAC,UAAU;YAC/B,eAAe,EAAE,OAAO,CAAC,UAAU;SACtC;KACJ,CAAC,CAAC;IACH,MAAM,mBAAmB,GAAG,IAAI,eAAe,EAAE,CAAC;IAClD,MAAM,gBAAgB,GAAG,UAAU,CAC/B,GAAG,EAAE,CACD,mBAAmB,CAAC,KAAK,CACrB,IAAI,YAAY,CAAC,+DAA+D,EAAE,cAAc,CAAC,CACpG,EACL,wBAAwB,CAC3B,CAAC;IACF,MAAA,gBAAgB,CAAC,KAAK,gEAAI,CAAC;IAC3B,MAAM,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC;IACpH,MAAM,WAAW,GAAG,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC;IACrD,MAAM,WAAW,GAAG,UAAU,EAAE,CAAC;IACjC,8EAA8E;IAC9E,+EAA+E;IAC/E,kFAAkF;IAClF,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,gBAAgB,GAAG,GAAG,OAAO,CAAC,kBAAkB,EAAE,UAAU,CAAC;IACnE,MAAM,iBAAiB,GAAG,CAAC,QAAgB,EAAU,EAAE,CAAC,GAAG,gBAAgB,IAAI,QAAQ,OAAO,CAAC;IAE/F,MAAM,oBAAoB,GAAG,CAAC,MAAwB,EAAW,EAAE;QAC/D,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,GAAG,UAAU,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAC;QACzE,IAAI,CAAC;YACD,EAAE,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACjE,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACvG,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,yDAAyD,GAAG,EAAE,CAAC,CAAC,CAAC;YACzF,OAAO,KAAK,CAAC;QACjB,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC;gBACD,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;oBAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;YACnE,CAAC;YAAC,WAAM,CAAC;gBACL,wEAAwE;YAC5E,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,mBAAmB,GAAG,CAAC,MAAwB,EAAQ,EAAE;QAC3D,IAAI,CAAC;YACD,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5F,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACpB,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,kEAAkE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtG,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,GAAqB,EAAE;QACjD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAqB,MAAM,CAAC,MAAM,CAAC;YAC3C,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,WAAW;YACrB,UAAU,EAAE,UAAU,CAAC,WAAW,EAAE;YACpC,cAAc,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,oBAAoB,CAAC,CAAC,WAAW,EAAE;YACnF,WAAW,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,oBAAoB,CAAC,CAAC,WAAW,EAAE;YAChF,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;QACxG,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,GAAgC,EAAE;QACzD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC;YAAE,OAAO,EAAE,CAAC;QAChD,IAAI,CAAC;YACD,OAAO,MAAM,CAAC,MAAM,CAChB,EAAE;iBACG,WAAW,CAAC,gBAAgB,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBACtD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;iBACjE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC;oBACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAC9B,EAAE,CAAC,YAAY,CAAC,GAAG,gBAAgB,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAC/D,CAAC;oBACF,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,mBAAM,MAAM,EAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzE,CAAC;gBAAC,WAAM,CAAC;oBACL,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,CAAC,CACT,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,+DAA+D,GAAG,EAAE,CAAC,CAAC,CAAC;YAC/F,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,MAAwB,EAAQ,EAAE;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACvB,yBAAyB,GAAG,SAAA,CAAC,EAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA,EAChE,wBAAwB,CAC3B,CAAC;QACF,oBAAoB,CAChB,MAAM,CAAC,MAAM,iCACN,MAAM,KACT,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,EACjC,WAAW,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE,IAC9D,CACL,CAAC;IACN,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,CAAC,GAAY,EAAW,EAAE;;QAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC1D,MAAM,SAAS,GAAG,GAGjB,CAAC;QACF,OAAO,SAAS,CAAC,IAAI,KAAK,mBAAmB,IAAI,CAAA,MAAA,SAAS,CAAC,SAAS,0CAAE,cAAc,MAAK,GAAG,CAAC;IACjG,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,0BAIH,EAAE,iEAHlB,UAAkB,EAClB,gBAAkC,MAAM,EACxC,YAAoB,OAAO,CAAC,EAAE;QAE9B,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;YACpD,SAAS;YACT,UAAU;SACb,CAAC,CAAC;QACH,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,sEAAsE;gBACtE,wEAAwE;gBACxE,MAAM,aAAa,CAAC,IAAI,CAAC,qBAAqB,EAAE,cAAc,EAAE,CAAC,CAAC;gBAClE,OAAO,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,eAAe,CAAC,GAAG,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACtC,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;oBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC;QACD,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,+DAA+D,SAAS,EAAE,CAAC,CAAC,CAAC;QACrG,OAAO,KAAK,CAAC;IACjB,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,YAU3B,EAAE,mDATA,gBAAkC,MAAM,EACxC,YAAoB,OAAO,CAAC,EAAE;;QAS9B,MAAM,KAAK,GAKL,EAAE,CAAC;QACT,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,IAAI,QAA4B,CAAC;QACjC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,qBAAqB,EAAE,IAAI,EAAE,EAAE,CAAC;YACtD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CACtC,IAAI,mBAAmB,iBAAG,SAAS,EAAE,KAAK,EAAE,GAAG,IAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAG,EACrF,cAAc,EAAE,CACnB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,SAAS,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,CAAC,QAAQ;gBAAE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrD,IAAI,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;YACrF,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACtC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAClC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,oCAAoC,qBAAqB,iBAAiB,CAAC,CAAC;IAChG,CAAC,CAAA,CAAC;IAEF,MAAM,iBAAiB,GAAG,CACtB,gBAAwB,EACxB,kBAA2B,EACG,EAAE;;QAChC,IAAI,SAAkB,CAAC;QACvB,IAAI,uBAAuB,GAAG,KAAK,CAAC;QACpC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,MAAM,SAAS,GAAG,MAAM,oBAAoB,EAAE,CAAC;gBAC/C,uBAAuB,GAAG,IAAI,CAAC;gBAC/B,MAAM,UAAU,GAAG,MAAA,SAAS,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CACL,IAAI,CAAC,QAAQ,KAAK,gBAAgB;oBAClC,CAAC,kBAAkB,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,kBAAkB,CAAC,CAC3E,0CAAE,EAAE,CAAC;gBACN,IAAI,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;gBACzE,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,SAAS,GAAG,GAAG,CAAC;YACpB,CAAC;YACD,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;gBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;YACtD,CAAC;QACL,CAAC;QACD,IAAI,CAAC,uBAAuB,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACtD,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,kEAAkE,SAAS,EAAE,CAAC,CAAC,CAAC;YACxG,OAAO,YAAY,CAAC;QACxB,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,CACzB,OAAoC,EACpC,aAA6B,EAC7B,YAA8B,EACjB,EAAE;;QACf,IAAI,SAAS,GAA2E,IAAI,CAAC;QAC7F,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,wBAAwB,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACD,SAAS,GAAG,MAAM,oBAAoB,CAAC,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACV,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,GAAG,wBAAwB,EAAE,CAAC;oBACrC,MAAM,KAAK,CAAC,0BAA0B,GAAG,OAAO,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,aAAa,CACT,KAAK,CAAC,GAAG,CACL,gEAAgE,aAAa,CAAC,EAAE,KAAK,SAAS,EAAE,CACnG,CACJ,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,MAAA,SAAS,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CACL,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,gBAAgB;gBACzC,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,UAAU,CAAC,CACzE,0CAAE,EAAE,CAAC;YACN,IAAI,UAAU,EAAE,CAAC;gBACb,IAAI,MAAM,cAAc,CAAC,UAAU,EAAE,YAAY,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;oBACnE,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACJ,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC/B,CAAC;gBACD,SAAS;YACb,CAAC;YACD,MAAM,yBAAyB,GAC3B,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,4BAA4B,CAAC;YAC/E,IAAI,yBAAyB;gBAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,oBAAoB,GAAG,GAAwB,EAAE;QACnD,OAAA,6BAA6B,CAAC,gBAAgB,EAAE,GAAS,EAAE;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,kBAAkB,EAAE,CAAC,MAAM,CAC9C,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,GAAG,CAChG,CAAC;YACF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAExC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;YAC3C,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChD,MAAM,UAAU,GAAG,GAAG,aAAa,CAAC,MAAM,KAAK,aAAa,CAAC,EAAE,EAAE,CAAC;gBAClE,IAAI,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAC/C,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACjC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CACjC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,KAAK,aAAa,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,CAC9F,CAAC;gBACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEnC,MAAM,gBAAgB,GAAG,aAAa,KAAK,OAAO,CAAC;gBACnD,MAAM,YAAY,GAAG,gBAAgB;oBACjC,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,IAAI,gBAAgB,CAAC;wBACjB,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,WAAW,EAAE;4BACT,WAAW,EAAE,aAAa,CAAC,UAAU;4BACrC,eAAe,EAAE,aAAa,CAAC,UAAU;yBAC5C;qBACJ,CAAC,CAAC;gBACT,IAAI,CAAC;oBACD,MAAM,oBAAoB,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;gBACrE,CAAC;wBAAS,CAAC;oBACP,IAAI,CAAC,gBAAgB;wBAAE,YAAY,CAAC,OAAO,EAAE,CAAC;gBAClD,CAAC;YACL,CAAC;QACL,CAAC,CAAA,CAAC,CAAA;MAAA,CAAC;IAEP,MAAM,0BAA0B,GAAG,CAAO,MAAwB,EAAiB,EAAE;QACjF,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACjE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;YAClC,aAAa,CACT,KAAK,CAAC,GAAG,CACL,8DAA8D,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CACrG,CACJ,CAAC;QACN,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,MAAM,iBAAiB,GAAG,CAAO,UAAkB,EAAiB,EAAE;QAClE,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,gBAAgB,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU;YACV,UAAU,EAAE,KAAK;YACjB,iBAAiB,EAAE,MAAM;YACzB,iBAAiB,EAAE;gBACf,2BAA2B,EAAE,KAAK;gBAClC,kCAAkC,EAAE,KAAK;gBACzC,+BAA+B,EAAE,KAAK;gBACtC,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,uCAAuC,EAAE,KAAK;gBAC9C,oCAAoC,EAAE,KAAK;gBAC3C,sCAAsC,EAAE,KAAK;gBAC7C,8BAA8B,EAAE,KAAK;gBACrC,uCAAuC,EAAE,KAAK;gBAC9C,wCAAwC,EAAE,KAAK;gBAC/C,4BAA4B,EAAE,KAAK;gBACnC,yCAAyC,EAAE,KAAK;gBAChD,wCAAwC,EAAE,KAAK;gBAC/C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;gBAC7C,sCAAsC,EAAE,KAAK;aAChD;SACJ,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,qBAAqB,CAAC;YACtB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU;YACV,UAAU,EAAE,KAAK;YACjB,qBAAqB,EAAE,KAAK;YAC5B,IAAI,EAAE,MAAM;YACZ,eAAe,EAAE,KAAK;YACtB,GAAG,EAAE,GAAG;SACX,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,wBAAwB,CAAC;YACzB,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU,EAAE,KAAK;SACpB,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAElC,MAAM,MAAM,CAAC,IAAI,CACb,IAAI,6BAA6B,CAAC;YAC9B,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,UAAU,EAAE,KAAK;SACpB,CAAC,EACF,WAAW,CACd,CAAC;QACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IACtC,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CACnB,QAAgB,EACkD,EAAE;QACpE,MAAM,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;QACrD,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC7B,IAAI,qBAAqB,CAAC;gBACtB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,QAAQ;gBACR,QAAQ,EAAE,gBAAgB;aAC7B,CAAC,EACF,WAAW,CACd,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YACjF,UAAU,GAAG,OAAO,CAAC,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,iEAAiE;YACjE,mEAAmE;YACnE,kDAAkD;YAClD,MAAM,0BAA0B,CAAC,oBAAoB,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC;QAChB,CAAC;QAED,MAAM,aAAa,GAAqB,MAAM,CAAC,MAAM,iCAC9C,oBAAoB,KACvB,UAAU,IACZ,CAAC;QACH,qEAAqE;QACrE,mDAAmD;QACnD,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAEpC,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAClC,MAAM,iBAAiB,CAAC,UAAU,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,MAAM,cAAc,CAAC,UAAU,CAAC;gBAAE,mBAAmB,CAAC,aAAa,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC,CAAA,CAAC;IAEF,IAAI,CAAC;QACD,MAAM,oBAAoB,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,oBAAoB,EAAE,CAAC;QAC/C,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAClC,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,MAAA,MAAA,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,0CAAE,EAAE,mCAAI,MAAA,SAAS,CAAC,CAAC,CAAC,0CAAE,QAAQ,CAAC;QACzF,IAAI,CAAC,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE,CAAA,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,QAAQ,GAAG,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE;YACjC,CAAC,CAAC,EAAE,EAAE,EAAE,gBAAgB,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,cAAc,CAAC,MAAO,CAAC,CAAC;QAEpC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC7B,IAAI,uBAAuB,CAAC;gBACxB,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,OAAO;aACV,CAAC,EACF,WAAW,CACd,CAAC;YACF,MAAM,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,MAAA,OAAO,CAAC,IAAI,mCAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACP,IAAI,QAAQ,CAAC,aAAa,IAAI,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChE,mBAAmB,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,EAAE,CAAC;IACrB,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,GAAG,GAAG,mBAAiG,EAAE,0DAA5F,GAAW,EAAE,UAAkC,EAAE,EAAE,MAAoB,IACtF,OAAA,CAAC,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA,GAAA,CAAC;AAEvD,OAAO,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC"}
@@ -0,0 +1,110 @@
1
+ export class OxylabsFallbackConfigurationError extends Error {
2
+ constructor(message) {
3
+ super(message);
4
+ this.name = "OxylabsFallbackConfigurationError";
5
+ }
6
+ }
7
+ let fallbackConfiguration = Object.freeze({ enabled: false });
8
+ let requestsByOrigin = new Map();
9
+ let totalRequests = 0;
10
+ let fallbackOrigins = new Set();
11
+ const nonEmpty = (value) => {
12
+ const normalized = value === null || value === void 0 ? void 0 : value.trim();
13
+ return normalized ? normalized : undefined;
14
+ };
15
+ const nonEmptySecret = (value) => value && value.trim().length > 0 ? value : undefined;
16
+ export const resolveOxylabsFallback = (input) => {
17
+ var _a, _b, _c, _d, _e, _f;
18
+ if (!input.enabled)
19
+ return Object.freeze({ enabled: false });
20
+ if (!((_a = input.loadedConfig) === null || _a === void 0 ? void 0 : _a.explicitlySupplied)) {
21
+ throw new OxylabsFallbackConfigurationError("Oxylabs WAF fallback requires a config selected with explicit --config or JS_RECON_CONFIG");
22
+ }
23
+ const yamlProxy = input.loadedConfig.data.oxylabs;
24
+ const yamlUsername = nonEmpty(yamlProxy.username);
25
+ const yamlPassword = nonEmptySecret(yamlProxy.password);
26
+ if (!yamlUsername || !yamlPassword) {
27
+ throw new OxylabsFallbackConfigurationError("Oxylabs WAF fallback requires non-empty oxylabs.username and oxylabs.password in the selected YAML config");
28
+ }
29
+ if (yamlProxy.city || yamlProxy.sessionId) {
30
+ throw new OxylabsFallbackConfigurationError("Oxylabs datacenter fallback does not support city or sessionId targeting; remove those values");
31
+ }
32
+ const perOriginBudget = Number(input.maxRequestsPerOrigin);
33
+ const totalBudget = Number((_b = input.maxTotalRequests) !== null && _b !== void 0 ? _b : 100);
34
+ const originBudget = Number((_c = input.maxOrigins) !== null && _c !== void 0 ? _c : 25);
35
+ if (!Number.isSafeInteger(perOriginBudget) || perOriginBudget < 1 || perOriginBudget > 10000) {
36
+ throw new OxylabsFallbackConfigurationError("Oxylabs fallback max requests per origin must be a positive integer no greater than 10000");
37
+ }
38
+ if (!Number.isSafeInteger(totalBudget) || totalBudget < 1 || totalBudget > 100000) {
39
+ throw new OxylabsFallbackConfigurationError("Oxylabs fallback max total requests must be a positive integer no greater than 100000");
40
+ }
41
+ if (!Number.isSafeInteger(originBudget) || originBudget < 1 || originBudget > 10000) {
42
+ throw new OxylabsFallbackConfigurationError("Oxylabs fallback max origins must be a positive integer no greater than 10000");
43
+ }
44
+ const readEnvironment = (key, secret = false) => {
45
+ if (input.ignoreEnvironment || input.env[key] === undefined)
46
+ return undefined;
47
+ const value = secret ? nonEmptySecret(input.env[key]) : nonEmpty(input.env[key]);
48
+ if (!value) {
49
+ throw new OxylabsFallbackConfigurationError(`${key} cannot be empty when it is set`);
50
+ }
51
+ return value;
52
+ };
53
+ const country = (_d = readEnvironment("JS_RECON_OXYLABS_COUNTRY")) !== null && _d !== void 0 ? _d : nonEmpty(yamlProxy.country);
54
+ if (country && !/^[a-z]{2}$/i.test(country)) {
55
+ throw new OxylabsFallbackConfigurationError("Oxylabs country must be a two-letter country code");
56
+ }
57
+ const proxy = Object.freeze(Object.assign({ username: (_e = readEnvironment("JS_RECON_OXYLABS_USERNAME")) !== null && _e !== void 0 ? _e : yamlUsername, password: (_f = readEnvironment("JS_RECON_OXYLABS_PASSWORD", true)) !== null && _f !== void 0 ? _f : yamlPassword }, (country ? { country: country.toLowerCase() } : {})));
58
+ return Object.freeze({
59
+ enabled: true,
60
+ proxy,
61
+ maxRequestsPerOrigin: perOriginBudget,
62
+ maxTotalRequests: totalBudget,
63
+ maxOrigins: originBudget,
64
+ });
65
+ };
66
+ export const configureOxylabsFallback = (configuration) => {
67
+ var _a, _b;
68
+ fallbackConfiguration = configuration.enabled
69
+ ? Object.freeze({
70
+ enabled: true,
71
+ proxy: Object.freeze(Object.assign({}, configuration.proxy)),
72
+ maxRequestsPerOrigin: configuration.maxRequestsPerOrigin,
73
+ maxTotalRequests: (_a = configuration.maxTotalRequests) !== null && _a !== void 0 ? _a : 100,
74
+ maxOrigins: (_b = configuration.maxOrigins) !== null && _b !== void 0 ? _b : 25,
75
+ })
76
+ : Object.freeze({ enabled: false });
77
+ requestsByOrigin = new Map();
78
+ totalRequests = 0;
79
+ fallbackOrigins = new Set();
80
+ };
81
+ export const resetOxylabsFallback = () => configureOxylabsFallback({ enabled: false });
82
+ export const getOxylabsFallbackConfiguration = () => fallbackConfiguration;
83
+ /**
84
+ * Atomically reserves one paid fallback request for the URL's origin.
85
+ * Returns a copied proxy config when the request is allowed, otherwise null.
86
+ */
87
+ export const claimOxylabsFallback = (url) => {
88
+ var _a, _b, _c;
89
+ if (!fallbackConfiguration.enabled)
90
+ return null;
91
+ let origin;
92
+ try {
93
+ origin = new URL(url).origin;
94
+ }
95
+ catch (_d) {
96
+ return null;
97
+ }
98
+ const used = (_a = requestsByOrigin.get(origin)) !== null && _a !== void 0 ? _a : 0;
99
+ if (used >= fallbackConfiguration.maxRequestsPerOrigin)
100
+ return null;
101
+ if (totalRequests >= ((_b = fallbackConfiguration.maxTotalRequests) !== null && _b !== void 0 ? _b : 100))
102
+ return null;
103
+ if (!fallbackOrigins.has(origin) && fallbackOrigins.size >= ((_c = fallbackConfiguration.maxOrigins) !== null && _c !== void 0 ? _c : 25))
104
+ return null;
105
+ requestsByOrigin = new Map(requestsByOrigin).set(origin, used + 1);
106
+ fallbackOrigins = new Set(fallbackOrigins).add(origin);
107
+ totalRequests += 1;
108
+ return Object.assign({}, fallbackConfiguration.proxy);
109
+ };
110
+ //# sourceMappingURL=oxylabsFallback.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"oxylabsFallback.js","sourceRoot":"","sources":["../../src/proxy/oxylabsFallback.ts"],"names":[],"mappings":"AA2BA,MAAM,OAAO,iCAAkC,SAAQ,KAAK;IACxD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mCAAmC,CAAC;IACpD,CAAC;CACJ;AAED,IAAI,qBAAqB,GAAiC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAC5F,IAAI,gBAAgB,GAAgC,IAAI,GAAG,EAAE,CAAC;AAC9D,IAAI,aAAa,GAAG,CAAC,CAAC;AACtB,IAAI,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;AAErD,MAAM,QAAQ,GAAG,CAAC,KAAgC,EAAsB,EAAE;IACtE,MAAM,UAAU,GAAG,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,EAAE,CAAC;IACjC,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAgC,EAAsB,EAAE,CAC5E,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAEzD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,KAAkC,EAAgC,EAAE;;IACvG,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC,CAAA,MAAA,KAAK,CAAC,YAAY,0CAAE,kBAAkB,CAAA,EAAE,CAAC;QAC1C,MAAM,IAAI,iCAAiC,CACvC,2FAA2F,CAC9F,CAAC;IACN,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,cAAc,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,iCAAiC,CACvC,2GAA2G,CAC9G,CAAC;IACN,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,IAAI,iCAAiC,CACvC,+FAA+F,CAClG,CAAC;IACN,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAA,KAAK,CAAC,gBAAgB,mCAAI,GAAG,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,CAAC,MAAA,KAAK,CAAC,UAAU,mCAAI,EAAE,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,KAAM,EAAE,CAAC;QAC5F,MAAM,IAAI,iCAAiC,CACvC,2FAA2F,CAC9F,CAAC;IACN,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,MAAO,EAAE,CAAC;QACjF,MAAM,IAAI,iCAAiC,CACvC,uFAAuF,CAC1F,CAAC;IACN,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,KAAM,EAAE,CAAC;QACnF,MAAM,IAAI,iCAAiC,CACvC,+EAA+E,CAClF,CAAC;IACN,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAW,EAAE,MAAM,GAAG,KAAK,EAAsB,EAAE;QACxE,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC9E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,iCAAiC,CAAC,GAAG,GAAG,iCAAiC,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,MAAA,eAAe,CAAC,0BAA0B,CAAC,mCAAI,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3F,IAAI,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,iCAAiC,CAAC,mDAAmD,CAAC,CAAC;IACrG,CAAC;IACD,MAAM,KAAK,GAAkB,MAAM,CAAC,MAAM,iBACtC,QAAQ,EAAE,MAAA,eAAe,CAAC,2BAA2B,CAAC,mCAAI,YAAY,EACtE,QAAQ,EAAE,MAAA,eAAe,CAAC,2BAA2B,EAAE,IAAI,CAAC,mCAAI,YAAY,IACzE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACxD,CAAC;IAEH,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,OAAO,EAAE,IAAI;QACb,KAAK;QACL,oBAAoB,EAAE,eAAe;QACrC,gBAAgB,EAAE,WAAW;QAC7B,UAAU,EAAE,YAAY;KAC3B,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,aAA2C,EAAQ,EAAE;;IAC1F,qBAAqB,GAAG,aAAa,CAAC,OAAO;QACzC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YACV,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,MAAM,CAAC,MAAM,mBAAM,aAAa,CAAC,KAAK,EAAG;YAChD,oBAAoB,EAAE,aAAa,CAAC,oBAAoB;YACxD,gBAAgB,EAAE,MAAA,aAAa,CAAC,gBAAgB,mCAAI,GAAG;YACvD,UAAU,EAAE,MAAA,aAAa,CAAC,UAAU,mCAAI,EAAE;SAC7C,CAAC;QACJ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACxC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;IAC7B,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,GAAG,IAAI,GAAG,EAAE,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAS,EAAE,CAAC,wBAAwB,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAE7F,MAAM,CAAC,MAAM,+BAA+B,GAAG,GAAiC,EAAE,CAAC,qBAAqB,CAAC;AAEzG;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAwB,EAAE;;IACtE,IAAI,CAAC,qBAAqB,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACD,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,CAAC;IAAC,WAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,IAAI,GAAG,MAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,mCAAI,CAAC,CAAC;IAC/C,IAAI,IAAI,IAAI,qBAAqB,CAAC,oBAAoB;QAAE,OAAO,IAAI,CAAC;IACpE,IAAI,aAAa,IAAI,CAAC,MAAA,qBAAqB,CAAC,gBAAgB,mCAAI,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAClF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,eAAe,CAAC,IAAI,IAAI,CAAC,MAAA,qBAAqB,CAAC,UAAU,mCAAI,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAClH,gBAAgB,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;IACnE,eAAe,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvD,aAAa,IAAI,CAAC,CAAC;IACnB,yBAAY,qBAAqB,CAAC,KAAK,EAAG;AAC9C,CAAC,CAAC"}
@@ -7,24 +7,26 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
7
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
8
8
  });
9
9
  };
10
- import { get } from "./genReq.js";
11
- import { buildUndiciDispatcher } from "../utility/makeReq.js";
10
+ import { getWithMetadata } from "./genReq.js";
11
+ import { buildUndiciDispatcher, withManagedDispatcher } from "../utility/makeReq.js";
12
12
  /**
13
13
  * Makes a single request to `url` through the given resolved proxy method and returns the
14
- * response body as text. Shared by `checkFeasibility` across all four proxy methods so the
15
- * feasibility flow doesn't special-case AWS vs undici-dispatched methods itself.
14
+ * response body, status, and headers. Shared by `checkFeasibility` across all four proxy
15
+ * methods so the feasibility flow doesn't special-case AWS vs undici-dispatched methods itself.
16
16
  *
17
17
  * @param resolved - The resolved proxy config to route the request through.
18
18
  * @param url - The target URL to request.
19
- * @returns The response body text.
19
+ * @returns The response body, status, and headers.
20
20
  */
21
21
  export const proxyFeasibilityRequest = (resolved, url) => __awaiter(void 0, void 0, void 0, function* () {
22
22
  if (resolved.method === "aws") {
23
- return yield get(url);
23
+ return yield getWithMetadata(url);
24
24
  }
25
25
  const dispatcher = buildUndiciDispatcher(resolved);
26
- const opts = dispatcher ? { dispatcher } : {};
27
- const res = yield fetch(url, opts);
28
- return yield res.text();
26
+ return yield withManagedDispatcher(dispatcher, (options) => __awaiter(void 0, void 0, void 0, function* () {
27
+ const response = yield fetch(url, options);
28
+ const body = yield response.text();
29
+ return { body, status: response.status, headers: response.headers };
30
+ }));
29
31
  });
30
32
  //# sourceMappingURL=proxyFeasibilityRequest.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"proxyFeasibilityRequest.js","sourceRoot":"","sources":["../../src/proxy/proxyFeasibilityRequest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAgC,MAAM,uBAAuB,CAAC;AAG5F;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAO,QAA6B,EAAE,GAAW,EAAmB,EAAE;IACzG,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,IAAI,GAA4B,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACnC,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;AAC5B,CAAC,CAAA,CAAC"}
1
+ {"version":3,"file":"proxyFeasibilityRequest.js","sourceRoot":"","sources":["../../src/proxy/proxyFeasibilityRequest.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AASrF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACnC,QAA6B,EAC7B,GAAW,EACsB,EAAE;IACnC,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,OAAO,MAAM,qBAAqB,CAAC,UAAU,EAAE,CAAO,OAAO,EAAE,EAAE;QAC7D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;IACxE,CAAC,CAAA,CAAC,CAAC;AACP,CAAC,CAAA,CAAC"}
@@ -0,0 +1,27 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ /**
11
+ * Runs an interruptible step without abandoning its work promise. If a skip
12
+ * request wins the race, the step receives an abort signal and this function
13
+ * still waits for its cleanup boundary before returning to the pipeline.
14
+ */
15
+ export const awaitCancellableStep = (work, cancellationRequested) => __awaiter(void 0, void 0, void 0, function* () {
16
+ const controller = new AbortController();
17
+ let cancellationWon = false;
18
+ const workPromise = work(controller.signal);
19
+ const cancellationPromise = cancellationRequested.then(() => {
20
+ cancellationWon = true;
21
+ controller.abort();
22
+ });
23
+ yield Promise.race([workPromise, cancellationPromise]);
24
+ if (cancellationWon)
25
+ yield workPromise;
26
+ });
27
+ //# sourceMappingURL=cancellableStep.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cancellableStep.js","sourceRoot":"","sources":["../../src/run/cancellableStep.ts"],"names":[],"mappings":";;;;;;;;;AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,IAA4C,EAC5C,qBAAoC,EACvB,EAAE;IACf,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,GAAG,EAAE;QACxD,eAAe,GAAG,IAAI,CAAC;QACvB,UAAU,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvD,IAAI,eAAe;QAAE,MAAM,WAAW,CAAC;AAC3C,CAAC,CAAA,CAAC"}
@@ -14,7 +14,6 @@ import * as globalsUtil from "../utility/globals.js";
14
14
  import * as fs from "fs";
15
15
  import lazyLoad from "../lazyLoad/index.js";
16
16
  import chalk from "chalk";
17
- import CONFIG from "../globalConfig.js";
18
17
  import analyze from "../analyze/index.js";
19
18
  import report from "../report/index.js";
20
19
  import refactor from "../refactor/index.js";
@@ -25,8 +24,13 @@ import path from "path";
25
24
  import { installSigintHandler, removeSigintHandler, getSkipStepPromise, resetSkipStep, shouldSkipTarget, resetSkipTarget, waitForPendingInterrupt, } from "./interruptHandler.js";
26
25
  import { detectBundler } from "./bundler-detect.js";
27
26
  import configureProxy from "../utility/configureProxy.js";
27
+ import { getOxylabsFallbackConfiguration } from "../proxy/oxylabsFallback.js";
28
28
  import { probeFeasibility } from "../proxy/checkFeasibility.js";
29
29
  import { getResolvedProxyConfigFromGlobals } from "../utility/makeReq.js";
30
+ import { resolveHostOutputDirectory } from "../lazyLoad/outputPath.js";
31
+ import { awaitCancellableStep } from "./cancellableStep.js";
32
+ import { resolveTargetInputs } from "../utility/targetInputs.js";
33
+ import { clearRunOutputDirectory, getBatchTargetDirectoryName, reserveRunOutputDirectory } from "./outputDirectory.js";
30
34
  /**
31
35
  * Determines the directory for a Content Delivery Network (CDN) if used by the target.
32
36
  *
@@ -37,17 +41,16 @@ import { getResolvedProxyConfigFromGlobals } from "../utility/makeReq.js";
37
41
  * @param outputDir - The base output directory
38
42
  * @returns Promise that resolves to the path of the CDN directory or undefined if no CDN is detected
39
43
  */
40
- const getCdnDir = (host, outputDir) => __awaiter(void 0, void 0, void 0, function* () {
44
+ const getCdnDir = (host, outputDir, isBatch) => __awaiter(void 0, void 0, void 0, function* () {
41
45
  // get the JS URLs
42
46
  let cdnDir;
43
47
  for (const url of getJsUrls()) {
44
48
  if (url.includes("_next/static/chunks")) {
45
49
  // check if the host and url.host match
46
- const urlHostDir = new URL(url).host.replace(":", "_"); // e.g. example.com_8443
47
50
  const urlHost = new URL(url).host; // e.g. example.com:8443
48
51
  const initialHost = new URL(host).host; // e.g. example.com:443
49
52
  if (urlHost !== initialHost) {
50
- cdnDir = path.join(outputDir, urlHostDir);
53
+ cdnDir = resolveHostOutputDirectory(outputDir, urlHost, isBatch);
51
54
  break;
52
55
  }
53
56
  }
@@ -74,7 +77,7 @@ const getCdnDir = (host, outputDir) => __awaiter(void 0, void 0, void 0, functio
74
77
  */
75
78
  const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_1) => __awaiter(void 0, [url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_1], void 0, function* (url, outputDir, workingDir, cmd, isBatch, includeMethods = [], excludeMethods = []) {
76
79
  var _a, _b, _c, _d;
77
- const targetHost = new URL(url).host.replace(":", "_");
80
+ const targetHost = new URL(url).host;
78
81
  console.log(chalk.bgGreenBright(`[+] Starting analysis for ${url}...`));
79
82
  if (isBatch) {
80
83
  clearJsUrls();
@@ -109,10 +112,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
109
112
  }
110
113
  console.log(chalk.bgCyan("[1/8] Running lazyload to download JavaScript files..."));
111
114
  resetSkipStep();
112
- yield Promise.race([
113
- lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
114
- getSkipStepPromise(),
115
- ]);
115
+ yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, false, "", cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
116
116
  console.log(chalk.bgGreen("[+] Lazyload complete."));
117
117
  if (shouldSkipTarget())
118
118
  return;
@@ -194,7 +194,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
194
194
  // of always assuming targetHost, mirroring getCdnDir's approach for the map step.
195
195
  let reactAssetsHostDir = targetHost;
196
196
  for (const jsUrl of getJsUrls()) {
197
- const jsUrlHost = new URL(jsUrl).host.replace(":", "_");
197
+ const jsUrlHost = new URL(jsUrl).host;
198
198
  if (jsUrlHost !== targetHost) {
199
199
  reactAssetsHostDir = jsUrlHost;
200
200
  break;
@@ -202,7 +202,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
202
202
  }
203
203
  resetSkipStep();
204
204
  yield Promise.race([
205
- refactor(mappedJsonFileReact, refactorOutputDirReact, detectedBundlerTechReact, false, undefined, undefined, `${outputDir}/${reactAssetsHostDir}/assets`),
205
+ refactor(mappedJsonFileReact, refactorOutputDirReact, detectedBundlerTechReact, false, undefined, undefined, path.join(resolveHostOutputDirectory(outputDir, reactAssetsHostDir, isBatch), "assets")),
206
206
  getSkipStepPromise(),
207
207
  ]);
208
208
  if (shouldSkipTarget())
@@ -437,7 +437,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
437
437
  const reportDbFile = isBatch ? `${workingDir}/js-recon.db` : "js-recon.db";
438
438
  const reportFile = isBatch ? `${workingDir}/report` : "report";
439
439
  const endpointsFile = isBatch ? `${workingDir}/endpoints` : "endpoints";
440
- const angularHostDir = `${outputDir}/${targetHost}`;
440
+ const angularHostDir = resolveHostOutputDirectory(outputDir, targetHost, isBatch);
441
441
  console.log(chalk.bgCyan("[2/4] Running map to find functions and API calls..."));
442
442
  globalsUtil.setOpenapi(true);
443
443
  if (isBatch) {
@@ -492,8 +492,8 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
492
492
  // if the target is using a CDN, then just passing the outputDir/host won't work, and would throw an error.
493
493
  // So, if the target was found to be using a CDN, scan the CDN directory rather than the outputDir/host
494
494
  // One IMPORTANT thing: this is only meant for modules that rely on just the code (map)
495
- const cdnDir = yield getCdnDir(url, outputDir);
496
- const cdnOutputDir = cdnDir ? cdnDir : outputDir + "/" + targetHost;
495
+ const cdnDir = yield getCdnDir(url, outputDir, isBatch);
496
+ const cdnOutputDir = cdnDir ? cdnDir : resolveHostOutputDirectory(outputDir, targetHost, isBatch);
497
497
  console.log(chalk.bgCyan("[2/8] Running strings to extract endpoints..."));
498
498
  resetSkipStep();
499
499
  yield Promise.race([
@@ -505,10 +505,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
505
505
  return;
506
506
  console.log(chalk.bgCyan("[3/8] Running lazyload with subsequent requests to download JavaScript files..."));
507
507
  resetSkipStep();
508
- yield Promise.race([
509
- lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, true, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
510
- getSkipStepPromise(),
511
- ]);
508
+ yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, true, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
512
509
  console.log(chalk.bgGreen("[+] Lazyload with subsequent requests complete."));
513
510
  if (shouldSkipTarget())
514
511
  return;
@@ -527,10 +524,7 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
527
524
  // updated paths picks up chunks for those dynamic routes (e.g. the post page).
528
525
  console.log(chalk.bgCyan("[4.5/8] Re-running lazyload with subsequent requests for newly discovered paths..."));
529
526
  resetSkipStep();
530
- yield Promise.race([
531
- lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000),
532
- getSkipStepPromise(),
533
- ]);
527
+ yield awaitCancellableStep((signal) => lazyLoad(url, outputDir, cmd.strictScope, cmd.scope.split(","), cmd.threads, true, `${extractedUrlsFile}.json`, cmd.insecure, false, cmd.sourcemapDir, cmd.research, cmd.researchOutput, Number(cmd.maxIterations), Number(cmd.maxJsSize), Number(cmd.lazyloadTimeout) * 60 * 1000, Number(cmd.maxPages), includeMethods, excludeMethods, Number(cmd.detectionTimeout) * 1000, signal, isBatch), getSkipStepPromise());
534
528
  console.log(chalk.bgGreen("[+] Lazyload re-pass complete."));
535
529
  if (shouldSkipTarget())
536
530
  return;
@@ -566,9 +560,10 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
566
560
  return;
567
561
  console.log(chalk.bgCyan("[6/8] Running endpoints to extract endpoints..."));
568
562
  resetSkipStep();
569
- if (fs.existsSync(`${outputDir}/${targetHost}/___subsequent_requests`)) {
563
+ const targetOutputDir = resolveHostOutputDirectory(outputDir, targetHost, isBatch);
564
+ if (fs.existsSync(path.join(targetOutputDir, "___subsequent_requests"))) {
570
565
  yield Promise.race([
571
- endpoints(url, `${outputDir}/${targetHost}/`, endpointsFile, ["json"], "next", false, mappedJsonFile),
566
+ endpoints(url, targetOutputDir, endpointsFile, ["json"], "next", false, mappedJsonFile),
572
567
  getSkipStepPromise(),
573
568
  ]);
574
569
  }
@@ -636,93 +631,93 @@ const processUrl = (url_1, outputDir_1, workingDir_1, cmd_1, isBatch_1, ...args_
636
631
  * @returns Promise that resolves when all URL processing is complete
637
632
  */
638
633
  export default (cmd) => __awaiter(void 0, void 0, void 0, function* () {
639
- var _a, _b, _c, _d, _e;
634
+ var _a, _b, _c, _d, _e, _f;
635
+ const resolvedTargets = resolveTargetInputs(cmd.url);
640
636
  configureProxy(cmd);
637
+ if (getOxylabsFallbackConfiguration().enabled)
638
+ globalsUtil.setUseProxy(false);
641
639
  // Snapshot the originally-configured proxy state so `processUrl`'s --proxy-waf-fallback check
642
640
  // can reset it before each target in batch mode, regardless of what a previous target left it as.
643
- cmd._proxyConfigured = globalsUtil.getUseProxy();
644
641
  globalsUtil.setDisableCache(cmd.disableCache);
645
642
  globalsUtil.setRespCacheFile(cmd.cacheFile);
646
643
  globalsUtil.setYes(cmd.yes);
647
- const isBatch = fs.existsSync(cmd.url);
644
+ const outputOverwrite = cmd.outputOverwrite === true;
645
+ const outputReservation = reserveRunOutputDirectory(cmd.output, outputOverwrite);
646
+ const isBatch = resolvedTargets.isBatch;
647
+ const runCommand = Object.freeze(Object.assign(Object.assign({}, cmd), { output: outputReservation.path, _proxyConfigured: globalsUtil.getUseProxy() }));
648
+ if (outputReservation.redirected) {
649
+ console.log(chalk.yellow(`[i] Output directory ${cmd.output} is already occupied; using ${outputReservation.path} instead.`));
650
+ }
651
+ if (outputOverwrite && outputReservation.occupied) {
652
+ console.log(chalk.yellow(`[!] Output directory ${runCommand.output} already exists. Overwriting it.`));
653
+ clearRunOutputDirectory(runCommand.output);
654
+ }
648
655
  installSigintHandler(isBatch);
649
656
  try {
650
- // check if the given URL is a file
651
657
  if (!isBatch) {
652
- // check if output directory exists. If so, ask the user to switch to other directory
653
- // if not done, it might conflict this process
654
- // for devs: run `npm run cleanup` to prepare this directory
655
- if (fs.existsSync(cmd.output)) {
656
- console.error(chalk.red(`[!] Output directory ${cmd.output} already exists. Please switch to other directory or it might conflict with this process.`));
657
- console.log(chalk.yellow(`[i] For advanced users: use the individual modules separately. See docs at ${CONFIG.modulesDocs}`));
658
- process.exit(11);
659
- }
660
- try {
661
- new URL(cmd.url);
662
- }
663
- catch (e) {
664
- console.error(chalk.red(`[!] Invalid URL: ${cmd.url}`));
665
- process.exit(12);
666
- }
667
- yield processUrl(cmd.url, cmd.output, ".", cmd, false, (_a = cmd._includeMethods) !== null && _a !== void 0 ? _a : [], (_b = cmd._excludeMethods) !== null && _b !== void 0 ? _b : []);
658
+ yield processUrl(resolvedTargets.targets[0], runCommand.output, ".", runCommand, false, (_a = runCommand._includeMethods) !== null && _a !== void 0 ? _a : [], (_b = runCommand._excludeMethods) !== null && _b !== void 0 ? _b : []);
668
659
  }
669
660
  else {
670
- // since this is a file, we need to first load the URLs in the memory remove empty strings
671
- const urls = fs
672
- .readFileSync(cmd.url, "utf-8")
673
- .split("\n")
674
- .filter((url) => url !== "");
675
- if (!fs.existsSync(cmd.output)) {
676
- fs.mkdirSync(cmd.output, { recursive: true });
677
- }
678
- const globalDbPath = `${cmd.output}/js-recon.db`;
661
+ const globalDbPath = `${runCommand.output}/js-recon.db`;
679
662
  yield initGlobalReportDb(globalDbPath);
680
- for (const url of urls) {
663
+ const targetHosts = resolvedTargets.targets.map((target) => new URL(target).host);
664
+ const hostFrequencies = targetHosts.reduce((frequencies, host) => { var _a; return new Map(frequencies).set(host, ((_a = frequencies.get(host)) !== null && _a !== void 0 ? _a : 0) + 1); }, new Map());
665
+ for (const url of resolvedTargets.targets) {
681
666
  resetSkipTarget();
682
- // Validate URL only
683
- let urlObj;
684
- try {
685
- urlObj = new URL(url);
667
+ const urlObj = new URL(url);
668
+ let hostDir = getBatchTargetDirectoryName(url, ((_c = hostFrequencies.get(urlObj.host)) !== null && _c !== void 0 ? _c : 0) > 1);
669
+ const requestedTargetDir = path.join(runCommand.output, hostDir);
670
+ const targetReservation = reserveRunOutputDirectory(requestedTargetDir, outputOverwrite);
671
+ const thisTargetDir = targetReservation.path;
672
+ hostDir = path.basename(thisTargetDir);
673
+ if (outputOverwrite && targetReservation.occupied) {
674
+ console.log(chalk.yellow(`[!] Output directory ${thisTargetDir} already exists. Overwriting it.`));
675
+ clearRunOutputDirectory(thisTargetDir);
686
676
  }
687
- catch (_f) {
688
- console.error(chalk.bgRed(`[!] Invalid URL: ${url}`));
689
- continue;
677
+ else if (targetReservation.redirected) {
678
+ console.log(chalk.yellow(`[i] Target output already exists; using ${thisTargetDir} for ${url}.`));
690
679
  }
691
- const hostDir = urlObj.host.replace(":", "_");
692
- const thisTargetDir = `${cmd.output}/${hostDir}`;
693
- if (fs.existsSync(thisTargetDir)) {
694
- console.error(chalk.red(`[!] Output directory ${thisTargetDir} already exists. Skipping ${url}.`));
695
- console.log(chalk.yellow(`[i] For advanced users: use the individual modules separately. See docs at ${CONFIG.modulesDocs}`));
696
- continue;
697
- }
698
- fs.mkdirSync(thisTargetDir, { recursive: true });
699
680
  try {
700
- yield processUrl(url, thisTargetDir, thisTargetDir, cmd, true, (_c = cmd._includeMethods) !== null && _c !== void 0 ? _c : [], (_d = cmd._excludeMethods) !== null && _d !== void 0 ? _d : []);
701
- }
702
- catch (error) {
703
- console.error(chalk.bgRed(`[!] Unhandled error while processing ${url}: ${error}`));
704
- process.exitCode = 1;
705
- continue;
706
- }
707
- const domainDbPath = `${thisTargetDir}/js-recon.db`;
708
- if (fs.existsSync(domainDbPath)) {
709
681
  try {
710
- mergeDomainIntoGlobalDb(globalDbPath, domainDbPath, hostDir);
682
+ yield processUrl(url, thisTargetDir, thisTargetDir, runCommand, true, (_d = runCommand._includeMethods) !== null && _d !== void 0 ? _d : [], (_e = runCommand._excludeMethods) !== null && _e !== void 0 ? _e : []);
711
683
  }
712
684
  catch (error) {
713
- console.error(chalk.yellow(`[!] Failed to merge ${hostDir} into the global js-recon.db: ${error}`));
685
+ console.error(chalk.bgRed(`[!] Unhandled error while processing ${url}: ${error}`));
686
+ process.exitCode = 1;
687
+ continue;
688
+ }
689
+ const domainDbPath = `${thisTargetDir}/js-recon.db`;
690
+ if (fs.existsSync(domainDbPath)) {
691
+ try {
692
+ mergeDomainIntoGlobalDb(globalDbPath, domainDbPath, hostDir);
693
+ }
694
+ catch (error) {
695
+ console.error(chalk.yellow(`[!] Failed to merge ${hostDir} into the global js-recon.db: ${error}`));
696
+ }
697
+ }
698
+ else {
699
+ console.log(chalk.yellow(`[i] No js-recon.db found for ${hostDir}, skipping merge into the global database.`));
714
700
  }
715
701
  }
716
- else {
717
- console.log(chalk.yellow(`[i] No js-recon.db found for ${hostDir}, skipping merge into the global database.`));
702
+ finally {
703
+ targetReservation.release();
718
704
  }
719
705
  }
720
706
  }
721
707
  }
708
+ catch (error) {
709
+ console.error(chalk.bgRed(`[!] Unhandled error: ${error}`));
710
+ process.exitCode = 1;
711
+ }
722
712
  finally {
723
- yield waitForPendingInterrupt();
724
- removeSigintHandler();
725
- process.exit((_e = process.exitCode) !== null && _e !== void 0 ? _e : 0);
713
+ try {
714
+ yield waitForPendingInterrupt();
715
+ }
716
+ finally {
717
+ removeSigintHandler();
718
+ outputReservation.release();
719
+ }
720
+ process.exit((_f = process.exitCode) !== null && _f !== void 0 ? _f : 0);
726
721
  }
727
722
  });
728
723
  //# sourceMappingURL=index.js.map