apify 2.3.1-beta.4 → 3.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/README.md +6 -5
  2. package/package.json +69 -128
  3. package/build/actor.d.ts +0 -113
  4. package/build/actor.d.ts.map +0 -1
  5. package/build/actor.js +0 -582
  6. package/build/actor.js.map +0 -1
  7. package/build/apify.d.ts +0 -752
  8. package/build/apify.d.ts.map +0 -1
  9. package/build/apify.js +0 -877
  10. package/build/apify.js.map +0 -1
  11. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  12. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  13. package/build/autoscaling/autoscaled_pool.js +0 -557
  14. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  15. package/build/autoscaling/snapshotter.d.ts +0 -278
  16. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  17. package/build/autoscaling/snapshotter.js +0 -447
  18. package/build/autoscaling/snapshotter.js.map +0 -1
  19. package/build/autoscaling/system_status.d.ts +0 -224
  20. package/build/autoscaling/system_status.d.ts.map +0 -1
  21. package/build/autoscaling/system_status.js +0 -228
  22. package/build/autoscaling/system_status.js.map +0 -1
  23. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  24. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  25. package/build/browser_launchers/browser_launcher.js +0 -160
  26. package/build/browser_launchers/browser_launcher.js.map +0 -1
  27. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  28. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  29. package/build/browser_launchers/browser_plugin.js +0 -25
  30. package/build/browser_launchers/browser_plugin.js.map +0 -1
  31. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  32. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  33. package/build/browser_launchers/playwright_launcher.js +0 -150
  34. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  35. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  36. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  37. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  38. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  39. package/build/cache_container.d.ts +0 -31
  40. package/build/cache_container.d.ts.map +0 -1
  41. package/build/cache_container.js +0 -48
  42. package/build/cache_container.js.map +0 -1
  43. package/build/configuration.d.ts +0 -226
  44. package/build/configuration.d.ts.map +0 -1
  45. package/build/configuration.js +0 -325
  46. package/build/configuration.js.map +0 -1
  47. package/build/constants.d.ts +0 -37
  48. package/build/constants.d.ts.map +0 -1
  49. package/build/constants.js +0 -41
  50. package/build/constants.js.map +0 -1
  51. package/build/crawlers/basic_crawler.d.ts +0 -443
  52. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  53. package/build/crawlers/basic_crawler.js +0 -664
  54. package/build/crawlers/basic_crawler.js.map +0 -1
  55. package/build/crawlers/browser_crawler.d.ts +0 -512
  56. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  57. package/build/crawlers/browser_crawler.js +0 -540
  58. package/build/crawlers/browser_crawler.js.map +0 -1
  59. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  60. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  61. package/build/crawlers/cheerio_crawler.js +0 -913
  62. package/build/crawlers/cheerio_crawler.js.map +0 -1
  63. package/build/crawlers/crawler_extension.d.ts +0 -10
  64. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  65. package/build/crawlers/crawler_extension.js +0 -19
  66. package/build/crawlers/crawler_extension.js.map +0 -1
  67. package/build/crawlers/crawler_utils.d.ts +0 -34
  68. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  69. package/build/crawlers/crawler_utils.js +0 -87
  70. package/build/crawlers/crawler_utils.js.map +0 -1
  71. package/build/crawlers/playwright_crawler.d.ts +0 -448
  72. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  73. package/build/crawlers/playwright_crawler.js +0 -299
  74. package/build/crawlers/playwright_crawler.js.map +0 -1
  75. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  76. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  77. package/build/crawlers/puppeteer_crawler.js +0 -299
  78. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  79. package/build/crawlers/statistics.d.ts +0 -185
  80. package/build/crawlers/statistics.d.ts.map +0 -1
  81. package/build/crawlers/statistics.js +0 -331
  82. package/build/crawlers/statistics.js.map +0 -1
  83. package/build/enqueue_links/click_elements.d.ts +0 -179
  84. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  85. package/build/enqueue_links/click_elements.js +0 -434
  86. package/build/enqueue_links/click_elements.js.map +0 -1
  87. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  88. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  89. package/build/enqueue_links/enqueue_links.js +0 -163
  90. package/build/enqueue_links/enqueue_links.js.map +0 -1
  91. package/build/enqueue_links/shared.d.ts +0 -42
  92. package/build/enqueue_links/shared.d.ts.map +0 -1
  93. package/build/enqueue_links/shared.js +0 -121
  94. package/build/enqueue_links/shared.js.map +0 -1
  95. package/build/errors.d.ts +0 -29
  96. package/build/errors.d.ts.map +0 -1
  97. package/build/errors.js +0 -38
  98. package/build/errors.js.map +0 -1
  99. package/build/events.d.ts +0 -11
  100. package/build/events.d.ts.map +0 -1
  101. package/build/events.js +0 -147
  102. package/build/events.js.map +0 -1
  103. package/build/index.d.ts +0 -4
  104. package/build/index.d.ts.map +0 -1
  105. package/build/index.js +0 -7
  106. package/build/index.js.map +0 -1
  107. package/build/main.d.ts +0 -179
  108. package/build/main.d.ts.map +0 -1
  109. package/build/main.js +0 -81
  110. package/build/main.js.map +0 -1
  111. package/build/playwright_utils.d.ts +0 -9
  112. package/build/playwright_utils.d.ts.map +0 -1
  113. package/build/playwright_utils.js +0 -90
  114. package/build/playwright_utils.js.map +0 -1
  115. package/build/proxy_configuration.d.ts +0 -411
  116. package/build/proxy_configuration.d.ts.map +0 -1
  117. package/build/proxy_configuration.js +0 -517
  118. package/build/proxy_configuration.js.map +0 -1
  119. package/build/pseudo_url.d.ts +0 -86
  120. package/build/pseudo_url.d.ts.map +0 -1
  121. package/build/pseudo_url.js +0 -153
  122. package/build/pseudo_url.js.map +0 -1
  123. package/build/puppeteer_request_interception.d.ts +0 -8
  124. package/build/puppeteer_request_interception.d.ts.map +0 -1
  125. package/build/puppeteer_request_interception.js +0 -235
  126. package/build/puppeteer_request_interception.js.map +0 -1
  127. package/build/puppeteer_utils.d.ts +0 -250
  128. package/build/puppeteer_utils.d.ts.map +0 -1
  129. package/build/puppeteer_utils.js +0 -551
  130. package/build/puppeteer_utils.js.map +0 -1
  131. package/build/request.d.ts +0 -180
  132. package/build/request.d.ts.map +0 -1
  133. package/build/request.js +0 -261
  134. package/build/request.js.map +0 -1
  135. package/build/request_list.d.ts +0 -581
  136. package/build/request_list.d.ts.map +0 -1
  137. package/build/request_list.js +0 -826
  138. package/build/request_list.js.map +0 -1
  139. package/build/serialization.d.ts +0 -5
  140. package/build/serialization.d.ts.map +0 -1
  141. package/build/serialization.js +0 -139
  142. package/build/serialization.js.map +0 -1
  143. package/build/session_pool/errors.d.ts +0 -11
  144. package/build/session_pool/errors.d.ts.map +0 -1
  145. package/build/session_pool/errors.js +0 -18
  146. package/build/session_pool/errors.js.map +0 -1
  147. package/build/session_pool/events.d.ts +0 -5
  148. package/build/session_pool/events.d.ts.map +0 -1
  149. package/build/session_pool/events.js +0 -6
  150. package/build/session_pool/events.js.map +0 -1
  151. package/build/session_pool/session.d.ts +0 -286
  152. package/build/session_pool/session.d.ts.map +0 -1
  153. package/build/session_pool/session.js +0 -355
  154. package/build/session_pool/session.js.map +0 -1
  155. package/build/session_pool/session_pool.d.ts +0 -280
  156. package/build/session_pool/session_pool.d.ts.map +0 -1
  157. package/build/session_pool/session_pool.js +0 -393
  158. package/build/session_pool/session_pool.js.map +0 -1
  159. package/build/session_pool/session_utils.d.ts +0 -4
  160. package/build/session_pool/session_utils.d.ts.map +0 -1
  161. package/build/session_pool/session_utils.js +0 -24
  162. package/build/session_pool/session_utils.js.map +0 -1
  163. package/build/stealth/hiding_tricks.d.ts +0 -22
  164. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  165. package/build/stealth/hiding_tricks.js +0 -308
  166. package/build/stealth/hiding_tricks.js.map +0 -1
  167. package/build/stealth/stealth.d.ts +0 -56
  168. package/build/stealth/stealth.d.ts.map +0 -1
  169. package/build/stealth/stealth.js +0 -125
  170. package/build/stealth/stealth.js.map +0 -1
  171. package/build/storages/dataset.d.ts +0 -288
  172. package/build/storages/dataset.d.ts.map +0 -1
  173. package/build/storages/dataset.js +0 -480
  174. package/build/storages/dataset.js.map +0 -1
  175. package/build/storages/key_value_store.d.ts +0 -243
  176. package/build/storages/key_value_store.d.ts.map +0 -1
  177. package/build/storages/key_value_store.js +0 -462
  178. package/build/storages/key_value_store.js.map +0 -1
  179. package/build/storages/request_queue.d.ts +0 -318
  180. package/build/storages/request_queue.d.ts.map +0 -1
  181. package/build/storages/request_queue.js +0 -636
  182. package/build/storages/request_queue.js.map +0 -1
  183. package/build/storages/storage_manager.d.ts +0 -87
  184. package/build/storages/storage_manager.d.ts.map +0 -1
  185. package/build/storages/storage_manager.js +0 -150
  186. package/build/storages/storage_manager.js.map +0 -1
  187. package/build/tsconfig.tsbuildinfo +0 -1
  188. package/build/typedefs.d.ts +0 -146
  189. package/build/typedefs.d.ts.map +0 -1
  190. package/build/typedefs.js +0 -88
  191. package/build/typedefs.js.map +0 -1
  192. package/build/utils.d.ts +0 -175
  193. package/build/utils.d.ts.map +0 -1
  194. package/build/utils.js +0 -731
  195. package/build/utils.js.map +0 -1
  196. package/build/utils_log.d.ts +0 -41
  197. package/build/utils_log.d.ts.map +0 -1
  198. package/build/utils_log.js +0 -192
  199. package/build/utils_log.js.map +0 -1
  200. package/build/utils_request.d.ts +0 -77
  201. package/build/utils_request.d.ts.map +0 -1
  202. package/build/utils_request.js +0 -385
  203. package/build/utils_request.js.map +0 -1
  204. package/build/utils_social.d.ts +0 -210
  205. package/build/utils_social.d.ts.map +0 -1
  206. package/build/utils_social.js +0 -787
  207. package/build/utils_social.js.map +0 -1
  208. package/build/validators.d.ts +0 -23
  209. package/build/validators.d.ts.map +0 -1
  210. package/build/validators.js +0 -29
  211. package/build/validators.js.map +0 -1
@@ -1 +0,0 @@
1
- {"version":3,"file":"basic_crawler.js","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":";;;;AAAA,0CAAkD;AAClD,gDAAwD;AACxD,sDAAuC;AACvC,yEAA2B;AAC3B,4CAAgE;AAChE,kGAA4D,CAAC,2CAA2C;AACxG,oEAA+B;AAC/B,+DAA+D,CAAC,2CAA2C;AAC3G,2EAAsC;AACtC,0EAAsC,CAAC,2CAA2C;AAClF,8CAA2C;AAa3C,iFAAiF;AAEjF;;;;;;;GAOG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH,MAAa,YAAY;IA4BrB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,qBAAqB,EACrB,wBAAwB,GAAG,EAAE,EAC7B,2BAA2B,EAC3B,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc;QAEd,WAAW;QACX,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,GAC5D,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE;YAC/B,MAAM,GAAG,GAAG,mGAAmG,CAAC;YAChH,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SAClD;QAED,6DAA6D;QAC7D,kBAAkB;QAClB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,2BAA2B,CAAC;QACxD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,GAAG,IAAI,CAAC;QAClE,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnD,4CAA4C;QAC5C,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAChI,iGAAiG;QACjG,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,YAAY,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACxE;QACD,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,CAAC,CAAC;QAC9F,iCAAiC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,EAAE;YACpD,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,0BAA0B,EAAE;kBACrE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,CAAC;SACtD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAG;YAC5C,cAAc;YACd,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,oBAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,uCAAuC,EAAE,qBAAqB,CAAC,CAAC;QAE5G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,wEAAwE;QACxE,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1E,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC;QAExD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,gBAAgB,CAAC;SAC/B;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,gBAAgB;gBAChB,cAAc;gBACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBAChD,GAAG,UAAU;aAChB,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,IAAA,8BAAe,EAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,gHAAgH;8BAChH,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,+CAA+C;8BACjE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;IAChD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAErD,IAAI,OAAO,CAAC;QACZ,IAAI,OAAO,CAAC;QAEZ,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,yCAAyC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CACvF,CAAC;QAEF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;gBACP,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAClD,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,oCAAoC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAClF,CAAC;SACL;QAED,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,MAAM,eAAe,GAAG;YACpB,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,OAAO;YACP,OAAO;SACV,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,EAClD,IAAI,CAAC,0BAA0B,EAC/B,yCAAyC,IAAI,CAAC,0BAA0B,GAAG,IAAI,aAAa,OAAO,CAAC,EAAE,IAAI,CAC7G,CAAC;YAEF,MAAM,IAAI,CAAC,gBAAgB,CACvB,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,EACxC,IAAI,CAAC,qBAAqB,EAC1B,mBAAmB,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC3H,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAI,CAAC,gBAAgB,CACvB,GAAG,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,CAAC,EACrE,IAAI,CAAC,qBAAqB,EAC1B,+BAA+B,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,qBAAqB,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC5H,CAAC;aACL;YAAC,OAAO,cAAc,EAAE;gBACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,oDAAoD;sBACjF,6GAA6G;sBAC7G,qGAAqG;sBACrG,4GAA4G;sBAC5G,sGAAsG,CAAC,CAAC;gBAC9G,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC;QACvE,IAAI;YACA,MAAM,IAAA,6BAAmB,EAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;SACtD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,OAAO,IAAI,UAAU,EAAE,EAAE,0CAA0C;gBACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,cAAc,OAAO,IAAI,UAAU,GAAG,CAAC,CAAC;gBACrE,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;aAClF;YAED,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB;QACtB,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,0BAA0B;QAC5B,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,4BAA4B,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC3F,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,KAAK,EACL,mFAAmF,EACnF,oBAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9C,eAAe,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,MAAM,IAAI,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC,CAAC,yCAAyC;SACtG;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,4BAA4B,CAAC,eAAe;QAC9C,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,eAAe,CAAC,KAAK,EACrB,4CAA4C,EAC5C,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;SACL;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,wBAAwB;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,IAAI;QAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;SACrC;IACL,CAAC;;AA3gBL,oCA4gBC;AA3gBG;;;GAGG;AACI;;;;WAAe;QAClB,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,qBAAqB,EAAE,YAAE,CAAC,QAAQ;QAClC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAElC,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC;AAofN;;;;GAIG;AACH;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;GAMG"}
@@ -1,512 +0,0 @@
1
- /**
2
- * @typedef BrowserCrawlingContext
3
- * @property {BrowserController} browserController
4
- */
5
- /**
6
- * @callback Hook
7
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
8
- * @param {Object<string,*>} gotoOptions
9
- * @returns {Promise<void>}
10
- */
11
- /**
12
- * @callback BrowserHandlePageFunction
13
- * @param {BrowserCrawlingContext & CrawlingContext} context
14
- * @returns {Promise<void>}
15
- */
16
- /**
17
- * @callback GotoFunction
18
- * @param {BrowserCrawlingContext & CrawlingContext} context
19
- * @param {Object<string,*>} gotoOptions
20
- * @returns {Promise<*>}
21
- */
22
- /**
23
- * @typedef BrowserCrawlerOptions
24
- * @property {BrowserHandlePageFunction} handlePageFunction
25
- * Function that is called to process each request.
26
- * It is passed an object with the following fields:
27
- *
28
- * ```
29
- * {
30
- * request: Request,
31
- * response: Response,
32
- * page: Page,
33
- * session: Session,
34
- * browserController: BrowserController,
35
- * proxyInfo: ProxyInfo,
36
- * crawler: BrowserCrawler,
37
- * }
38
- * ```
39
- *
40
- * `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
41
- * `page` is an instance of the `Puppeteer`
42
- * [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright`
43
- * [`Page`](https://playwright.dev/docs/api/class-page)
44
- * `browserPool` is an instance of the
45
- * [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
46
- * `browserController` is an instance of the
47
- * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
48
- * `response` is an instance of the `Puppeteer`
49
- * [`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
50
- * [`Response`](https://playwright.dev/docs/api/class-response),
51
- * which is the main resource response as returned by `page.goto(request.url)`.
52
- * The function must return a promise, which is then awaited by the crawler.
53
- *
54
- * If the function throws an exception, the crawler will try to re-crawl the
55
- * request later, up to `option.maxRequestRetries` times.
56
- * If all the retries fail, the crawler calls the function
57
- * provided to the `handleFailedRequestFunction` parameter.
58
- * To make this work, you should **always**
59
- * let your function throw exceptions rather than catch them.
60
- * The exceptions are logged to the request using the
61
- * {@link Request#pushErrorMessage} function.
62
- * @property {number} [handlePageTimeoutSecs=60]
63
- * Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
64
- * @property {GotoFunction} [gotoFunction]
65
- * Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
66
- * @property {HandleFailedRequest} [handleFailedRequestFunction]
67
- * A function to handle requests that failed more than `option.maxRequestRetries` times.
68
- *
69
- * The function receives the following object as an argument:
70
- * ```
71
- * {
72
- * request: Request,
73
- * response: Response,
74
- * page: Page,
75
- * browserPool: BrowserPool,
76
- * autoscaledPool: AutoscaledPool,
77
- * session: Session,
78
- * browserController: BrowserController,
79
- * proxyInfo: ProxyInfo,
80
- * }
81
- * ```
82
- * Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
83
- * represents the last error thrown during processing of the request.
84
- * @property {BrowserPoolOptions} [browserPoolOptions]
85
- * Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
86
- * You can tweak those to fine-tune browser management.
87
- * @property {boolean} [persistCookiesPerSession=true]
88
- * Automatically saves cookies to Session. Works only if Session Pool is used.
89
- * @property {ProxyConfiguration} [proxyConfiguration]
90
- * If set, `PuppeteerCrawler` will be configured for all connections to use
91
- * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
92
- * For more information, see the [documentation](https://docs.apify.com/proxy).
93
- * @property {Array<Hook>} [preNavigationHooks]
94
- * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
95
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
96
- * which are passed to the `page.goto()` function the crawler calls to navigate.
97
- * Example:
98
- * ```
99
- * preNavigationHooks: [
100
- * async (crawlingContext, gotoOptions) => {
101
- * const { page } = crawlingContext;
102
- * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
103
- * },
104
- * ]
105
- * ```
106
- * @property {Array<Hook>} [postNavigationHooks]
107
- * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
108
- * The function accepts `crawlingContext` as the only parameter.
109
- * Example:
110
- * ```
111
- * postNavigationHooks: [
112
- * async (crawlingContext) => {
113
- * const { page } = crawlingContext;
114
- * if (hasCaptcha(page)) {
115
- * await solveCaptcha (page);
116
- * }
117
- * },
118
- * ]
119
- * ```
120
- * @property {RequestList} [requestList]
121
- * Static list of URLs to be processed.
122
- * Either `requestList` or `requestQueue` option must be provided (or both).
123
- * @property {RequestQueue} [requestQueue]
124
- * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
125
- * Either `requestList` or `requestQueue` option must be provided (or both).
126
- * @property {number} [handleRequestTimeoutSecs=60]
127
- * Timeout in which the function passed as `handleRequestFunction` needs to finish, in seconds.
128
- * @property {HandleFailedRequest} [handleFailedRequestFunction]
129
- * A function to handle requests that failed more than `option.maxRequestRetries` times.
130
- *
131
- * The function receives the following object as an argument:
132
- * ```
133
- * {
134
- * request: Request,
135
- * error: Error,
136
- * session: Session,
137
- * crawler: BrowserCrawler,
138
- * }
139
- * ```
140
- * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
141
- * represents the last error thrown during processing of the request.
142
- *
143
- * See
144
- * [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
145
- * for the default implementation of this function.
146
- * @property {number} [maxRequestRetries=3]
147
- * Indicates how many times the request is retried if {@link BasicCrawlerOptions.handleRequestFunction} fails.
148
- * @property {number} [maxRequestsPerCrawl]
149
- * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
150
- * Always set this value in order to prevent infinite loops in misconfigured crawlers.
151
- * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
152
- * @property {AutoscaledPoolOptions} [autoscaledPoolOptions]
153
- * Custom options passed to the underlying {@link AutoscaledPool} constructor.
154
- * Note that the `runTaskFunction` and `isTaskReadyFunction` options
155
- * are provided by `BasicCrawler` and cannot be overridden.
156
- * However, you can provide a custom implementation of `isFinishedFunction`.
157
- * @property {number} [minConcurrency=1]
158
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
159
- *
160
- * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
161
- * If you're not sure, just keep the default value and the concurrency will scale up automatically.
162
- * @property {number} [maxConcurrency=1000]
163
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
164
- * @property {boolean} [useSessionPool=true]
165
- * Browser crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
166
- * The session instance will be than available in the `handleRequestFunction`.
167
- * @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.
168
- * @ignore
169
- */
170
- /**
171
- * Provides a simple framework for parallel crawling of web pages
172
- * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
173
- * and [Playwright](https://github.com/microsoft/playwright).
174
- * The URLs to crawl are fed either from a static list of URLs
175
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
176
- *
177
- * Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data,
178
- * it is useful for crawling of websites that require to execute JavaScript.
179
- * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
180
- * which downloads the pages using raw HTTP requests and is about 10x faster. @TODO: more?
181
- *
182
- * The source URLs are represented using {@link Request} objects that are fed from
183
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BrowserCrawlerOptions.requestList}
184
- * or {@link BrowserCrawlerOptions.requestQueue} constructor options, respectively.
185
- *
186
- * If both {@link BrowserCrawlerOptions.requestList} and {@link BrowserCrawlerOptions.requestQueue} are used,
187
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
188
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
189
- *
190
- * The crawler finishes when there are no more {@link Request} objects to crawl.
191
- *
192
- * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
193
- * and then calls the function provided by user as the {@link BrowserCrawlerOptions.handlePageFunction} option.
194
- *
195
- * New pages are only opened when there is enough free CPU and memory available,
196
- * using the functionality provided by the {@link AutoscaledPool} class.
197
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions}
198
- * parameter of the `BrowserCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
199
- * {@link AutoscaledPoolOptions} are available directly in the `BrowserCrawler` constructor.
200
- *
201
- * Note that the pool of browser instances is internally managed by the {@link BrowserPool} class.
202
- *
203
- * await crawler.run();
204
- * ```
205
- * @property {Statistics} stats
206
- * Contains statistics about the current run.
207
- * @property {RequestList} [requestList]
208
- * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
209
- * Only available if used by the crawler.
210
- * @property {RequestQueue} [requestQueue]
211
- * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
212
- * Only available if used by the crawler.
213
- * @property {SessionPool} [sessionPool]
214
- * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
215
- * Only available if used by the crawler.
216
- * @property {ProxyConfiguration} [proxyConfiguration]
217
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
218
- * Only available if used by the crawler.
219
- * @property {BrowserPool} browserPool
220
- * A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
221
- * For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
222
- * @property {AutoscaledPool} autoscaledPool
223
- * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
224
- * Note that this property is only initialized after calling the {@link CheerioCrawler#run} function.
225
- * You can use it to change the concurrency settings on the fly,
226
- * to pause the crawler by calling {@link AutoscaledPool#pause}
227
- * or to abort it by calling {@link AutoscaledPool#abort}.
228
- * @ignore
229
- */
230
- export default class BrowserCrawler extends BasicCrawler {
231
- /**
232
- * @param {BrowserCrawlerOptions} options
233
- * All `BrowserCrawler` parameters are passed via an options object.
234
- */
235
- constructor(options: BrowserCrawlerOptions);
236
- handlePageFunction: BrowserHandlePageFunction;
237
- handlePageTimeoutSecs: number;
238
- handlePageTimeoutMillis: number;
239
- navigationTimeoutMillis: number;
240
- gotoFunction: GotoFunction | undefined;
241
- defaultGotoOptions: {
242
- timeout: number;
243
- };
244
- proxyConfiguration: ProxyConfiguration | undefined;
245
- /** @type {Array<Hook>} */
246
- preNavigationHooks: Array<Hook>;
247
- /** @type {Array<Hook>} */
248
- postNavigationHooks: Array<Hook>;
249
- persistCookiesPerSession: boolean;
250
- browserPool: BrowserPool<{
251
- preLaunchHooks: any[];
252
- postLaunchHooks: any[];
253
- browserPlugins: readonly import("browser-pool").BrowserPlugin<import("browser-pool/dist/abstract-classes/browser-plugin").CommonLibrary, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonBrowser, unknown, import("browser-pool").CommonPage>[];
254
- maxOpenPagesPerBrowser?: number | undefined;
255
- retireBrowserAfterPageCount?: number | undefined;
256
- operationTimeoutSecs?: number | undefined;
257
- closeInactiveBrowserAfterSecs?: number | undefined;
258
- useFingerprints?: boolean | undefined;
259
- fingerprintsOptions?: import("browser-pool").FingerprintsOptions | undefined;
260
- }, [], never, never, never, never>;
261
- /**
262
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
263
- * @param {*} page
264
- * @param {boolean} useIncognitoPages
265
- * @ignore
266
- * @protected
267
- * @internal
268
- */
269
- protected _enhanceCrawlingContextWithPageInfo(crawlingContext: BrowserCrawlingContext & CrawlingContext, page: any, useIncognitoPages: boolean): void;
270
- /**
271
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
272
- * @ignore
273
- * @protected
274
- * @internal
275
- */
276
- protected _handleNavigation(crawlingContext: BrowserCrawlingContext & CrawlingContext): Promise<void>;
277
- /**
278
- * Marks session bad in case of navigation timeout.
279
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
280
- * @param {Error} error
281
- * @ignore
282
- * @protected
283
- * @internal
284
- */
285
- protected _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext & CrawlingContext, error: Error): void;
286
- /**
287
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
288
- * @param {Object<string,*>} gotoOptions
289
- * @ignore
290
- * @protected
291
- * @internal
292
- */
293
- protected _navigationHandler(crawlingContext: BrowserCrawlingContext & CrawlingContext, gotoOptions: {
294
- [x: string]: any;
295
- }): Promise<any>;
296
- /**
297
- * Should be overriden in case of different automation library that does not support this response API.
298
- * // @TODO: This can be also done as a postNavigation hook except the loadedUrl marking.
299
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
300
- * @return {Promise<void>}
301
- * @ignore
302
- * @protected
303
- * @internal
304
- */
305
- protected _responseHandler(crawlingContext: BrowserCrawlingContext & CrawlingContext): Promise<void>;
306
- /**
307
- * @param {string} pageId
308
- * @param {*} launchContext
309
- * @ignore
310
- * @protected
311
- * @internal
312
- */
313
- protected _extendLaunchContext(pageId: string, launchContext: any): Promise<void>;
314
- /**
315
- *
316
- * @param {string} pageId
317
- * @param {BrowserController} browserController
318
- * @ignore
319
- * @protected
320
- * @internal
321
- */
322
- protected _maybeAddSessionRetiredListener(pageId: string, browserController: BrowserController): void;
323
- }
324
- export type BrowserCrawlingContext = {
325
- browserController: BrowserController;
326
- };
327
- export type Hook = (crawlingContext: BrowserCrawlingContext & CrawlingContext, gotoOptions: {
328
- [x: string]: any;
329
- }) => Promise<void>;
330
- export type BrowserHandlePageFunction = (context: BrowserCrawlingContext & CrawlingContext) => Promise<void>;
331
- export type GotoFunction = (context: BrowserCrawlingContext & CrawlingContext, gotoOptions: {
332
- [x: string]: any;
333
- }) => Promise<any>;
334
- export type BrowserCrawlerOptions = {
335
- /**
336
- * Function that is called to process each request.
337
- * It is passed an object with the following fields:
338
- *
339
- * ```
340
- * {
341
- * request: Request,
342
- * response: Response,
343
- * page: Page,
344
- * session: Session,
345
- * browserController: BrowserController,
346
- * proxyInfo: ProxyInfo,
347
- * crawler: BrowserCrawler,
348
- * }
349
- * ```
350
- *
351
- * `request` is an instance of the {@link Request } object with details about the URL to open, HTTP method etc.
352
- * `page` is an instance of the `Puppeteer`
353
- * [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright`
354
- * [`Page`](https://playwright.dev/docs/api/class-page)
355
- * `browserPool` is an instance of the
356
- * [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
357
- * `browserController` is an instance of the
358
- * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
359
- * `response` is an instance of the `Puppeteer`
360
- * [`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
361
- * [`Response`](https://playwright.dev/docs/api/class-response),
362
- * which is the main resource response as returned by `page.goto(request.url)`.
363
- * The function must return a promise, which is then awaited by the crawler.
364
- *
365
- * If the function throws an exception, the crawler will try to re-crawl the
366
- * request later, up to `option.maxRequestRetries` times.
367
- * If all the retries fail, the crawler calls the function
368
- * provided to the `handleFailedRequestFunction` parameter.
369
- * To make this work, you should **always**
370
- * let your function throw exceptions rather than catch them.
371
- * The exceptions are logged to the request using the
372
- * {@link RequestpushErrorMessage } function.
373
- */
374
- handlePageFunction: BrowserHandlePageFunction;
375
- /**
376
- * Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
377
- */
378
- handlePageTimeoutSecs?: number | undefined;
379
- /**
380
- * Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
381
- */
382
- gotoFunction?: GotoFunction | undefined;
383
- /**
384
- * A function to handle requests that failed more than `option.maxRequestRetries` times.
385
- *
386
- * The function receives the following object as an argument:
387
- * ```
388
- * {
389
- * request: Request,
390
- * response: Response,
391
- * page: Page,
392
- * browserPool: BrowserPool,
393
- * autoscaledPool: AutoscaledPool,
394
- * session: Session,
395
- * browserController: BrowserController,
396
- * proxyInfo: ProxyInfo,
397
- * }
398
- * ```
399
- * Where the {@link Request } instance corresponds to the failed request, and the `Error` instance
400
- * represents the last error thrown during processing of the request.
401
- */
402
- handleFailedRequestFunction?: HandleFailedRequest | undefined;
403
- /**
404
- * Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
405
- * You can tweak those to fine-tune browser management.
406
- */
407
- browserPoolOptions?: BrowserPoolOptions<import("browser-pool").BrowserPlugin<import("browser-pool/dist/abstract-classes/browser-plugin").CommonLibrary, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonBrowser, unknown, import("browser-pool").CommonPage>> | undefined;
408
- /**
409
- * Automatically saves cookies to Session. Works only if Session Pool is used.
410
- */
411
- persistCookiesPerSession?: boolean | undefined;
412
- /**
413
- * If set, `PuppeteerCrawler` will be configured for all connections to use
414
- * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
415
- * For more information, see the [documentation](https://docs.apify.com/proxy).
416
- */
417
- proxyConfiguration?: ProxyConfiguration | undefined;
418
- /**
419
- * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
420
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
421
- * which are passed to the `page.goto()` function the crawler calls to navigate.
422
- * Example:
423
- * ```
424
- * preNavigationHooks: [
425
- * async (crawlingContext, gotoOptions) => {
426
- * const { page } = crawlingContext;
427
- * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
428
- * },
429
- * ]
430
- * ```
431
- */
432
- preNavigationHooks?: Hook[] | undefined;
433
- /**
434
- * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
435
- * The function accepts `crawlingContext` as the only parameter.
436
- * Example:
437
- * ```
438
- * postNavigationHooks: [
439
- * async (crawlingContext) => {
440
- * const { page } = crawlingContext;
441
- * if (hasCaptcha(page)) {
442
- * await solveCaptcha (page);
443
- * }
444
- * },
445
- * ]
446
- * ```
447
- */
448
- postNavigationHooks?: Hook[] | undefined;
449
- /**
450
- * Static list of URLs to be processed.
451
- * Either `requestList` or `requestQueue` option must be provided (or both).
452
- */
453
- requestList?: RequestList | undefined;
454
- /**
455
- * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
456
- * Either `requestList` or `requestQueue` option must be provided (or both).
457
- */
458
- requestQueue?: RequestQueue | undefined;
459
- /**
460
- * Timeout in which the function passed as `handleRequestFunction` needs to finish, in seconds.
461
- */
462
- handleRequestTimeoutSecs?: number | undefined;
463
- /**
464
- * Indicates how many times the request is retried if {@link BasicCrawlerOptions.handleRequestFunction } fails.
465
- */
466
- maxRequestRetries?: number | undefined;
467
- /**
468
- * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
469
- * Always set this value in order to prevent infinite loops in misconfigured crawlers.
470
- * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
471
- */
472
- maxRequestsPerCrawl?: number | undefined;
473
- /**
474
- * Custom options passed to the underlying {@link AutoscaledPool } constructor.
475
- * Note that the `runTaskFunction` and `isTaskReadyFunction` options
476
- * are provided by `BasicCrawler` and cannot be overridden.
477
- * However, you can provide a custom implementation of `isFinishedFunction`.
478
- */
479
- autoscaledPoolOptions?: AutoscaledPoolOptions | undefined;
480
- /**
481
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
482
- *
483
- * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
484
- * If you're not sure, just keep the default value and the concurrency will scale up automatically.
485
- */
486
- minConcurrency?: number | undefined;
487
- /**
488
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
489
- */
490
- maxConcurrency?: number | undefined;
491
- /**
492
- * Browser crawler will initialize the {@link SessionPool } with the corresponding `sessionPoolOptions`.
493
- * The session instance will be than available in the `handleRequestFunction`.
494
- */
495
- useSessionPool?: boolean | undefined;
496
- /**
497
- * The configuration options for {@link SessionPool } to use.
498
- */
499
- sessionPoolOptions?: SessionPoolOptions | undefined;
500
- };
501
- import { BasicCrawler } from "./basic_crawler";
502
- import { ProxyConfiguration } from "../proxy_configuration";
503
- import { BrowserPool } from "browser-pool/dist/browser-pool";
504
- import { CrawlingContext } from "./basic_crawler";
505
- import { BrowserController } from "browser-pool";
506
- import { HandleFailedRequest } from "./basic_crawler";
507
- import { BrowserPoolOptions } from "browser-pool/dist/browser-pool";
508
- import { RequestList } from "../request_list";
509
- import { RequestQueue } from "../storages/request_queue";
510
- import { AutoscaledPoolOptions } from "../autoscaling/autoscaled_pool";
511
- import { SessionPoolOptions } from "../session_pool/session_pool";
512
- //# sourceMappingURL=browser_crawler.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G;;;;;;;;;;uCAUE;IAoFN;;;;;;;OAOG;IACH,+DAPW,sBAAsB,GAAG,eAAe,gCAExC,OAAO,QAsBjB;IAED;;;;;OAKG;IACH,6CALW,sBAAsB,GAAG,eAAe,iBAqBlD;IAED;;;;;;;OAOG;IACH,oDANW,sBAAsB,GAAG,eAAe,SACxC,KAAK,QAWf;IAED;;;;;;OAMG;IACH,8CANW,sBAAsB,GAAG,eAAe;YACjC,MAAM;qBAYvB;IAED;;;;;;;;OAQG;IACH,4CANW,sBAAsB,GAAG,eAAe,GACvC,QAAQ,IAAI,CAAC,CAiBxB;IAED;;;;;;OAMG;IACH,uCANW,MAAM,qCA6BhB;IAED;;;;;;;OAOG;IACH,kDANW,MAAM,qBACN,iBAAiB,QAiB3B;CAUJ;;uBAhkBa,iBAAiB;;qCAIpB,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,QAAQ,IAAI,CAAC;kDAIf,sBAAsB,GAAG,eAAe,KACtC,QAAQ,IAAI,CAAC;qCAIf,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,YAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAKT,yBAAyB"}