apify 2.3.1-beta.4 → 3.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +69 -128
- package/build/actor.d.ts +0 -113
- package/build/actor.d.ts.map +0 -1
- package/build/actor.js +0 -582
- package/build/actor.js.map +0 -1
- package/build/apify.d.ts +0 -752
- package/build/apify.d.ts.map +0 -1
- package/build/apify.js +0 -877
- package/build/apify.js.map +0 -1
- package/build/autoscaling/autoscaled_pool.d.ts +0 -384
- package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/build/autoscaling/autoscaled_pool.js +0 -557
- package/build/autoscaling/autoscaled_pool.js.map +0 -1
- package/build/autoscaling/snapshotter.d.ts +0 -278
- package/build/autoscaling/snapshotter.d.ts.map +0 -1
- package/build/autoscaling/snapshotter.js +0 -447
- package/build/autoscaling/snapshotter.js.map +0 -1
- package/build/autoscaling/system_status.d.ts +0 -224
- package/build/autoscaling/system_status.d.ts.map +0 -1
- package/build/autoscaling/system_status.js +0 -228
- package/build/autoscaling/system_status.js.map +0 -1
- package/build/browser_launchers/browser_launcher.d.ts +0 -154
- package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
- package/build/browser_launchers/browser_launcher.js +0 -160
- package/build/browser_launchers/browser_launcher.js.map +0 -1
- package/build/browser_launchers/browser_plugin.d.ts +0 -23
- package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
- package/build/browser_launchers/browser_plugin.js +0 -25
- package/build/browser_launchers/browser_plugin.js.map +0 -1
- package/build/browser_launchers/playwright_launcher.d.ts +0 -131
- package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
- package/build/browser_launchers/playwright_launcher.js +0 -150
- package/build/browser_launchers/playwright_launcher.js.map +0 -1
- package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
- package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
- package/build/browser_launchers/puppeteer_launcher.js +0 -197
- package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
- package/build/cache_container.d.ts +0 -31
- package/build/cache_container.d.ts.map +0 -1
- package/build/cache_container.js +0 -48
- package/build/cache_container.js.map +0 -1
- package/build/configuration.d.ts +0 -226
- package/build/configuration.d.ts.map +0 -1
- package/build/configuration.js +0 -325
- package/build/configuration.js.map +0 -1
- package/build/constants.d.ts +0 -37
- package/build/constants.d.ts.map +0 -1
- package/build/constants.js +0 -41
- package/build/constants.js.map +0 -1
- package/build/crawlers/basic_crawler.d.ts +0 -443
- package/build/crawlers/basic_crawler.d.ts.map +0 -1
- package/build/crawlers/basic_crawler.js +0 -664
- package/build/crawlers/basic_crawler.js.map +0 -1
- package/build/crawlers/browser_crawler.d.ts +0 -512
- package/build/crawlers/browser_crawler.d.ts.map +0 -1
- package/build/crawlers/browser_crawler.js +0 -540
- package/build/crawlers/browser_crawler.js.map +0 -1
- package/build/crawlers/cheerio_crawler.d.ts +0 -931
- package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
- package/build/crawlers/cheerio_crawler.js +0 -913
- package/build/crawlers/cheerio_crawler.js.map +0 -1
- package/build/crawlers/crawler_extension.d.ts +0 -10
- package/build/crawlers/crawler_extension.d.ts.map +0 -1
- package/build/crawlers/crawler_extension.js +0 -19
- package/build/crawlers/crawler_extension.js.map +0 -1
- package/build/crawlers/crawler_utils.d.ts +0 -34
- package/build/crawlers/crawler_utils.d.ts.map +0 -1
- package/build/crawlers/crawler_utils.js +0 -87
- package/build/crawlers/crawler_utils.js.map +0 -1
- package/build/crawlers/playwright_crawler.d.ts +0 -448
- package/build/crawlers/playwright_crawler.d.ts.map +0 -1
- package/build/crawlers/playwright_crawler.js +0 -299
- package/build/crawlers/playwright_crawler.js.map +0 -1
- package/build/crawlers/puppeteer_crawler.d.ts +0 -425
- package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
- package/build/crawlers/puppeteer_crawler.js +0 -299
- package/build/crawlers/puppeteer_crawler.js.map +0 -1
- package/build/crawlers/statistics.d.ts +0 -185
- package/build/crawlers/statistics.d.ts.map +0 -1
- package/build/crawlers/statistics.js +0 -331
- package/build/crawlers/statistics.js.map +0 -1
- package/build/enqueue_links/click_elements.d.ts +0 -179
- package/build/enqueue_links/click_elements.d.ts.map +0 -1
- package/build/enqueue_links/click_elements.js +0 -434
- package/build/enqueue_links/click_elements.js.map +0 -1
- package/build/enqueue_links/enqueue_links.d.ts +0 -117
- package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/build/enqueue_links/enqueue_links.js +0 -163
- package/build/enqueue_links/enqueue_links.js.map +0 -1
- package/build/enqueue_links/shared.d.ts +0 -42
- package/build/enqueue_links/shared.d.ts.map +0 -1
- package/build/enqueue_links/shared.js +0 -121
- package/build/enqueue_links/shared.js.map +0 -1
- package/build/errors.d.ts +0 -29
- package/build/errors.d.ts.map +0 -1
- package/build/errors.js +0 -38
- package/build/errors.js.map +0 -1
- package/build/events.d.ts +0 -11
- package/build/events.d.ts.map +0 -1
- package/build/events.js +0 -147
- package/build/events.js.map +0 -1
- package/build/index.d.ts +0 -4
- package/build/index.d.ts.map +0 -1
- package/build/index.js +0 -7
- package/build/index.js.map +0 -1
- package/build/main.d.ts +0 -179
- package/build/main.d.ts.map +0 -1
- package/build/main.js +0 -81
- package/build/main.js.map +0 -1
- package/build/playwright_utils.d.ts +0 -9
- package/build/playwright_utils.d.ts.map +0 -1
- package/build/playwright_utils.js +0 -90
- package/build/playwright_utils.js.map +0 -1
- package/build/proxy_configuration.d.ts +0 -411
- package/build/proxy_configuration.d.ts.map +0 -1
- package/build/proxy_configuration.js +0 -517
- package/build/proxy_configuration.js.map +0 -1
- package/build/pseudo_url.d.ts +0 -86
- package/build/pseudo_url.d.ts.map +0 -1
- package/build/pseudo_url.js +0 -153
- package/build/pseudo_url.js.map +0 -1
- package/build/puppeteer_request_interception.d.ts +0 -8
- package/build/puppeteer_request_interception.d.ts.map +0 -1
- package/build/puppeteer_request_interception.js +0 -235
- package/build/puppeteer_request_interception.js.map +0 -1
- package/build/puppeteer_utils.d.ts +0 -250
- package/build/puppeteer_utils.d.ts.map +0 -1
- package/build/puppeteer_utils.js +0 -551
- package/build/puppeteer_utils.js.map +0 -1
- package/build/request.d.ts +0 -180
- package/build/request.d.ts.map +0 -1
- package/build/request.js +0 -261
- package/build/request.js.map +0 -1
- package/build/request_list.d.ts +0 -581
- package/build/request_list.d.ts.map +0 -1
- package/build/request_list.js +0 -826
- package/build/request_list.js.map +0 -1
- package/build/serialization.d.ts +0 -5
- package/build/serialization.d.ts.map +0 -1
- package/build/serialization.js +0 -139
- package/build/serialization.js.map +0 -1
- package/build/session_pool/errors.d.ts +0 -11
- package/build/session_pool/errors.d.ts.map +0 -1
- package/build/session_pool/errors.js +0 -18
- package/build/session_pool/errors.js.map +0 -1
- package/build/session_pool/events.d.ts +0 -5
- package/build/session_pool/events.d.ts.map +0 -1
- package/build/session_pool/events.js +0 -6
- package/build/session_pool/events.js.map +0 -1
- package/build/session_pool/session.d.ts +0 -286
- package/build/session_pool/session.d.ts.map +0 -1
- package/build/session_pool/session.js +0 -355
- package/build/session_pool/session.js.map +0 -1
- package/build/session_pool/session_pool.d.ts +0 -280
- package/build/session_pool/session_pool.d.ts.map +0 -1
- package/build/session_pool/session_pool.js +0 -393
- package/build/session_pool/session_pool.js.map +0 -1
- package/build/session_pool/session_utils.d.ts +0 -4
- package/build/session_pool/session_utils.d.ts.map +0 -1
- package/build/session_pool/session_utils.js +0 -24
- package/build/session_pool/session_utils.js.map +0 -1
- package/build/stealth/hiding_tricks.d.ts +0 -22
- package/build/stealth/hiding_tricks.d.ts.map +0 -1
- package/build/stealth/hiding_tricks.js +0 -308
- package/build/stealth/hiding_tricks.js.map +0 -1
- package/build/stealth/stealth.d.ts +0 -56
- package/build/stealth/stealth.d.ts.map +0 -1
- package/build/stealth/stealth.js +0 -125
- package/build/stealth/stealth.js.map +0 -1
- package/build/storages/dataset.d.ts +0 -288
- package/build/storages/dataset.d.ts.map +0 -1
- package/build/storages/dataset.js +0 -480
- package/build/storages/dataset.js.map +0 -1
- package/build/storages/key_value_store.d.ts +0 -243
- package/build/storages/key_value_store.d.ts.map +0 -1
- package/build/storages/key_value_store.js +0 -462
- package/build/storages/key_value_store.js.map +0 -1
- package/build/storages/request_queue.d.ts +0 -318
- package/build/storages/request_queue.d.ts.map +0 -1
- package/build/storages/request_queue.js +0 -636
- package/build/storages/request_queue.js.map +0 -1
- package/build/storages/storage_manager.d.ts +0 -87
- package/build/storages/storage_manager.d.ts.map +0 -1
- package/build/storages/storage_manager.js +0 -150
- package/build/storages/storage_manager.js.map +0 -1
- package/build/tsconfig.tsbuildinfo +0 -1
- package/build/typedefs.d.ts +0 -146
- package/build/typedefs.d.ts.map +0 -1
- package/build/typedefs.js +0 -88
- package/build/typedefs.js.map +0 -1
- package/build/utils.d.ts +0 -175
- package/build/utils.d.ts.map +0 -1
- package/build/utils.js +0 -731
- package/build/utils.js.map +0 -1
- package/build/utils_log.d.ts +0 -41
- package/build/utils_log.d.ts.map +0 -1
- package/build/utils_log.js +0 -192
- package/build/utils_log.js.map +0 -1
- package/build/utils_request.d.ts +0 -77
- package/build/utils_request.d.ts.map +0 -1
- package/build/utils_request.js +0 -385
- package/build/utils_request.js.map +0 -1
- package/build/utils_social.d.ts +0 -210
- package/build/utils_social.d.ts.map +0 -1
- package/build/utils_social.js +0 -787
- package/build/utils_social.js.map +0 -1
- package/build/validators.d.ts +0 -23
- package/build/validators.d.ts.map +0 -1
- package/build/validators.js +0 -29
- package/build/validators.js.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio_crawler.js","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;AAAA,2CAA2C;AAC3C,gDAA4E;AAC5E,4CAAgE;AAChE,mEAA8B,CAAC,2CAA2C;AAC1E,6EAA6C;AAC7C,6CAAyC;AACzC,mEAAgE;AAChE,yEAA+B;AAC/B,yDAAoB;AACpB,6DAAwB;AACxB,+CAA4C;AAC5C,4CAAiE;AACjE,oCAAwD;AACxD,oDAAoD,CAAC,2CAA2C;AAChG,mDAA4D;AAC5D,mDAA+C,CAAC,2CAA2C;AAC3F,yFAAmD;AAcnD,8CAA2C;AAI3C,iFAAiF;AAEjF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,yCAAyC,GAAG;IAC9C,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH,MAAM,cAAe,SAAQ,4BAAY;IA0BrC;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QAExF,MAAM,EACF,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,yCAAyC,EACjE,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,gDAAgD;YAChD,qBAAqB,EAAE,kBAAkB;YACzC,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,wBAAwB,EAAE,kBAAkB,GAAG,qBAAqB,GAAG,6CAAiC;SAC3G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;SAC5H;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACtD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;QACrD,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC;YACvE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAAS;QACT,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,2BAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACvD,gDAAgD;QAChD,gBAAgB,CAAC,mBAAmB,GAAG,gBAAgB,CAAC,kBAAkB,CAAC;QAC3E,OAAO,gBAAgB,CAAC,kBAAkB,CAAC;QAE3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,sBAAsB;YACvE,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YAEjC,IAAI,CAAC,cAAc,EAAE,EAAE,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,wDAAwD,CAAC,CAAC;aAC5H;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,sBAAsB,GAAG,KAAK,YAAY,IAAI,CAC5H,CAAC;aACL;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,kCAAkC,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE5G,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACrB;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SAC/E;QAED,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;QAC9C,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QACjH,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;SAC7D;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;SAC5C;QAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEjC,MAAM,CAAC,GAAG,GAAG;YACT,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,OAAO,EAAE,KAAK;gBACd,yGAAyG;gBACzG,gFAAgF;gBAChF,2GAA2G;gBAC3G,eAAe,EAAE,IAAI;aACxB,CAAC;YACF,CAAC,CAAC,IAAI,CAAC;QAEX,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC;QACtB,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACpC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;SACJ,CAAC,CAAC;QACH,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,6GAA6G;gBAC7G,mFAAmF;gBACnF,sEAAsE;gBACtE,IAAI,GAAG,EAAE;oBACL,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;iBAC9D;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,IAAA,6BAAmB,EACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAC/C,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kFAAkF,CAAC,CAAC;YACxG,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;YACnD,IAAA,mBAAS,GAAE,CAAC;SACf;QAED,MAAM,uBAAuB,GAAG,EAAE,CAAC;QAEnC,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,mBAAmB,CAAC,eAAe,EAAE,uBAAuB,CAAC,CAAC;SACtE;QAED,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACxE,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC5F,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC;QAC5E,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE/E,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,CAAC,EACpF,IAAI,CAAC,oBAAoB,EACzB,2BAA2B,IAAI,CAAC,oBAAoB,GAAG,IAAI,WAAW,CACzE,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC7F,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iFAAiF,CAAC,CAAC;YACvG,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;YACjD,IAAA,mBAAS,GAAE,CAAC;SACf;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB;QACpE,MAAM,UAAU,GAAG,IAAA,2BAAW,EAAC,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE9G,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,uBAAuB,CAAC,OAAO,KAA/B,uBAAuB,CAAC,OAAO,GAAK,EAAE,EAAC;YACvC,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE;gBAC/D,uBAAuB,CAAC,OAAO,CAAC,MAAM;gBACtC,UAAU;aACb,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE;QAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,CAAC,CAAC;QAC1F,IAAI,kBAAkB,CAAC;QAEvB,IAAI;YACA,kBAAkB,GAAG,MAAM,IAAA,gCAAgB,EAAC,IAAI,CAAC,CAAC;SACrD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,YAAY,2BAAY,EAAE;gBAC3B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;aACvC;iBAAM;gBACH,MAAM,CAAC,CAAC;aACX;SACJ;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,uBAAuB;QAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACpE,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,CAAC;QAE7E,0CAA0C;QAC1C,IAAI,aAAa,EAAE;YACf,uBAAuB,CAAC,OAAO,KAA/B,uBAAuB,CAAC,OAAO,GAAK,EAAE,EAAC;YACvC,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC;SAC1D;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,cAAc;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,oCAA4B,EAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAU,IAAI,GAAG,EAAE;YACnB,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE;gBACrC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;aACjD;YAED,6EAA6E;YAC7E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;SACpF;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;SACxE;aAAM;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SAC1C;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB;QAClE,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE;YAC/C,YAAY,EAAE,OAAO;YACrB,GAAG,uBAAuB;YAC1B,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,uBAAuB,CAAC,OAAO,EAAE;YACnE,KAAK,EAAE;gBACH,GAAG,uBAAuB,CAAC,KAAK;gBAChC,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,2EAA2E;QAC3E,oEAAoE;QACpE,yEAAyE;QACzE,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YACrE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;SACL;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAEjF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ;QACvC,IAAI,IAAI,CAAC,qBAAqB,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACzC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE;YAClD,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC;SAC3C;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAChC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvE,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAe;gBACzB,QAAQ,EAAE,IAAI;aACjB,CAAC;SACL;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,QAAQ;QAC1B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3C,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ;iBACH,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,yBAAyB,CAAC,mBAAmB;QACzC,mBAAmB,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrC,IAAI;gBACA,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;aAC9F;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,sBAAsB,CAAC,OAAO,EAAE,UAAU;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAEjE,IAAI,SAAS,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,cAAc,CAAC,CAAC;SAC3E;IACL,CAAC;IAED;;;;;;OAMG;IACH,qBAAqB,CAAC,OAAO;QACzB,IAAI,OAAO;YAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,QAAQ;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAA,oCAA4B,EAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE;YACpB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;SAChI;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,UAAU,GAAG,GAAG,EAAE;YACxD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBAC7D,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;SAC3G;IACL,CAAC;;AA7hBD;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;QAC9E,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC5C,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC1C,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;GAAC;AAygBN,kBAAe,cAAc,CAAC;AAE9B;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;GAWG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;GAIG"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
|
|
3
|
-
* @ignore
|
|
4
|
-
*/
|
|
5
|
-
export default class CrawlerExtension {
|
|
6
|
-
name: string;
|
|
7
|
-
log: import("@apify/log/log").Log;
|
|
8
|
-
getCrawlerOptions(): void;
|
|
9
|
-
}
|
|
10
|
-
//# sourceMappingURL=crawler_extension.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_extension.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.js"],"names":[],"mappings":"AAEA;;;GAGG;AACH;IAEQ,aAAiC;IACjC,kCAAkD;IAGtD,0BAEC;CACJ"}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const tslib_1 = require("tslib");
|
|
4
|
-
const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
|
|
5
|
-
/**
|
|
6
|
-
* Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
|
|
7
|
-
* @ignore
|
|
8
|
-
*/
|
|
9
|
-
class CrawlerExtension {
|
|
10
|
-
constructor() {
|
|
11
|
-
this.name = this.constructor.name;
|
|
12
|
-
this.log = utils_log_1.default.child({ prefix: this.name });
|
|
13
|
-
}
|
|
14
|
-
getCrawlerOptions() {
|
|
15
|
-
throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
exports.default = CrawlerExtension;
|
|
19
|
-
//# sourceMappingURL=crawler_extension.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_extension.js","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.js"],"names":[],"mappings":";;;AAAA,0EAAsC;AAEtC;;;GAGG;AACH,MAAqB,gBAAgB;IACjC;QACI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,iBAAiB;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,kDAAkD,CAAC,CAAC;IACpF,CAAC;CACJ;AATD,mCASC"}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Handles timeout request
|
|
3
|
-
* @param {Session} session
|
|
4
|
-
* @param {string} errorMessage
|
|
5
|
-
* @private
|
|
6
|
-
*/
|
|
7
|
-
export function handleRequestTimeout(session: Session, errorMessage: string): void;
|
|
8
|
-
/**
|
|
9
|
-
* Handles blocked request
|
|
10
|
-
* @param {Session} session
|
|
11
|
-
* @param {number} statusCode
|
|
12
|
-
* @private
|
|
13
|
-
*/
|
|
14
|
-
export function throwOnBlockedRequest(session: Session, statusCode: number): void;
|
|
15
|
-
/**
|
|
16
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
17
|
-
* if we see two cookies with same keys but different casing.
|
|
18
|
-
*
|
|
19
|
-
* @param {string} url
|
|
20
|
-
* @param {string[]} sourceCookies
|
|
21
|
-
* @return {string}
|
|
22
|
-
* @private
|
|
23
|
-
*/
|
|
24
|
-
export function mergeCookies(url: string, sourceCookies: string[]): string;
|
|
25
|
-
/**
|
|
26
|
-
* @param {string} url
|
|
27
|
-
* @param {string} [cookieString1='']
|
|
28
|
-
* @param {string} [cookieString2='']
|
|
29
|
-
* @return {string}
|
|
30
|
-
* @private
|
|
31
|
-
*/
|
|
32
|
-
export function diffCookies(url: string, cookieString1?: string | undefined, cookieString2?: string | undefined): string;
|
|
33
|
-
import { Session } from "../session_pool/session";
|
|
34
|
-
//# sourceMappingURL=crawler_utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.js"],"names":[],"mappings":"AAOA;;;;;GAKG;AACH,8CAJW,OAAO,gBACP,MAAM,QAQhB;AAED;;;;;GAKG;AACH,+CAJW,OAAO,cACP,MAAM,QAShB;AAED;;;;;;;;GAQG;AACH,kCALW,MAAM,iBACN,MAAM,EAAE,GACP,MAAM,CAyBjB;AAED;;;;;;GAMG;AACH,iCANW,MAAM,2EAGL,MAAM,CAsBjB"}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.diffCookies = exports.mergeCookies = exports.throwOnBlockedRequest = exports.handleRequestTimeout = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
const tough_cookie_1 = require("tough-cookie");
|
|
6
|
-
const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
|
|
7
|
-
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
|
|
8
|
-
/**
|
|
9
|
-
* Handles timeout request
|
|
10
|
-
* @param {Session} session
|
|
11
|
-
* @param {string} errorMessage
|
|
12
|
-
* @private
|
|
13
|
-
*/
|
|
14
|
-
function handleRequestTimeout(session, errorMessage) {
|
|
15
|
-
if (session)
|
|
16
|
-
session.markBad();
|
|
17
|
-
const timeoutMillis = errorMessage.match(/(\d+)\s?ms/)[1]; // first capturing group
|
|
18
|
-
const timeoutSecs = Number(timeoutMillis) / 1000;
|
|
19
|
-
throw new Error(`Navigation timed out after ${timeoutSecs} seconds.`);
|
|
20
|
-
}
|
|
21
|
-
exports.handleRequestTimeout = handleRequestTimeout;
|
|
22
|
-
/**
|
|
23
|
-
* Handles blocked request
|
|
24
|
-
* @param {Session} session
|
|
25
|
-
* @param {number} statusCode
|
|
26
|
-
* @private
|
|
27
|
-
*/
|
|
28
|
-
function throwOnBlockedRequest(session, statusCode) {
|
|
29
|
-
const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
|
|
30
|
-
if (isBlocked) {
|
|
31
|
-
throw new Error(`Request blocked - received ${statusCode} status code.`);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
exports.throwOnBlockedRequest = throwOnBlockedRequest;
|
|
35
|
-
/**
|
|
36
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
37
|
-
* if we see two cookies with same keys but different casing.
|
|
38
|
-
*
|
|
39
|
-
* @param {string} url
|
|
40
|
-
* @param {string[]} sourceCookies
|
|
41
|
-
* @return {string}
|
|
42
|
-
* @private
|
|
43
|
-
*/
|
|
44
|
-
function mergeCookies(url, sourceCookies) {
|
|
45
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
46
|
-
// ignore empty cookies
|
|
47
|
-
for (const sourceCookieString of sourceCookies.filter((c) => c)) {
|
|
48
|
-
const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces
|
|
49
|
-
for (const cookieString of cookies) {
|
|
50
|
-
const cookie = tough_cookie_1.Cookie.parse(cookieString);
|
|
51
|
-
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
52
|
-
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
53
|
-
});
|
|
54
|
-
if (similarKeyCookie) {
|
|
55
|
-
utils_log_1.default.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
|
|
56
|
-
}
|
|
57
|
-
jar.setCookieSync(cookie, url);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
return jar.getCookieStringSync(url);
|
|
61
|
-
}
|
|
62
|
-
exports.mergeCookies = mergeCookies;
|
|
63
|
-
/**
|
|
64
|
-
* @param {string} url
|
|
65
|
-
* @param {string} [cookieString1='']
|
|
66
|
-
* @param {string} [cookieString2='']
|
|
67
|
-
* @return {string}
|
|
68
|
-
* @private
|
|
69
|
-
*/
|
|
70
|
-
function diffCookies(url, cookieString1 = '', cookieString2 = '') {
|
|
71
|
-
if (cookieString1 === cookieString2 || !cookieString2) {
|
|
72
|
-
return '';
|
|
73
|
-
}
|
|
74
|
-
if (!cookieString1) {
|
|
75
|
-
return cookieString2;
|
|
76
|
-
}
|
|
77
|
-
const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
78
|
-
const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
79
|
-
const added = cookies2.filter((newCookie) => {
|
|
80
|
-
return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
|
|
81
|
-
});
|
|
82
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
83
|
-
added.forEach((cookie) => jar.setCookieSync(cookie, url));
|
|
84
|
-
return jar.getCookieStringSync(url);
|
|
85
|
-
}
|
|
86
|
-
exports.diffCookies = diffCookies;
|
|
87
|
-
//# sourceMappingURL=crawler_utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.js"],"names":[],"mappings":";;;;AAAA,+CAAiD;AAIjD,0EAA+B;AAC/B,iFAAiF;AAEjF;;;;;GAKG;AACH,SAAgB,oBAAoB,CAAC,OAAO,EAAE,YAAY;IACtD,IAAI,OAAO;QAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACnF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AAC1E,CAAC;AALD,oDAKC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,OAAO,EAAE,UAAU;IACrD,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IAEjE,IAAI,SAAS,EAAE;QACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,eAAe,CAAC,CAAC;KAC5E;AACL,CAAC;AAND,sDAMC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,GAAG,EAAE,aAAa;IAC3C,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE;QAC7D,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB;QAE1F,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE;YAChC,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC1C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE;gBAClB,mBAAG,CAAC,UAAU,CAAC,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC;aAC1H;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC;KACJ;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAtBD,oCAsBC;AAED;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,GAAG,EAAE,aAAa,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE;IACnE,IAAI,aAAa,KAAK,aAAa,IAAI,CAAC,aAAa,EAAE;QACnD,OAAO,EAAE,CAAC;KACb;IAED,IAAI,CAAC,aAAa,EAAE;QAChB,OAAO,aAAa,CAAC;KACxB;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACpH,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAEpH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAC5B,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAE1D,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAnBD,kCAmBC"}
|
|
@@ -1,448 +0,0 @@
|
|
|
1
|
-
export default PlaywrightCrawler;
|
|
2
|
-
export type PlaywrightGotoOptions = {
|
|
3
|
-
/**
|
|
4
|
-
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout.
|
|
5
|
-
* The default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
|
|
6
|
-
* browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or page.setDefaultTimeout(timeout) methods.
|
|
7
|
-
*/
|
|
8
|
-
timeout?: number | undefined;
|
|
9
|
-
/**
|
|
10
|
-
* When to consider operation succeeded, defaults to `load`. Events can be either:
|
|
11
|
-
* - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
|
|
12
|
-
* - `'load'` - consider operation to be finished when the `load` event is fired.
|
|
13
|
-
* - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
|
|
14
|
-
*/
|
|
15
|
-
waitUntil?: "domcontentloaded" | "load" | "networkidle" | undefined;
|
|
16
|
-
/**
|
|
17
|
-
* Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
|
|
18
|
-
*/
|
|
19
|
-
referer?: string | undefined;
|
|
20
|
-
};
|
|
21
|
-
export type PlaywrightHook = (crawlingContext: {
|
|
22
|
-
page: Page;
|
|
23
|
-
crawler: PlaywrightCrawler;
|
|
24
|
-
} & BrowserCrawlingContext & CrawlingContext, gotoOptions: PlaywrightGotoOptions) => Promise<void>;
|
|
25
|
-
export type PlaywrightHandlePageFunctionParam = {
|
|
26
|
-
page: Page;
|
|
27
|
-
crawler: PlaywrightCrawler;
|
|
28
|
-
};
|
|
29
|
-
export type PlaywrightHandlePageFunction = (context: PlaywrightHandlePageFunctionParam & BrowserCrawlingContext & CrawlingContext) => Promise<void>;
|
|
30
|
-
export type PlaywrightCrawlerOptions = {
|
|
31
|
-
/**
|
|
32
|
-
* Function that is called to process each request.
|
|
33
|
-
* It is passed an object with the following fields:
|
|
34
|
-
*
|
|
35
|
-
* ```
|
|
36
|
-
* {
|
|
37
|
-
* request: Request,
|
|
38
|
-
* response: Response,
|
|
39
|
-
* page: Page,
|
|
40
|
-
* session: Session,
|
|
41
|
-
* browserController: BrowserController,
|
|
42
|
-
* proxyInfo: ProxyInfo,
|
|
43
|
-
* crawler: PlaywrightCrawler,
|
|
44
|
-
* }
|
|
45
|
-
* ```
|
|
46
|
-
*
|
|
47
|
-
* `request` is an instance of the {@link Request } object with details about the URL to open, HTTP method etc.
|
|
48
|
-
* `page` is an instance of the `Playwright`
|
|
49
|
-
* [`Page`](https://playwright.dev/docs/api/class-page)
|
|
50
|
-
* `browserController` is an instance of the
|
|
51
|
-
* [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
|
|
52
|
-
* `response` is an instance of the `Playwright`
|
|
53
|
-
* [`Response`](https://playwright.dev/docs/api/class-response),
|
|
54
|
-
* which is the main resource response as returned by `page.goto(request.url)`.
|
|
55
|
-
* The function must return a promise, which is then awaited by the crawler.
|
|
56
|
-
*
|
|
57
|
-
* If the function throws an exception, the crawler will try to re-crawl the
|
|
58
|
-
* request later, up to `option.maxRequestRetries` times.
|
|
59
|
-
* If all the retries fail, the crawler calls the function
|
|
60
|
-
* provided to the `handleFailedRequestFunction` parameter.
|
|
61
|
-
* To make this work, you should **always**
|
|
62
|
-
* let your function throw exceptions rather than catch them.
|
|
63
|
-
* The exceptions are logged to the request using the
|
|
64
|
-
* {@link RequestpushErrorMessage } function.
|
|
65
|
-
*/
|
|
66
|
-
handlePageFunction: PlaywrightHandlePageFunction;
|
|
67
|
-
/**
|
|
68
|
-
* Timeout in which page navigation needs to finish, in seconds.
|
|
69
|
-
*/
|
|
70
|
-
navigationTimeoutSecs?: number | undefined;
|
|
71
|
-
/**
|
|
72
|
-
* A function to handle requests that failed more than `option.maxRequestRetries` times.
|
|
73
|
-
*
|
|
74
|
-
* The function receives the following object as an argument:
|
|
75
|
-
* ```
|
|
76
|
-
* {
|
|
77
|
-
* request: Request,
|
|
78
|
-
* response: Response,
|
|
79
|
-
* page: Page,
|
|
80
|
-
* session: Session,
|
|
81
|
-
* browserController: BrowserController,
|
|
82
|
-
* proxyInfo: ProxyInfo,
|
|
83
|
-
* crawler: PlaywrightCrawler,
|
|
84
|
-
* }
|
|
85
|
-
* ```
|
|
86
|
-
* Where the {@link Request } instance corresponds to the failed request, and the `Error` instance
|
|
87
|
-
* represents the last error thrown during processing of the request.
|
|
88
|
-
*/
|
|
89
|
-
handleFailedRequestFunction?: HandleFailedRequest | undefined;
|
|
90
|
-
/**
|
|
91
|
-
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
92
|
-
* or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
|
|
93
|
-
* which are passed to the `page.goto()` function the crawler calls to navigate.
|
|
94
|
-
* Example:
|
|
95
|
-
* ```
|
|
96
|
-
* preNavigationHooks: [
|
|
97
|
-
* async (crawlingContext, gotoOptions) => {
|
|
98
|
-
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
99
|
-
* },
|
|
100
|
-
* ]
|
|
101
|
-
* ```
|
|
102
|
-
*/
|
|
103
|
-
preNavigationHooks?: PlaywrightHook[] | undefined;
|
|
104
|
-
/**
|
|
105
|
-
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
106
|
-
* The function accepts `crawlingContext` as the only parameter.
|
|
107
|
-
* Example:
|
|
108
|
-
* ```
|
|
109
|
-
* postNavigationHooks: [
|
|
110
|
-
* async (crawlingContext) => {
|
|
111
|
-
* const { page } = crawlingContext;
|
|
112
|
-
* if (hasCaptcha(page)) {
|
|
113
|
-
* await solveCaptcha (page);
|
|
114
|
-
* }
|
|
115
|
-
* },
|
|
116
|
-
* ]
|
|
117
|
-
* ```
|
|
118
|
-
*/
|
|
119
|
-
postNavigationHooks?: PlaywrightHook[] | undefined;
|
|
120
|
-
/**
|
|
121
|
-
* The same options as used by {@link ApifylaunchPlaywright }.
|
|
122
|
-
*/
|
|
123
|
-
launchContext?: PlaywrightLaunchContext | undefined;
|
|
124
|
-
/**
|
|
125
|
-
* Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
|
|
126
|
-
*/
|
|
127
|
-
handlePageTimeoutSecs?: number | undefined;
|
|
128
|
-
/**
|
|
129
|
-
* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
|
|
130
|
-
* You can tweak those to fine-tune browser management.
|
|
131
|
-
*/
|
|
132
|
-
browserPoolOptions?: BrowserPoolOptions<import("browser-pool").BrowserPlugin<import("browser-pool/dist/abstract-classes/browser-plugin").CommonLibrary, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonBrowser, unknown, import("browser-pool").CommonPage>> | undefined;
|
|
133
|
-
/**
|
|
134
|
-
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
135
|
-
*/
|
|
136
|
-
persistCookiesPerSession?: boolean | undefined;
|
|
137
|
-
/**
|
|
138
|
-
* If set, `PlaywrightCrawler` will be configured for all connections to use
|
|
139
|
-
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
140
|
-
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
141
|
-
*/
|
|
142
|
-
proxyConfiguration?: ProxyConfiguration | undefined;
|
|
143
|
-
/**
|
|
144
|
-
* Static list of URLs to be processed.
|
|
145
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
146
|
-
*/
|
|
147
|
-
requestList?: RequestList | undefined;
|
|
148
|
-
/**
|
|
149
|
-
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
150
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
151
|
-
*/
|
|
152
|
-
requestQueue?: RequestQueue | undefined;
|
|
153
|
-
/**
|
|
154
|
-
* Indicates how many times the request is retried if {@link PlaywrightCrawlerOptions.handlePageFunction } fails.
|
|
155
|
-
*/
|
|
156
|
-
maxRequestRetries?: number | undefined;
|
|
157
|
-
/**
|
|
158
|
-
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
|
|
159
|
-
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
|
|
160
|
-
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
|
|
161
|
-
*/
|
|
162
|
-
maxRequestsPerCrawl?: number | undefined;
|
|
163
|
-
/**
|
|
164
|
-
* Custom options passed to the underlying {@link AutoscaledPool } constructor.
|
|
165
|
-
* Note that the `runTaskFunction` and `isTaskReadyFunction` options
|
|
166
|
-
* are provided by `BasicCrawler` and cannot be overridden.
|
|
167
|
-
* However, you can provide a custom implementation of `isFinishedFunction`.
|
|
168
|
-
*/
|
|
169
|
-
autoscaledPoolOptions?: AutoscaledPoolOptions | undefined;
|
|
170
|
-
/**
|
|
171
|
-
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
|
|
172
|
-
*
|
|
173
|
-
* *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
|
|
174
|
-
* If you're not sure, just keep the default value and the concurrency will scale up automatically.
|
|
175
|
-
*/
|
|
176
|
-
minConcurrency?: number | undefined;
|
|
177
|
-
/**
|
|
178
|
-
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
|
|
179
|
-
*/
|
|
180
|
-
maxConcurrency?: number | undefined;
|
|
181
|
-
/**
|
|
182
|
-
* Playwright crawler will initialize the {@link SessionPool } with the corresponding `sessionPoolOptions`.
|
|
183
|
-
* The session instance will be than available in the `handleRequestFunction`.
|
|
184
|
-
*/
|
|
185
|
-
useSessionPool?: boolean | undefined;
|
|
186
|
-
/**
|
|
187
|
-
* The configuration options for {@link SessionPool } to use.
|
|
188
|
-
*/
|
|
189
|
-
sessionPoolOptions?: SessionPoolOptions | undefined;
|
|
190
|
-
};
|
|
191
|
-
/**
|
|
192
|
-
* @typedef PlaywrightGotoOptions
|
|
193
|
-
* @property {number} [timeout]
|
|
194
|
-
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout.
|
|
195
|
-
* The default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
|
|
196
|
-
* browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or page.setDefaultTimeout(timeout) methods.
|
|
197
|
-
* @property {"domcontentloaded"|"load"|"networkidle"} [waitUntil]
|
|
198
|
-
* When to consider operation succeeded, defaults to `load`. Events can be either:
|
|
199
|
-
* - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
|
|
200
|
-
* - `'load'` - consider operation to be finished when the `load` event is fired.
|
|
201
|
-
* - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
|
|
202
|
-
* @property {string} [referer]
|
|
203
|
-
* Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
|
|
204
|
-
*/
|
|
205
|
-
/**
|
|
206
|
-
* @callback PlaywrightHook
|
|
207
|
-
* @param {{ page: Page, crawler: PlaywrightCrawler } & BrowserCrawlingContext & CrawlingContext} crawlingContext
|
|
208
|
-
* @param {PlaywrightGotoOptions} gotoOptions
|
|
209
|
-
* @returns {Promise<void>}
|
|
210
|
-
*/
|
|
211
|
-
/**
|
|
212
|
-
* @typedef PlaywrightHandlePageFunctionParam
|
|
213
|
-
* @property {Page} page
|
|
214
|
-
* @property {PlaywrightCrawler} crawler
|
|
215
|
-
*/
|
|
216
|
-
/**
|
|
217
|
-
* @callback PlaywrightHandlePageFunction
|
|
218
|
-
* @param {PlaywrightHandlePageFunctionParam & BrowserCrawlingContext & CrawlingContext} context
|
|
219
|
-
* @returns {Promise<void>}
|
|
220
|
-
*/
|
|
221
|
-
/**
|
|
222
|
-
* @typedef PlaywrightCrawlerOptions
|
|
223
|
-
* @property {PlaywrightHandlePageFunction} handlePageFunction
|
|
224
|
-
* Function that is called to process each request.
|
|
225
|
-
* It is passed an object with the following fields:
|
|
226
|
-
*
|
|
227
|
-
* ```
|
|
228
|
-
* {
|
|
229
|
-
* request: Request,
|
|
230
|
-
* response: Response,
|
|
231
|
-
* page: Page,
|
|
232
|
-
* session: Session,
|
|
233
|
-
* browserController: BrowserController,
|
|
234
|
-
* proxyInfo: ProxyInfo,
|
|
235
|
-
* crawler: PlaywrightCrawler,
|
|
236
|
-
* }
|
|
237
|
-
* ```
|
|
238
|
-
*
|
|
239
|
-
* `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
|
|
240
|
-
* `page` is an instance of the `Playwright`
|
|
241
|
-
* [`Page`](https://playwright.dev/docs/api/class-page)
|
|
242
|
-
* `browserController` is an instance of the
|
|
243
|
-
* [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
|
|
244
|
-
* `response` is an instance of the `Playwright`
|
|
245
|
-
* [`Response`](https://playwright.dev/docs/api/class-response),
|
|
246
|
-
* which is the main resource response as returned by `page.goto(request.url)`.
|
|
247
|
-
* The function must return a promise, which is then awaited by the crawler.
|
|
248
|
-
*
|
|
249
|
-
* If the function throws an exception, the crawler will try to re-crawl the
|
|
250
|
-
* request later, up to `option.maxRequestRetries` times.
|
|
251
|
-
* If all the retries fail, the crawler calls the function
|
|
252
|
-
* provided to the `handleFailedRequestFunction` parameter.
|
|
253
|
-
* To make this work, you should **always**
|
|
254
|
-
* let your function throw exceptions rather than catch them.
|
|
255
|
-
* The exceptions are logged to the request using the
|
|
256
|
-
* {@link Request#pushErrorMessage} function.
|
|
257
|
-
* @property {number} [navigationTimeoutSecs=60]
|
|
258
|
-
* Timeout in which page navigation needs to finish, in seconds.
|
|
259
|
-
* @property {HandleFailedRequest} [handleFailedRequestFunction]
|
|
260
|
-
* A function to handle requests that failed more than `option.maxRequestRetries` times.
|
|
261
|
-
*
|
|
262
|
-
* The function receives the following object as an argument:
|
|
263
|
-
* ```
|
|
264
|
-
* {
|
|
265
|
-
* request: Request,
|
|
266
|
-
* response: Response,
|
|
267
|
-
* page: Page,
|
|
268
|
-
* session: Session,
|
|
269
|
-
* browserController: BrowserController,
|
|
270
|
-
* proxyInfo: ProxyInfo,
|
|
271
|
-
* crawler: PlaywrightCrawler,
|
|
272
|
-
* }
|
|
273
|
-
* ```
|
|
274
|
-
* Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
|
|
275
|
-
* represents the last error thrown during processing of the request.
|
|
276
|
-
* @property {Array<PlaywrightHook>} [preNavigationHooks]
|
|
277
|
-
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
278
|
-
* or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
|
|
279
|
-
* which are passed to the `page.goto()` function the crawler calls to navigate.
|
|
280
|
-
* Example:
|
|
281
|
-
* ```
|
|
282
|
-
* preNavigationHooks: [
|
|
283
|
-
* async (crawlingContext, gotoOptions) => {
|
|
284
|
-
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
285
|
-
* },
|
|
286
|
-
* ]
|
|
287
|
-
* ```
|
|
288
|
-
* @property {Array<PlaywrightHook>} [postNavigationHooks]
|
|
289
|
-
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
290
|
-
* The function accepts `crawlingContext` as the only parameter.
|
|
291
|
-
* Example:
|
|
292
|
-
* ```
|
|
293
|
-
* postNavigationHooks: [
|
|
294
|
-
* async (crawlingContext) => {
|
|
295
|
-
* const { page } = crawlingContext;
|
|
296
|
-
* if (hasCaptcha(page)) {
|
|
297
|
-
* await solveCaptcha (page);
|
|
298
|
-
* }
|
|
299
|
-
* },
|
|
300
|
-
* ]
|
|
301
|
-
* ```
|
|
302
|
-
* @property {PlaywrightLaunchContext} [launchContext]
|
|
303
|
-
* The same options as used by {@link Apify#launchPlaywright}.
|
|
304
|
-
* @property {number} [handlePageTimeoutSecs=60]
|
|
305
|
-
* Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
|
|
306
|
-
* @property {BrowserPoolOptions} [browserPoolOptions]
|
|
307
|
-
* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
|
|
308
|
-
* You can tweak those to fine-tune browser management.
|
|
309
|
-
* @property {boolean} [persistCookiesPerSession=true]
|
|
310
|
-
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
311
|
-
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
312
|
-
* If set, `PlaywrightCrawler` will be configured for all connections to use
|
|
313
|
-
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
314
|
-
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
315
|
-
* @property {RequestList} [requestList]
|
|
316
|
-
* Static list of URLs to be processed.
|
|
317
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
318
|
-
* @property {RequestQueue} [requestQueue]
|
|
319
|
-
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
320
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
321
|
-
* @property {number} [maxRequestRetries=3]
|
|
322
|
-
* Indicates how many times the request is retried if {@link PlaywrightCrawlerOptions.handlePageFunction} fails.
|
|
323
|
-
* @property {number} [maxRequestsPerCrawl]
|
|
324
|
-
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
|
|
325
|
-
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
|
|
326
|
-
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
|
|
327
|
-
* @property {AutoscaledPoolOptions} [autoscaledPoolOptions]
|
|
328
|
-
* Custom options passed to the underlying {@link AutoscaledPool} constructor.
|
|
329
|
-
* Note that the `runTaskFunction` and `isTaskReadyFunction` options
|
|
330
|
-
* are provided by `BasicCrawler` and cannot be overridden.
|
|
331
|
-
* However, you can provide a custom implementation of `isFinishedFunction`.
|
|
332
|
-
* @property {number} [minConcurrency=1]
|
|
333
|
-
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
|
|
334
|
-
*
|
|
335
|
-
* *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
|
|
336
|
-
* If you're not sure, just keep the default value and the concurrency will scale up automatically.
|
|
337
|
-
* @property {number} [maxConcurrency=1000]
|
|
338
|
-
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
|
|
339
|
-
* @property {boolean} [useSessionPool=true]
|
|
340
|
-
* Playwright crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
|
|
341
|
-
* The session instance will be than available in the `handleRequestFunction`.
|
|
342
|
-
* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.
|
|
343
|
-
*/
|
|
344
|
-
/**
|
|
345
|
-
* Provides a simple framework for parallel crawling of web pages
|
|
346
|
-
* using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
|
|
347
|
-
* The URLs to crawl are fed either from a static list of URLs
|
|
348
|
-
* or from a dynamic queue of URLs enabling recursive crawling of websites.
|
|
349
|
-
*
|
|
350
|
-
* Since `Playwright` uses headless browser to download web pages and extract data,
|
|
351
|
-
* it is useful for crawling of websites that require to execute JavaScript.
|
|
352
|
-
* If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
|
|
353
|
-
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
354
|
-
*
|
|
355
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
356
|
-
* {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
|
|
357
|
-
* or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
|
|
358
|
-
*
|
|
359
|
-
* If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
|
|
360
|
-
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
|
|
361
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
362
|
-
*
|
|
363
|
-
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
364
|
-
*
|
|
365
|
-
* `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
|
|
366
|
-
* and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.handlePageFunction} option.
|
|
367
|
-
*
|
|
368
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
369
|
-
* using the functionality provided by the {@link AutoscaledPool} class.
|
|
370
|
-
* All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
|
|
371
|
-
* parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
|
|
372
|
-
* {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
|
|
373
|
-
*
|
|
374
|
-
* Note that the pool of Playwright instances is internally managed by the {@link BrowserPool} class.
|
|
375
|
-
* Many constructor options such as {@link PlaywrightCrawlerOptions.maxOpenPagesPerInstance} or
|
|
376
|
-
*
|
|
377
|
-
* **Example usage:**
|
|
378
|
-
*
|
|
379
|
-
* ```javascript
|
|
380
|
-
* const crawler = new Apify.PlaywrightCrawler({
|
|
381
|
-
* requestList,
|
|
382
|
-
* handlePageFunction: async ({ page, request }) => {
|
|
383
|
-
* // This function is called to extract data from a single web page
|
|
384
|
-
* // 'page' is an instance of Playwright.Page with page.goto(request.url) already called
|
|
385
|
-
* // 'request' is an instance of Request class with information about the page to load
|
|
386
|
-
* await Apify.pushData({
|
|
387
|
-
* title: await page.title(),
|
|
388
|
-
* url: request.url,
|
|
389
|
-
* succeeded: true,
|
|
390
|
-
* })
|
|
391
|
-
* },
|
|
392
|
-
* handleFailedRequestFunction: async ({ request }) => {
|
|
393
|
-
* // This function is called when the crawling of a request failed too many times
|
|
394
|
-
* await Apify.pushData({
|
|
395
|
-
* url: request.url,
|
|
396
|
-
* succeeded: false,
|
|
397
|
-
* errors: request.errorMessages,
|
|
398
|
-
* })
|
|
399
|
-
* },
|
|
400
|
-
* });
|
|
401
|
-
*
|
|
402
|
-
* await crawler.run();
|
|
403
|
-
* ```
|
|
404
|
-
* @property {Statistics} stats
|
|
405
|
-
* Contains statistics about the current run.
|
|
406
|
-
* @property {RequestList} requestList
|
|
407
|
-
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
|
|
408
|
-
* Only available if used by the crawler.
|
|
409
|
-
* @property {RequestQueue} requestQueue
|
|
410
|
-
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
|
|
411
|
-
* Only available if used by the crawler.
|
|
412
|
-
* @property {SessionPool} sessionPool
|
|
413
|
-
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
|
|
414
|
-
* Only available if used by the crawler.
|
|
415
|
-
* @property {ProxyConfiguration} proxyConfiguration
|
|
416
|
-
* A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
|
|
417
|
-
* Only available if used by the crawler.
|
|
418
|
-
* @property {BrowserPool} browserPool
|
|
419
|
-
* A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
|
|
420
|
-
* For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
|
|
421
|
-
* @property {AutoscaledPool} autoscaledPool
|
|
422
|
-
* A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
|
|
423
|
-
* Note that this property is only initialized after calling the {@link CheerioCrawler#run} function.
|
|
424
|
-
* You can use it to change the concurrency settings on the fly,
|
|
425
|
-
* to pause the crawler by calling {@link AutoscaledPool#pause}
|
|
426
|
-
* or to abort it by calling {@link AutoscaledPool#abort}.
|
|
427
|
-
*/
|
|
428
|
-
declare class PlaywrightCrawler extends BrowserCrawler {
|
|
429
|
-
/**
|
|
430
|
-
* @param {PlaywrightCrawlerOptions} options
|
|
431
|
-
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
432
|
-
*/
|
|
433
|
-
constructor(options: PlaywrightCrawlerOptions);
|
|
434
|
-
launchContext: PlaywrightLaunchContext;
|
|
435
|
-
}
|
|
436
|
-
import { Page } from "playwright-core";
|
|
437
|
-
import { BrowserCrawlingContext } from "./browser_crawler";
|
|
438
|
-
import { CrawlingContext } from "./basic_crawler";
|
|
439
|
-
import { HandleFailedRequest } from "./basic_crawler";
|
|
440
|
-
import { PlaywrightLaunchContext } from "../browser_launchers/playwright_launcher";
|
|
441
|
-
import { BrowserPoolOptions } from "browser-pool/dist/browser-pool";
|
|
442
|
-
import { ProxyConfiguration } from "../proxy_configuration";
|
|
443
|
-
import { RequestList } from "../request_list";
|
|
444
|
-
import { RequestQueue } from "../storages/request_queue";
|
|
445
|
-
import { AutoscaledPoolOptions } from "../autoscaling/autoscaled_pool";
|
|
446
|
-
import { SessionPoolOptions } from "../session_pool/session_pool";
|
|
447
|
-
import BrowserCrawler from "./browser_crawler";
|
|
448
|
-
//# sourceMappingURL=playwright_crawler.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"playwright_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/playwright_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;+CAmCW;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,GAAG,sBAAsB,GAAG,eAAe,eACrF,qBAAqB,KACnB,QAAQ,IAAI,CAAC;;UAIZ,IAAI;aACJ,iBAAiB;;qDAIpB,iCAAiC,GAAG,sBAAsB,GAAG,eAAe,KAC1E,QAAQ,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAKZ,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAjC1C;;;;;;;;;;;;;GAaG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmFG;AACH;IAYI;;;OAGG;IACH,qBAHW,wBAAwB,EA4BlC;IADG,uCAAkC;CAiBzC"}
|