apify 2.3.1-beta.4 → 3.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +69 -128
- package/build/actor.d.ts +0 -113
- package/build/actor.d.ts.map +0 -1
- package/build/actor.js +0 -582
- package/build/actor.js.map +0 -1
- package/build/apify.d.ts +0 -752
- package/build/apify.d.ts.map +0 -1
- package/build/apify.js +0 -877
- package/build/apify.js.map +0 -1
- package/build/autoscaling/autoscaled_pool.d.ts +0 -384
- package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/build/autoscaling/autoscaled_pool.js +0 -557
- package/build/autoscaling/autoscaled_pool.js.map +0 -1
- package/build/autoscaling/snapshotter.d.ts +0 -278
- package/build/autoscaling/snapshotter.d.ts.map +0 -1
- package/build/autoscaling/snapshotter.js +0 -447
- package/build/autoscaling/snapshotter.js.map +0 -1
- package/build/autoscaling/system_status.d.ts +0 -224
- package/build/autoscaling/system_status.d.ts.map +0 -1
- package/build/autoscaling/system_status.js +0 -228
- package/build/autoscaling/system_status.js.map +0 -1
- package/build/browser_launchers/browser_launcher.d.ts +0 -154
- package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
- package/build/browser_launchers/browser_launcher.js +0 -160
- package/build/browser_launchers/browser_launcher.js.map +0 -1
- package/build/browser_launchers/browser_plugin.d.ts +0 -23
- package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
- package/build/browser_launchers/browser_plugin.js +0 -25
- package/build/browser_launchers/browser_plugin.js.map +0 -1
- package/build/browser_launchers/playwright_launcher.d.ts +0 -131
- package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
- package/build/browser_launchers/playwright_launcher.js +0 -150
- package/build/browser_launchers/playwright_launcher.js.map +0 -1
- package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
- package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
- package/build/browser_launchers/puppeteer_launcher.js +0 -197
- package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
- package/build/cache_container.d.ts +0 -31
- package/build/cache_container.d.ts.map +0 -1
- package/build/cache_container.js +0 -48
- package/build/cache_container.js.map +0 -1
- package/build/configuration.d.ts +0 -226
- package/build/configuration.d.ts.map +0 -1
- package/build/configuration.js +0 -325
- package/build/configuration.js.map +0 -1
- package/build/constants.d.ts +0 -37
- package/build/constants.d.ts.map +0 -1
- package/build/constants.js +0 -41
- package/build/constants.js.map +0 -1
- package/build/crawlers/basic_crawler.d.ts +0 -443
- package/build/crawlers/basic_crawler.d.ts.map +0 -1
- package/build/crawlers/basic_crawler.js +0 -664
- package/build/crawlers/basic_crawler.js.map +0 -1
- package/build/crawlers/browser_crawler.d.ts +0 -512
- package/build/crawlers/browser_crawler.d.ts.map +0 -1
- package/build/crawlers/browser_crawler.js +0 -540
- package/build/crawlers/browser_crawler.js.map +0 -1
- package/build/crawlers/cheerio_crawler.d.ts +0 -931
- package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
- package/build/crawlers/cheerio_crawler.js +0 -913
- package/build/crawlers/cheerio_crawler.js.map +0 -1
- package/build/crawlers/crawler_extension.d.ts +0 -10
- package/build/crawlers/crawler_extension.d.ts.map +0 -1
- package/build/crawlers/crawler_extension.js +0 -19
- package/build/crawlers/crawler_extension.js.map +0 -1
- package/build/crawlers/crawler_utils.d.ts +0 -34
- package/build/crawlers/crawler_utils.d.ts.map +0 -1
- package/build/crawlers/crawler_utils.js +0 -87
- package/build/crawlers/crawler_utils.js.map +0 -1
- package/build/crawlers/playwright_crawler.d.ts +0 -448
- package/build/crawlers/playwright_crawler.d.ts.map +0 -1
- package/build/crawlers/playwright_crawler.js +0 -299
- package/build/crawlers/playwright_crawler.js.map +0 -1
- package/build/crawlers/puppeteer_crawler.d.ts +0 -425
- package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
- package/build/crawlers/puppeteer_crawler.js +0 -299
- package/build/crawlers/puppeteer_crawler.js.map +0 -1
- package/build/crawlers/statistics.d.ts +0 -185
- package/build/crawlers/statistics.d.ts.map +0 -1
- package/build/crawlers/statistics.js +0 -331
- package/build/crawlers/statistics.js.map +0 -1
- package/build/enqueue_links/click_elements.d.ts +0 -179
- package/build/enqueue_links/click_elements.d.ts.map +0 -1
- package/build/enqueue_links/click_elements.js +0 -434
- package/build/enqueue_links/click_elements.js.map +0 -1
- package/build/enqueue_links/enqueue_links.d.ts +0 -117
- package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/build/enqueue_links/enqueue_links.js +0 -163
- package/build/enqueue_links/enqueue_links.js.map +0 -1
- package/build/enqueue_links/shared.d.ts +0 -42
- package/build/enqueue_links/shared.d.ts.map +0 -1
- package/build/enqueue_links/shared.js +0 -121
- package/build/enqueue_links/shared.js.map +0 -1
- package/build/errors.d.ts +0 -29
- package/build/errors.d.ts.map +0 -1
- package/build/errors.js +0 -38
- package/build/errors.js.map +0 -1
- package/build/events.d.ts +0 -11
- package/build/events.d.ts.map +0 -1
- package/build/events.js +0 -147
- package/build/events.js.map +0 -1
- package/build/index.d.ts +0 -4
- package/build/index.d.ts.map +0 -1
- package/build/index.js +0 -7
- package/build/index.js.map +0 -1
- package/build/main.d.ts +0 -179
- package/build/main.d.ts.map +0 -1
- package/build/main.js +0 -81
- package/build/main.js.map +0 -1
- package/build/playwright_utils.d.ts +0 -9
- package/build/playwright_utils.d.ts.map +0 -1
- package/build/playwright_utils.js +0 -90
- package/build/playwright_utils.js.map +0 -1
- package/build/proxy_configuration.d.ts +0 -411
- package/build/proxy_configuration.d.ts.map +0 -1
- package/build/proxy_configuration.js +0 -517
- package/build/proxy_configuration.js.map +0 -1
- package/build/pseudo_url.d.ts +0 -86
- package/build/pseudo_url.d.ts.map +0 -1
- package/build/pseudo_url.js +0 -153
- package/build/pseudo_url.js.map +0 -1
- package/build/puppeteer_request_interception.d.ts +0 -8
- package/build/puppeteer_request_interception.d.ts.map +0 -1
- package/build/puppeteer_request_interception.js +0 -235
- package/build/puppeteer_request_interception.js.map +0 -1
- package/build/puppeteer_utils.d.ts +0 -250
- package/build/puppeteer_utils.d.ts.map +0 -1
- package/build/puppeteer_utils.js +0 -551
- package/build/puppeteer_utils.js.map +0 -1
- package/build/request.d.ts +0 -180
- package/build/request.d.ts.map +0 -1
- package/build/request.js +0 -261
- package/build/request.js.map +0 -1
- package/build/request_list.d.ts +0 -581
- package/build/request_list.d.ts.map +0 -1
- package/build/request_list.js +0 -826
- package/build/request_list.js.map +0 -1
- package/build/serialization.d.ts +0 -5
- package/build/serialization.d.ts.map +0 -1
- package/build/serialization.js +0 -139
- package/build/serialization.js.map +0 -1
- package/build/session_pool/errors.d.ts +0 -11
- package/build/session_pool/errors.d.ts.map +0 -1
- package/build/session_pool/errors.js +0 -18
- package/build/session_pool/errors.js.map +0 -1
- package/build/session_pool/events.d.ts +0 -5
- package/build/session_pool/events.d.ts.map +0 -1
- package/build/session_pool/events.js +0 -6
- package/build/session_pool/events.js.map +0 -1
- package/build/session_pool/session.d.ts +0 -286
- package/build/session_pool/session.d.ts.map +0 -1
- package/build/session_pool/session.js +0 -355
- package/build/session_pool/session.js.map +0 -1
- package/build/session_pool/session_pool.d.ts +0 -280
- package/build/session_pool/session_pool.d.ts.map +0 -1
- package/build/session_pool/session_pool.js +0 -393
- package/build/session_pool/session_pool.js.map +0 -1
- package/build/session_pool/session_utils.d.ts +0 -4
- package/build/session_pool/session_utils.d.ts.map +0 -1
- package/build/session_pool/session_utils.js +0 -24
- package/build/session_pool/session_utils.js.map +0 -1
- package/build/stealth/hiding_tricks.d.ts +0 -22
- package/build/stealth/hiding_tricks.d.ts.map +0 -1
- package/build/stealth/hiding_tricks.js +0 -308
- package/build/stealth/hiding_tricks.js.map +0 -1
- package/build/stealth/stealth.d.ts +0 -56
- package/build/stealth/stealth.d.ts.map +0 -1
- package/build/stealth/stealth.js +0 -125
- package/build/stealth/stealth.js.map +0 -1
- package/build/storages/dataset.d.ts +0 -288
- package/build/storages/dataset.d.ts.map +0 -1
- package/build/storages/dataset.js +0 -480
- package/build/storages/dataset.js.map +0 -1
- package/build/storages/key_value_store.d.ts +0 -243
- package/build/storages/key_value_store.d.ts.map +0 -1
- package/build/storages/key_value_store.js +0 -462
- package/build/storages/key_value_store.js.map +0 -1
- package/build/storages/request_queue.d.ts +0 -318
- package/build/storages/request_queue.d.ts.map +0 -1
- package/build/storages/request_queue.js +0 -636
- package/build/storages/request_queue.js.map +0 -1
- package/build/storages/storage_manager.d.ts +0 -87
- package/build/storages/storage_manager.d.ts.map +0 -1
- package/build/storages/storage_manager.js +0 -150
- package/build/storages/storage_manager.js.map +0 -1
- package/build/tsconfig.tsbuildinfo +0 -1
- package/build/typedefs.d.ts +0 -146
- package/build/typedefs.d.ts.map +0 -1
- package/build/typedefs.js +0 -88
- package/build/typedefs.js.map +0 -1
- package/build/utils.d.ts +0 -175
- package/build/utils.d.ts.map +0 -1
- package/build/utils.js +0 -731
- package/build/utils.js.map +0 -1
- package/build/utils_log.d.ts +0 -41
- package/build/utils_log.d.ts.map +0 -1
- package/build/utils_log.js +0 -192
- package/build/utils_log.js.map +0 -1
- package/build/utils_request.d.ts +0 -77
- package/build/utils_request.d.ts.map +0 -1
- package/build/utils_request.js +0 -385
- package/build/utils_request.js.map +0 -1
- package/build/utils_social.d.ts +0 -210
- package/build/utils_social.d.ts.map +0 -1
- package/build/utils_social.js +0 -787
- package/build/utils_social.js.map +0 -1
- package/build/validators.d.ts +0 -23
- package/build/validators.d.ts.map +0 -1
- package/build/validators.js +0 -29
- package/build/validators.js.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.js","sourceRoot":"","sources":["../src/request_list.js"],"names":[],"mappings":";;;;AAAA,sDAAuC;AACvC,yEAA2B;AAC3B,2CAAmD;AACnD,qEAAgC,CAAC,2CAA2C;AAC5E,mEAA8B;AAC9B,yEAA8B;AAC9B,mCAAmD;AACnD,gEAAgE;AAChE,mDAAoE;AAKpE,iFAAiF;AAEpE,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAC7C,QAAA,wBAAwB,GAAG,uBAAuB,CAAC;AAEhE,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAEvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAa,WAAW;IACpB;;OAEG;IACH,YAAY,OAAO,GAAG,EAAE;QACpB,MAAM,EACF,OAAO,EACP,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,KAAK,EACL,kBAAkB,EAClB,iBAAiB,GAAG,KAAK,GAC5B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,EAAE;YAC/B,MAAM,IAAI,kBAAa,CAAC,kEAAkE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACjH;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC1B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YACrC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,YAAE,CAAC,MAAM;gBACpB,aAAa,EAAE,YAAE,CAAC,MAAM;gBACxB,UAAU,EAAE,YAAE,CAAC,MAAM;aACxB,CAAC;YACF,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,GAAG,GAAG,mBAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,oFAAoF;QACpF,qDAAqD;QACrD,6BAA6B;QAC7B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QAEnB,8GAA8G;QAC9G,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QAEnB,4FAA4F;QAC5F,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAE3B,mEAAmE;QACnE,uCAAuC;QACvC,2BAA2B;QAC3B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QAErB,gEAAgE;QAChE,gEAAgE;QAChE,gEAAgE;QAChE,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QAEpB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,eAAe,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACpF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,kBAAkB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAEhG,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAE1B,0HAA0H;QAC1H,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,gHAAgH;QAChH,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,oHAAoH;QACpH,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,qDAAqD;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,gEAAgE;QAChE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;SAC9E;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAE/E,mFAAmF;QACnF,8CAA8C;QAC9C,IAAI,iBAAiB,EAAE;YACnB,MAAM,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;SACvD;aAAM;YACH,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACxC;QAED,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzF,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,gBAAM,CAAC,EAAE,CAAC,gCAAoB,CAAC,aAAa,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SAC/E;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,qBAAqB,CAAC,iBAAiB;QACzC,gDAAgD;QAChD,uCAAuC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1B;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,MAAM,aAAa,GAAG,IAAA,iCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,aAAa,EAAE;YACvC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;SAC7B;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,uBAAuB;QACzB,wFAAwF;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,oEAAoE;YACpE,mDAAmD;YACnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,MAAM,CAAC,eAAe,EAAE;gBACxB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;gBACjE,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;aAC3D;iBAAM;gBACH,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC5B;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI;gBACA,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,wBAAwB,GAAG,mBAAmB,CAAC,MAAM,CAAC;gBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,EAAE,CAAC,EAAE,EAAE;oBAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;oBAC3C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;iBAC5B;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,yDAAyD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC3F;SACJ;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;SAChF;QACD,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI;YACA,MAAM,IAAA,0BAAQ,EAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yCAAyC,CAAC,CAAC;SACtE;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAc,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,MAAM,IAAA,0BAAQ,EAAC,IAAI,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAClG,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;;;;;;OAOG;IACH,aAAa,CAAC,KAAK;QACf,8DAA8D;QAC9D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,0BAA0B;QAC1B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;SAC5F;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;SACnG;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;eACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,aAAa,EAAE;YACrE,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;SACnH;QAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAChC,oBAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC3B,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACtH;YACD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,EAAE;gBAC1B,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACxC;QACL,CAAC,CAAC,CAAC;QAEH,cAAc;QACd,wEAAwE;QACxE,IAAI;QACJ,wBAAwB;QACxB,iDAAiD;QACjD,oBAAoB;QACpB,uCAAuC;QACvC,WAAW;QACX,2CAA2C;QAC3C,IAAI;QACJ,qGAAqG;QACrG,oFAAoF;QACpF,oCAAoC;QACpC,IAAI,oBAAoB,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0FAA0F,EAAE;gBACzG,oBAAoB;aACvB,CAAC,CAAC;YACH,oBAAC,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,SAAS,EAAE,EAAE;gBACvC,OAAO,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAEnC,gDAAgD;QAChD,IAAI,CAAC,SAAS,GAAG,oBAAC,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,8BAA8B;QAChC,IAAI,KAAK,CAAC;QACV,IAAI,iBAAiB,CAAC;QACtB,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC/D;aAAM,IAAI,IAAI,CAAC,eAAe,EAAE;YAC7B,KAAK,GAAG,IAAA,0BAAQ,EAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACvC,IAAI,KAAK;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;SAC7F;QACD,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,iBAAiB,GAAG,MAAM,IAAA,0BAAQ,EAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC5D,IAAI,iBAAiB;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;SAC/G;QACD,sDAAsD;QACtD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACnD,CAAC;IAED;;;;;OAKG;IACH,QAAQ;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAChD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,SAAS;gBACzC,CAAC,CAAC,IAAI;YACV,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,CAAC,IAAA,mBAAW,EAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAClF,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,CAAC,IAAA,mBAAW,EAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IACnF,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,SAAS,GAAG,IAAA,mBAAW,EAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,SAAS,EAAE;YACX,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC/B;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;YAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,OAAO,OAAO,CAAC;SAClB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAO;QAC5B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,OAAO;QACxB,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,mBAAmB,CAAC,MAAM,EAAE,eAAe;QAC7C,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QAE5D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY;YACZ,aAAa;YACb,cAAc,EAAE,YAAY,GAAG,aAAa;YAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,qBAAqB,CAAC,MAAM;QAC9B,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QACzD,MAAM,EAAE,kBAAkB,EAAE,GAAG,mBAAW,CAAC;QAE3C,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,kBAAkB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;SAC/H;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;SAClF;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;SACb;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;OASG;IACH,WAAW,CAAC,MAAM;QACd,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC;QAC3B,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1C;aAAM,IAAI,MAAM,YAAY,iBAAO,EAAE;YAClC,OAAO,GAAG,MAAM,CAAC;SACpB;aAAM,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE;YACpC,OAAO,GAAG,IAAI,iBAAO,CAAC,MAAM,CAAC,CAAC;SACjC;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QAExC,sDAAsD;QACtD,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,YAAY,EAAE;YACzC,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACnD;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtC,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,EAAE,4CAA4C;YAChG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC/B;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,SAAS,qFAAqF,CAAC,CAAC,CAAC,8BAA8B;SAC3K;IACL,CAAC;IAED;;;;;;;OAOG;IACH,qBAAqB,CAAC,SAAS;QAC3B,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;SAC7E;IACL,CAAC;IAED;;;;;;OAMG;IACH,gCAAgC,CAAC,SAAS;QACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;SACnF;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,GAAG,CAAC,CAAC;SAClF;IACL,CAAC;IAED;;;;;;OAMG;IACH,oBAAoB;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;;;OAIG;IACH,MAAM;QACF,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,YAAY;QACR,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,GAAG,oBAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpD,CAAC;CACJ;AA7jBD,kCA6jBC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AACI,MAAM,eAAe,GAAG,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACrE,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACzC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACtB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;QACvB,GAAG,OAAO;QACV,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,6BAAqB,EAAE,CAAC,CAAC,CAAC,SAAS;QAC9E,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,gCAAwB,EAAE,CAAC,CAAC,CAAC,SAAS;QACpF,OAAO;KACV,CAAC,CAAC;IACH,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE,CAAC;AACd,CAAC,CAAC;AAbW,QAAA,eAAe,mBAa1B;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH;;;GAGG"}
|
package/build/serialization.d.ts
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
export function serializeArray(data: Array<any>): Promise<Buffer>;
|
|
2
|
-
export function deserializeArray(compressedData: Buffer): Promise<Array<any>>;
|
|
3
|
-
export function createDeserialize(compressedData: any): Readable;
|
|
4
|
-
import { Readable } from "stream";
|
|
5
|
-
//# sourceMappingURL=serialization.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.js"],"names":[],"mappings":"AAkEO,qCAJI,UAAQ,GACN,QAAQ,MAAM,CAAC,CAY3B;AAcM,iDAJI,MAAM,GACJ,QAAQ,UAAQ,CAAC,CAa7B;AAaM,wDAHM,QAAQ,CAcpB"}
|
package/build/serialization.js
DELETED
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createDeserialize = exports.deserializeArray = exports.serializeArray = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
|
|
6
|
-
const stream_1 = (0, tslib_1.__importDefault)(require("stream")); // eslint-disable-line import/no-duplicates
|
|
7
|
-
const StreamArray_1 = (0, tslib_1.__importDefault)(require("stream-json/streamers/StreamArray"));
|
|
8
|
-
const util_1 = (0, tslib_1.__importDefault)(require("util"));
|
|
9
|
-
const zlib_1 = (0, tslib_1.__importDefault)(require("zlib"));
|
|
10
|
-
// eslint-enable-line import/no-duplicates
|
|
11
|
-
const pipeline = util_1.default.promisify(stream_1.default.pipeline);
|
|
12
|
-
/**
|
|
13
|
-
* Transforms an array of items to a JSON in a streaming
|
|
14
|
-
* fashion to save memory. It operates in batches to speed
|
|
15
|
-
* up the process.
|
|
16
|
-
* @ignore
|
|
17
|
-
* @internal
|
|
18
|
-
*/
|
|
19
|
-
class ArrayToJson extends stream_1.default.Readable {
|
|
20
|
-
constructor(data, options = {}) {
|
|
21
|
-
super({
|
|
22
|
-
...options,
|
|
23
|
-
autoDestroy: true,
|
|
24
|
-
emitClose: true,
|
|
25
|
-
});
|
|
26
|
-
const { batchSize = 10000 } = options;
|
|
27
|
-
this.offset = 0;
|
|
28
|
-
this.batchSize = batchSize;
|
|
29
|
-
this.data = data;
|
|
30
|
-
this.push('[');
|
|
31
|
-
}
|
|
32
|
-
_read() {
|
|
33
|
-
try {
|
|
34
|
-
const items = this.data.slice(this.offset, this.offset + this.batchSize);
|
|
35
|
-
if (items.length) {
|
|
36
|
-
const json = JSON.stringify(items);
|
|
37
|
-
// Strip brackets to flatten the batch.
|
|
38
|
-
const itemString = json.substring(1, json.length - 1);
|
|
39
|
-
if (this.offset > 0)
|
|
40
|
-
this.push(',', 'utf8');
|
|
41
|
-
this.push(itemString, 'utf8');
|
|
42
|
-
this.offset += this.batchSize;
|
|
43
|
-
}
|
|
44
|
-
else {
|
|
45
|
-
this.push(']');
|
|
46
|
-
this.push(null);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
catch (err) {
|
|
50
|
-
this.emit('error', err);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Uses Gzip compression to take an array of values, which can be anything
|
|
56
|
-
* from entries in a Dataset to Requests in a RequestList and compresses
|
|
57
|
-
* them to a Buffer in a memory-efficient way (streaming one by one). Ideally,
|
|
58
|
-
* the largest chunk of memory consumed will be the final compressed Buffer.
|
|
59
|
-
* This could be further improved by outputting a Stream, if and when
|
|
60
|
-
* apify-client supports streams.
|
|
61
|
-
*
|
|
62
|
-
* @param {Array<*>} data
|
|
63
|
-
* @returns {Promise<Buffer>}
|
|
64
|
-
* @ignore
|
|
65
|
-
*/
|
|
66
|
-
const serializeArray = async (data) => {
|
|
67
|
-
(0, ow_1.default)(data, ow_1.default.array);
|
|
68
|
-
const { chunks, collector } = createChunkCollector();
|
|
69
|
-
await pipeline(new ArrayToJson(data), zlib_1.default.createGzip(), collector);
|
|
70
|
-
return Buffer.concat(chunks);
|
|
71
|
-
};
|
|
72
|
-
exports.serializeArray = serializeArray;
|
|
73
|
-
/**
|
|
74
|
-
* Decompresses a Buffer previously created with compressData (technically,
|
|
75
|
-
* any JSON that is an Array) and collects it into an Array of values
|
|
76
|
-
* in a memory-efficient way (streaming the array items one by one instead
|
|
77
|
-
* of creating a fully decompressed buffer -> full JSON -> full Array all
|
|
78
|
-
* in memory at once. Could be further optimized to ingest a Stream if and
|
|
79
|
-
* when apify-client supports streams.
|
|
80
|
-
*
|
|
81
|
-
* @param {Buffer} compressedData
|
|
82
|
-
* @returns {Promise<Array<*>>}
|
|
83
|
-
* @ignore
|
|
84
|
-
*/
|
|
85
|
-
const deserializeArray = async (compressedData) => {
|
|
86
|
-
(0, ow_1.default)(compressedData, ow_1.default.buffer);
|
|
87
|
-
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
88
|
-
await pipeline(stream_1.default.Readable.from([compressedData]), zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
|
|
89
|
-
return chunks;
|
|
90
|
-
};
|
|
91
|
-
exports.deserializeArray = deserializeArray;
|
|
92
|
-
/**
|
|
93
|
-
* Creates a stream that decompresses a Buffer previously created with
|
|
94
|
-
* compressData (technically, any JSON that is an Array) and collects it
|
|
95
|
-
* into an Array of values in a memory-efficient way (streaming the array
|
|
96
|
-
* items one by one instead of creating a fully decompressed buffer
|
|
97
|
-
* -> full JSON -> full Array all in memory at once. Could be further
|
|
98
|
-
* optimized to ingest a Stream if and when apify-client supports streams.
|
|
99
|
-
* @param compressedData
|
|
100
|
-
* @returns {Readable}
|
|
101
|
-
* @ignore
|
|
102
|
-
*/
|
|
103
|
-
const createDeserialize = (compressedData) => {
|
|
104
|
-
(0, ow_1.default)(compressedData, ow_1.default.buffer);
|
|
105
|
-
const streamArray = StreamArray_1.default.withParser();
|
|
106
|
-
const destination = pluckValue(streamArray);
|
|
107
|
-
stream_1.default.pipeline(stream_1.default.Readable.from([compressedData]), zlib_1.default.createGunzip(), destination, (err) => destination.emit(err));
|
|
108
|
-
return destination;
|
|
109
|
-
};
|
|
110
|
-
exports.createDeserialize = createDeserialize;
|
|
111
|
-
/**
|
|
112
|
-
* @return {{chunks: Array<string|Buffer>, collector: module:stream.internal.Writable}}
|
|
113
|
-
*/
|
|
114
|
-
function createChunkCollector(options = {}) {
|
|
115
|
-
const { fromValuesStream = false } = options;
|
|
116
|
-
const chunks = [];
|
|
117
|
-
const collector = new stream_1.default.Writable({
|
|
118
|
-
decodeStrings: false,
|
|
119
|
-
objectMode: fromValuesStream,
|
|
120
|
-
write(chunk, nil, callback) {
|
|
121
|
-
chunks.push(fromValuesStream ? chunk.value : chunk);
|
|
122
|
-
callback();
|
|
123
|
-
},
|
|
124
|
-
writev(chunkObjects, callback) {
|
|
125
|
-
const buffers = chunkObjects.map(({ chunk }) => {
|
|
126
|
-
return fromValuesStream ? chunk.value : chunk;
|
|
127
|
-
});
|
|
128
|
-
chunkObjects.push(...buffers);
|
|
129
|
-
callback();
|
|
130
|
-
},
|
|
131
|
-
});
|
|
132
|
-
return { collector, chunks };
|
|
133
|
-
}
|
|
134
|
-
function pluckValue(streamArray) {
|
|
135
|
-
const realPush = streamArray.push.bind(streamArray);
|
|
136
|
-
streamArray.push = (obj) => realPush(obj && obj.value);
|
|
137
|
-
return streamArray;
|
|
138
|
-
}
|
|
139
|
-
//# sourceMappingURL=serialization.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AACpB,iEAA4B,CAAC,2CAA2C;AACxE,iGAA4D;AAC5D,6DAAwB;AACxB,6DAAwB;AAKxB,0CAA0C;AAE1C,MAAM,QAAQ,GAAG,cAAI,CAAC,SAAS,CAAC,gBAAM,CAAC,QAAQ,CAAC,CAAC;AAEjD;;;;;;GAMG;AACH,MAAM,WAAY,SAAQ,gBAAM,CAAC,QAAQ;IACrC,YAAY,IAAI,EAAE,OAAO,GAAG,EAAE;QAC1B,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,KAAK;QACD,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACI,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE;IACzC,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,cAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC,CAAC;AATW,QAAA,cAAc,kBASzB;AAEF;;;;;;;;;;;GAWG;AACI,MAAM,gBAAgB,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;IACrD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/E,MAAM,QAAQ,CACV,gBAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EACtC,cAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IACF,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAVW,QAAA,gBAAgB,oBAU3B;AAEF;;;;;;;;;;GAUG;AACI,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,EAAE;IAChD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAC5C,gBAAM,CAAC,QAAQ,CACX,gBAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EACtC,cAAI,CAAC,YAAY,EAAE,EACnB,WAAW,EACX,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IACF,OAAO,WAAW,CAAC;AACvB,CAAC,CAAC;AAXW,QAAA,iBAAiB,qBAW5B;AAEF;;GAEG;AACH,SAAS,oBAAoB,CAAC,OAAO,GAAG,EAAE;IACtC,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,SAAS,GAAG,IAAI,gBAAM,CAAC,QAAQ,CAAC;QAClC,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ;YACtB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IACH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAW;IAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/session_pool/errors.js"],"names":[],"mappings":"AAAA;;GAEG;AACH;IACI;;OAEG;IACH,gCAFW,MAAM,EAOhB;IAHG,2BAA4C;CAInD"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CookieParseError = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* @ignore
|
|
6
|
-
*/
|
|
7
|
-
class CookieParseError extends Error {
|
|
8
|
-
/**
|
|
9
|
-
* @param {string} cookieHeaderString
|
|
10
|
-
*/
|
|
11
|
-
constructor(cookieHeaderString) {
|
|
12
|
-
super(`Could not parse cookie header string: ${cookieHeaderString}`);
|
|
13
|
-
this.cookieHeaderString = cookieHeaderString;
|
|
14
|
-
Error.captureStackTrace(this, CookieParseError);
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
exports.CookieParseError = CookieParseError;
|
|
18
|
-
//# sourceMappingURL=errors.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/session_pool/errors.js"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAa,gBAAiB,SAAQ,KAAK;IACvC;;OAEG;IACH,YAAY,kBAAkB;QAC1B,KAAK,CAAC,yCAAyC,kBAAkB,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACpD,CAAC;CACJ;AAVD,4CAUC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/session_pool/events.js"],"names":[],"mappings":""}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/session_pool/events.js"],"names":[],"mappings":";;AAAA,kBAAe;IACX,eAAe,EAAE,gBAAgB;CACpC,CAAC"}
|
|
@@ -1,286 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
/**
|
|
3
|
-
* Persistable {@link Session} state.
|
|
4
|
-
* @typedef SessionState
|
|
5
|
-
* @property {string} id
|
|
6
|
-
* @property {CookieJar} cookieJar
|
|
7
|
-
* @property {object} userData
|
|
8
|
-
* @property {number} errorScore
|
|
9
|
-
* @property {number} maxErrorScore
|
|
10
|
-
* @property {number} errorScoreDecrement
|
|
11
|
-
* @property {number} usageCount
|
|
12
|
-
* @property {string} expiresAt
|
|
13
|
-
* @property {string} createdAt
|
|
14
|
-
*/
|
|
15
|
-
/**
|
|
16
|
-
* @typedef SessionOptions
|
|
17
|
-
* @property {string} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
|
|
18
|
-
* @property {number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
|
|
19
|
-
* @property {Object<string,*>} [userData] - Object where custom user data can be stored. For example custom headers.
|
|
20
|
-
* @property {number} [maxErrorScore=3] - Maximum number of marking session as blocked usage.
|
|
21
|
-
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
|
|
22
|
-
* It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
|
|
23
|
-
* Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
|
|
24
|
-
* @property {number} [errorScoreDecrement=0.5] - It is used for healing the session.
|
|
25
|
-
* For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore is decremented by this
|
|
26
|
-
* number.
|
|
27
|
-
* @property {Date} [createdAt] - Date of creation.
|
|
28
|
-
* @property {Date} [expiresAt] - Date of expiration.
|
|
29
|
-
* @property {number} [usageCount=0] - Indicates how many times the session has been used.
|
|
30
|
-
* @property {number} [errorCount=0] - Indicates how many times the session is marked bad.
|
|
31
|
-
* @property {number} [maxUsageCount=50] - Session should be used only a limited amount of times.
|
|
32
|
-
* This number indicates how many times the session is going to be used, before it is thrown away.
|
|
33
|
-
* @property {SessionPool} sessionPool - SessionPool instance. Session will emit the `sessionRetired` event on this instance.
|
|
34
|
-
*/
|
|
35
|
-
/**
|
|
36
|
-
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
37
|
-
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
38
|
-
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
39
|
-
*
|
|
40
|
-
*/
|
|
41
|
-
export class Session {
|
|
42
|
-
/**
|
|
43
|
-
* Session configuration.
|
|
44
|
-
*
|
|
45
|
-
* @param {SessionOptions} options
|
|
46
|
-
*/
|
|
47
|
-
constructor(options: SessionOptions);
|
|
48
|
-
log: any;
|
|
49
|
-
/**
|
|
50
|
-
* @type {CookieJar}
|
|
51
|
-
* @private
|
|
52
|
-
*/
|
|
53
|
-
private cookieJar;
|
|
54
|
-
id: string;
|
|
55
|
-
maxAgeSecs: number;
|
|
56
|
-
/** @type {Object<string,*>} */
|
|
57
|
-
userData: {
|
|
58
|
-
[x: string]: any;
|
|
59
|
-
};
|
|
60
|
-
maxErrorScore: number;
|
|
61
|
-
errorScoreDecrement: number;
|
|
62
|
-
expiresAt: Date;
|
|
63
|
-
createdAt: Date;
|
|
64
|
-
usageCount: number;
|
|
65
|
-
errorScore: number;
|
|
66
|
-
maxUsageCount: number;
|
|
67
|
-
sessionPool: SessionPool & object;
|
|
68
|
-
/**
|
|
69
|
-
* Transforms puppeteer cookie to tough-cookie.
|
|
70
|
-
* @param {PuppeteerCookie} puppeteerCookie Cookie from puppeteer `page.cookies method.
|
|
71
|
-
* @return {Cookie}
|
|
72
|
-
* @ignore
|
|
73
|
-
* @protected
|
|
74
|
-
* @internal
|
|
75
|
-
*/
|
|
76
|
-
protected _puppeteerCookieToTough(puppeteerCookie: PuppeteerCookie): Cookie;
|
|
77
|
-
/**
|
|
78
|
-
* indicates whether the session is blocked.
|
|
79
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
80
|
-
* @return {boolean}
|
|
81
|
-
*/
|
|
82
|
-
isBlocked(): boolean;
|
|
83
|
-
/**
|
|
84
|
-
* Indicates whether the session is expired.
|
|
85
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
86
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
87
|
-
* @return {boolean}
|
|
88
|
-
*/
|
|
89
|
-
isExpired(): boolean;
|
|
90
|
-
/**
|
|
91
|
-
* Indicates whether the session is used maximum number of times.
|
|
92
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
93
|
-
* @return {boolean}
|
|
94
|
-
*/
|
|
95
|
-
isMaxUsageCountReached(): boolean;
|
|
96
|
-
/**
|
|
97
|
-
* Indicates whether the session can be used for next requests.
|
|
98
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
|
|
99
|
-
* @return {boolean}
|
|
100
|
-
*/
|
|
101
|
-
isUsable(): boolean;
|
|
102
|
-
/**
|
|
103
|
-
* This method should be called after a successful session usage.
|
|
104
|
-
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
105
|
-
*/
|
|
106
|
-
markGood(): void;
|
|
107
|
-
/**
|
|
108
|
-
* Gets session state for persistence in KeyValueStore.
|
|
109
|
-
* @return {SessionState} represents session internal state.
|
|
110
|
-
*/
|
|
111
|
-
getState(): SessionState;
|
|
112
|
-
/**
|
|
113
|
-
* Marks session as blocked and emits event on the `SessionPool`
|
|
114
|
-
* This method should be used if the session usage was unsuccessful
|
|
115
|
-
* and you are sure that it is because of the session configuration and not any external matters.
|
|
116
|
-
* For example when server returns 403 status code.
|
|
117
|
-
* If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
|
|
118
|
-
*/
|
|
119
|
-
retire(): void;
|
|
120
|
-
/**
|
|
121
|
-
* Increases usage and error count.
|
|
122
|
-
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
123
|
-
*/
|
|
124
|
-
markBad(): void;
|
|
125
|
-
/**
|
|
126
|
-
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
127
|
-
* that the target website is blocking us. This function helps to do this conveniently
|
|
128
|
-
* by retiring the session when such code is received. Optionally the default status
|
|
129
|
-
* codes can be extended in the second parameter.
|
|
130
|
-
* @param statusCode {number} - HTTP status code
|
|
131
|
-
* @param [blockedStatusCodes] {number[]} - Custom HTTP status codes that means blocking on particular website.
|
|
132
|
-
* @return {boolean} whether the session was retired.
|
|
133
|
-
*/
|
|
134
|
-
retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes?: number[] | undefined): boolean;
|
|
135
|
-
/**
|
|
136
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
137
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
138
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
139
|
-
*
|
|
140
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
141
|
-
|
|
142
|
-
* @param {(PuppeteerResponse|IncomingMessage)} response
|
|
143
|
-
*/
|
|
144
|
-
setCookiesFromResponse(response: (PuppeteerResponse | IncomingMessage)): void;
|
|
145
|
-
/**
|
|
146
|
-
* Saves an array with cookie objects to be used with the session.
|
|
147
|
-
* The objects should be in the format that
|
|
148
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
149
|
-
* but you can also use this function to set cookies manually:
|
|
150
|
-
*
|
|
151
|
-
* ```
|
|
152
|
-
* [
|
|
153
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
154
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
155
|
-
* ]
|
|
156
|
-
* ```
|
|
157
|
-
*
|
|
158
|
-
* @param {PuppeteerCookie[]} cookies
|
|
159
|
-
* @param {string} url
|
|
160
|
-
*/
|
|
161
|
-
setPuppeteerCookies(cookies: PuppeteerCookie[], url: string): void;
|
|
162
|
-
/**
|
|
163
|
-
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
164
|
-
* @param {string} url website url. Only cookies stored for this url will be returned
|
|
165
|
-
* @return {PuppeteerCookie[]}
|
|
166
|
-
*/
|
|
167
|
-
getPuppeteerCookies(url: string): PuppeteerCookie[];
|
|
168
|
-
/**
|
|
169
|
-
* Returns cookies saved with the session in the typical
|
|
170
|
-
* key1=value1; key2=value2 format, ready to be used in
|
|
171
|
-
* a cookie header or elsewhere.
|
|
172
|
-
* @param {string} url
|
|
173
|
-
* @return {string} represents `Cookie` header.
|
|
174
|
-
*/
|
|
175
|
-
getCookieString(url: string): string;
|
|
176
|
-
/**
|
|
177
|
-
* Transforms tough-cookie to puppeteer cookie .
|
|
178
|
-
* @param {Cookie} toughCookie - Cookie from CookieJar
|
|
179
|
-
* @return {PuppeteerCookie} - Cookie from Puppeteer
|
|
180
|
-
* @ignore
|
|
181
|
-
* @protected
|
|
182
|
-
* @internal
|
|
183
|
-
*/
|
|
184
|
-
protected _toughCookieToPuppeteer(toughCookie: Cookie): PuppeteerCookie;
|
|
185
|
-
/**
|
|
186
|
-
* Sets cookies.
|
|
187
|
-
* @param {Cookie[]} cookies
|
|
188
|
-
* @param {string} url
|
|
189
|
-
* @ignore
|
|
190
|
-
* @protected
|
|
191
|
-
* @internal
|
|
192
|
-
*/
|
|
193
|
-
protected _setCookies(cookies: Cookie[], url: string): void;
|
|
194
|
-
/**
|
|
195
|
-
* Calculate cookie expiration date
|
|
196
|
-
* @param {number} maxAgeSecs
|
|
197
|
-
* @return {Date} - calculated date by session max age seconds.
|
|
198
|
-
* @ignore
|
|
199
|
-
* @protected
|
|
200
|
-
* @internal
|
|
201
|
-
*/
|
|
202
|
-
protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
|
|
203
|
-
/**
|
|
204
|
-
* Checks if session is not usable. if it is not retires the session.
|
|
205
|
-
* @ignore
|
|
206
|
-
* @protected
|
|
207
|
-
* @internal
|
|
208
|
-
*/
|
|
209
|
-
protected _maybeSelfRetire(): void;
|
|
210
|
-
}
|
|
211
|
-
export type PuppeteerCookie = object;
|
|
212
|
-
/**
|
|
213
|
-
* Persistable {@link Session } state.
|
|
214
|
-
*/
|
|
215
|
-
export type SessionState = {
|
|
216
|
-
id: string;
|
|
217
|
-
cookieJar: CookieJar;
|
|
218
|
-
userData: object;
|
|
219
|
-
errorScore: number;
|
|
220
|
-
maxErrorScore: number;
|
|
221
|
-
errorScoreDecrement: number;
|
|
222
|
-
usageCount: number;
|
|
223
|
-
expiresAt: string;
|
|
224
|
-
createdAt: string;
|
|
225
|
-
};
|
|
226
|
-
export type SessionOptions = {
|
|
227
|
-
/**
|
|
228
|
-
* - Id of session used for generating fingerprints. It is used as proxy session name.
|
|
229
|
-
*/
|
|
230
|
-
id?: string | undefined;
|
|
231
|
-
/**
|
|
232
|
-
* - Number of seconds after which the session is considered as expired.
|
|
233
|
-
*/
|
|
234
|
-
maxAgeSecs?: number | undefined;
|
|
235
|
-
/**
|
|
236
|
-
* - Object where custom user data can be stored. For example custom headers.
|
|
237
|
-
*/
|
|
238
|
-
userData?: {
|
|
239
|
-
[x: string]: any;
|
|
240
|
-
} | undefined;
|
|
241
|
-
/**
|
|
242
|
-
* - Maximum number of marking session as blocked usage.
|
|
243
|
-
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
|
|
244
|
-
* It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
|
|
245
|
-
* Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
|
|
246
|
-
*/
|
|
247
|
-
maxErrorScore?: number | undefined;
|
|
248
|
-
/**
|
|
249
|
-
* - It is used for healing the session.
|
|
250
|
-
* For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore is decremented by this
|
|
251
|
-
* number.
|
|
252
|
-
*/
|
|
253
|
-
errorScoreDecrement?: number | undefined;
|
|
254
|
-
/**
|
|
255
|
-
* - Date of creation.
|
|
256
|
-
*/
|
|
257
|
-
createdAt?: Date | undefined;
|
|
258
|
-
/**
|
|
259
|
-
* - Date of expiration.
|
|
260
|
-
*/
|
|
261
|
-
expiresAt?: Date | undefined;
|
|
262
|
-
/**
|
|
263
|
-
* - Indicates how many times the session has been used.
|
|
264
|
-
*/
|
|
265
|
-
usageCount?: number | undefined;
|
|
266
|
-
/**
|
|
267
|
-
* - Indicates how many times the session is marked bad.
|
|
268
|
-
*/
|
|
269
|
-
errorCount?: number | undefined;
|
|
270
|
-
/**
|
|
271
|
-
* - Session should be used only a limited amount of times.
|
|
272
|
-
* This number indicates how many times the session is going to be used, before it is thrown away.
|
|
273
|
-
*/
|
|
274
|
-
maxUsageCount?: number | undefined;
|
|
275
|
-
/**
|
|
276
|
-
* - SessionPool instance. Session will emit the `sessionRetired` event on this instance.
|
|
277
|
-
*/
|
|
278
|
-
sessionPool: SessionPool;
|
|
279
|
-
};
|
|
280
|
-
import { SessionPool } from "./session_pool";
|
|
281
|
-
import { Cookie } from "tough-cookie";
|
|
282
|
-
// @ts-ignore optional peer dependency
|
|
283
|
-
import { HTTPResponse as PuppeteerResponse } from "puppeteer";
|
|
284
|
-
import { IncomingMessage } from "http";
|
|
285
|
-
import { CookieJar } from "tough-cookie";
|
|
286
|
-
//# sourceMappingURL=session.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.js"],"names":[],"mappings":";AAsBA;;;;;;;;;;;;GAYG;AAEH;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;GAKG;AACH;IACI;;;;OAIG;IACH,qBAFW,cAAc,EA2DxB;IAvBG,SAA2C;IAE3C;;;OAGG;IACH,kBAAgG;IAChG,WAAY;IACZ,mBAA4B;IAC5B,+BAA+B;IAC/B;YADkB,MAAM;MACA;IACxB,sBAAkC;IAClC,4BAA8C;IAG9C,gBAA0B;IAC1B,gBAA0B;IAC1B,mBAA4B;IAC5B,mBAA4B;IAC5B,sBAAkC;IAClC,kCAA8B;IAsLlC;;;;;;;OAOG;IACH,mDANW,eAAe,GACd,MAAM,CAoBjB;IAxMD;;;;OAIG;IACH,aAFY,OAAO,CAIlB;IAED;;;;;OAKG;IACH,aAFY,OAAO,CAIlB;IAED;;;;OAIG;IACH,0BAFY,OAAO,CAIlB;IAED;;;;OAIG;IACH,YAFY,OAAO,CAIlB;IAED;;;OAGG;IACH,iBAQC;IAED;;;OAGG;IACH,YAFY,YAAY,CAevB;IAED;;;;;;OAMG;IACH,eAOC;IAED;;;OAGG;IACH,gBAKC;IAED;;;;;;;;OAQG;IACH,uCAJsB,MAAM,8CAEhB,OAAO,CAQlB;IAED;;;;;;;;OAQG;IACH,iCAFW,CAAC,iBAAiB,GAAC,eAAe,CAAC,QAW7C;IAED;;;;;;;;;;;;;;;OAeG;IACH,6BAHW,eAAe,EAAE,OACjB,MAAM,QAKhB;IAED;;;;OAIG;IACH,yBAHW,MAAM,GACL,eAAe,EAAE,CAM5B;IAED;;;;;;OAMG;IACH,qBAHW,MAAM,GACL,MAAM,CAIjB;IA2BD;;;;;;;OAOG;IACH,+CANW,MAAM,GACL,eAAe,CAgB1B;IAED;;;;;;;OAOG;IACH,+BANW,MAAM,EAAE,OACR,MAAM,QAkBhB;IAED;;;;;;;OAOG;IACH,sDANW,MAAM,GACL,IAAI,CAOf;IAED;;;;;OAKG;IACH,mCAIC;CACJ;8BA7XY,MAAM;;;;;QASL,MAAM;eACN,SAAS;cACT,MAAM;gBACN,MAAM;mBACN,MAAM;yBACN,MAAM;gBACN,MAAM;eACN,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAqBN,WAAW"}
|