@crawlee/core 3.0.0-alpha.1 → 3.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +286 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -0
- package/autoscaling/autoscaled_pool.js +624 -0
- package/autoscaling/autoscaled_pool.js.map +1 -0
- package/autoscaling/index.d.ts +4 -0
- package/autoscaling/index.d.ts.map +1 -0
- package/autoscaling/index.js +7 -0
- package/autoscaling/index.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +193 -0
- package/autoscaling/snapshotter.d.ts.map +1 -0
- package/autoscaling/snapshotter.js +377 -0
- package/autoscaling/snapshotter.js.map +1 -0
- package/autoscaling/system_status.d.ts +181 -0
- package/autoscaling/system_status.d.ts.map +1 -0
- package/autoscaling/system_status.js +194 -0
- package/autoscaling/system_status.js.map +1 -0
- package/cache_container.d.ts +18 -0
- package/cache_container.d.ts.map +1 -0
- package/cache_container.js +42 -0
- package/cache_container.js.map +1 -0
- package/configuration.d.ts +150 -0
- package/configuration.d.ts.map +1 -0
- package/configuration.js +284 -0
- package/configuration.js.map +1 -0
- package/constants.d.ts +3 -0
- package/constants.d.ts.map +1 -0
- package/constants.js +6 -0
- package/constants.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +23 -0
- package/crawlers/crawler_commons.d.ts.map +1 -0
- package/crawlers/crawler_commons.js +3 -0
- package/crawlers/crawler_commons.js.map +1 -0
- package/crawlers/crawler_extension.d.ts +11 -0
- package/crawlers/crawler_extension.d.ts.map +1 -0
- package/crawlers/crawler_extension.js +30 -0
- package/crawlers/crawler_extension.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +25 -0
- package/crawlers/crawler_utils.d.ts.map +1 -0
- package/crawlers/crawler_utils.js +72 -0
- package/crawlers/crawler_utils.js.map +1 -0
- package/crawlers/index.d.ts +5 -0
- package/crawlers/index.d.ts.map +1 -0
- package/crawlers/index.js +8 -0
- package/crawlers/index.js.map +1 -0
- package/crawlers/statistics.d.ts +147 -0
- package/crawlers/statistics.d.ts.map +1 -0
- package/crawlers/statistics.js +360 -0
- package/crawlers/statistics.js.map +1 -0
- package/enqueue_links/enqueue_links.d.ts +138 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -0
- package/enqueue_links/enqueue_links.js +123 -0
- package/enqueue_links/enqueue_links.js.map +1 -0
- package/enqueue_links/index.d.ts +3 -0
- package/enqueue_links/index.d.ts.map +1 -0
- package/enqueue_links/index.js +6 -0
- package/enqueue_links/index.js.map +1 -0
- package/enqueue_links/shared.d.ts +60 -0
- package/enqueue_links/shared.d.ts.map +1 -0
- package/enqueue_links/shared.js +148 -0
- package/enqueue_links/shared.js.map +1 -0
- package/errors.d.ts +8 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +12 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +41 -0
- package/events/event_manager.d.ts.map +1 -0
- package/events/event_manager.js +103 -0
- package/events/event_manager.js.map +1 -0
- package/events/index.d.ts +3 -0
- package/events/index.d.ts.map +1 -0
- package/events/index.js +6 -0
- package/events/index.js.map +1 -0
- package/events/local_event_manager.d.ts +32 -0
- package/events/local_event_manager.d.ts.map +1 -0
- package/events/local_event_manager.js +114 -0
- package/events/local_event_manager.js.map +1 -0
- package/index.d.ts +14 -0
- package/index.d.ts.map +1 -0
- package/index.js +17 -0
- package/index.js.map +1 -0
- package/index.mjs +63 -0
- package/log.d.ts +3 -0
- package/log.d.ts.map +1 -0
- package/log.js +12 -0
- package/log.js.map +1 -0
- package/package.json +4 -7
- package/proxy_configuration.d.ts +181 -0
- package/proxy_configuration.d.ts.map +1 -0
- package/proxy_configuration.js +209 -0
- package/proxy_configuration.js.map +1 -0
- package/request.d.ts +174 -0
- package/request.d.ts.map +1 -0
- package/request.js +270 -0
- package/request.js.map +1 -0
- package/serialization.d.ts +33 -0
- package/serialization.d.ts.map +1 -0
- package/serialization.js +146 -0
- package/serialization.js.map +1 -0
- package/session_pool/errors.d.ts +8 -0
- package/session_pool/errors.d.ts.map +1 -0
- package/session_pool/errors.js +20 -0
- package/session_pool/errors.js.map +1 -0
- package/session_pool/events.d.ts +3 -0
- package/session_pool/events.d.ts.map +1 -0
- package/session_pool/events.js +6 -0
- package/session_pool/events.js.map +1 -0
- package/session_pool/index.d.ts +6 -0
- package/session_pool/index.d.ts.map +1 -0
- package/session_pool/index.js +9 -0
- package/session_pool/index.js.map +1 -0
- package/session_pool/session.d.ts +212 -0
- package/session_pool/session.d.ts.map +1 -0
- package/session_pool/session.js +360 -0
- package/session_pool/session.js.map +1 -0
- package/session_pool/session_pool.d.ts +222 -0
- package/session_pool/session_pool.d.ts.map +1 -0
- package/session_pool/session_pool.js +389 -0
- package/session_pool/session_pool.js.map +1 -0
- package/session_pool/session_utils.d.ts +9 -0
- package/session_pool/session_utils.d.ts.map +1 -0
- package/session_pool/session_utils.js +19 -0
- package/session_pool/session_utils.js.map +1 -0
- package/storages/dataset.d.ts +358 -0
- package/storages/dataset.d.ts.map +1 -0
- package/storages/dataset.js +415 -0
- package/storages/dataset.js.map +1 -0
- package/storages/index.d.ts +7 -0
- package/storages/index.d.ts.map +1 -0
- package/storages/index.js +11 -0
- package/storages/index.js.map +1 -0
- package/storages/key_value_store.d.ts +242 -0
- package/storages/key_value_store.d.ts.map +1 -0
- package/storages/key_value_store.js +319 -0
- package/storages/key_value_store.js.map +1 -0
- package/storages/request_list.d.ts +461 -0
- package/storages/request_list.d.ts.map +1 -0
- package/storages/request_list.js +715 -0
- package/storages/request_list.js.map +1 -0
- package/storages/request_queue.d.ts +312 -0
- package/storages/request_queue.d.ts.map +1 -0
- package/storages/request_queue.js +708 -0
- package/storages/request_queue.js.map +1 -0
- package/storages/storage.d.ts +260 -0
- package/storages/storage.d.ts.map +1 -0
- package/storages/storage.js +3 -0
- package/storages/storage.js.map +1 -0
- package/storages/storage_manager.d.ts +40 -0
- package/storages/storage_manager.d.ts.map +1 -0
- package/storages/storage_manager.js +119 -0
- package/storages/storage_manager.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/typedefs.d.ts +12 -0
- package/typedefs.d.ts.map +1 -0
- package/typedefs.js +15 -0
- package/typedefs.js.map +1 -0
- package/validators.d.ts +21 -0
- package/validators.d.ts.map +1 -0
- package/validators.js +26 -0
- package/validators.js.map +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request_list.js","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";;;;AAAA,0CAAoD;AACpD,iDAAuC;AAEvC,oDAAiD;AACjD,gCAA6B;AAC7B,wCAAqD;AACrD,oDAAqE;AACrE,iEAA4D;AAG5D,gBAAgB;AACH,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAE1D,gBAAgB;AACH,QAAA,wBAAwB,GAAG,uBAAuB,CAAC;AAEhE,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAiJvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,MAAa,WAAW;IAmDpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QArD5C;;;;mBAAc,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;WAAC;QAEnD;;;;WAIG;QACH;;;;mBAAsB,EAAE;WAAC;QAEzB,kHAAkH;QAClH;;;;mBAAoB,CAAC;WAAC;QAEtB,gGAAgG;QAChG;;;;mBAAmD,EAAE;WAAC;QAEtD;;;WAGG;QACH;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;QAE/B;;;WAGG;QACH;;;;mBAAY,IAAI,GAAG,EAAU;WAAC;QAE9B;;;WAGG;QACH;;;;mBAAmB,IAAI;WAAC;QAExB;;;WAGG;QACH;;;;mBAAuB,KAAK;WAAC;QAE7B;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;;WAAiC;QACjC;;;;;WAAoC;QACpC;;;;;WAAwC;QACxC;;;;;WAA8B;QAC9B;;;;;WAAmC;QACnC;;;;;WAA0B;QAC1B;;;;;WAAqD;QACrD;;;;;WAA6B;QAMzB,MAAM,EACF,OAAO,EACP,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,KAAK,EACL,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,EAAE;YAC/B,MAAM,IAAI,kBAAa,CAAC,kEAAkE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACjH;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC1B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YACrC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,YAAE,CAAC,MAAM;gBACpB,aAAa,EAAE,YAAE,CAAC,MAAM;gBACxB,UAAU,EAAE,YAAE,CAAC,MAAM;aACxB,CAAC;YACF,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACzC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,eAAe,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACpF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,kBAAkB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,0HAA0H;QAC1H,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,qDAAqD;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;SAC9E;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAE/E,mFAAmF;QACnF,8CAA8C;QAC9C,IAAI,iBAAiB,EAAE;YACnB,MAAM,IAAI,CAAC,qBAAqB,CAAC,iBAA2B,CAAC,CAAC;SACjE;aAAM;YACH,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACxC;QAED,IAAI,CAAC,aAAa,CAAC,KAAyB,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzF,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,EAAE,qCAA0B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SACzE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,qBAAqB,CAAC,iBAAyB;QAC3D,gDAAgD;QAChD,uCAAuC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1B;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,MAAM,aAAa,GAAG,IAAA,iCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,aAAa,EAAE;YACvC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;SAC7B;IACL,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,uBAAuB;QACnC,wFAAwF;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,oEAAoE;YACpE,mDAAmD;YACnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAK,MAAqB,CAAC,eAAe,EAAE;gBACtE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAwB,CAAC,CAAC;gBACnF,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAwB,EAAE,eAAe,CAAC,CAAC;aAC7E;iBAAM;gBACH,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC5B;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI;gBACA,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,wBAAwB,GAAG,mBAAmB,CAAC,MAAM,CAAC;gBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,EAAE,CAAC,EAAE,EAAE;oBAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;oBAC3C,IAAI,CAAC,WAAW,CAAC,MAAO,CAAC,CAAC;iBAC7B;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC3F;SACJ;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;SAChF;QACD,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI;YACA,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;YAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;SAChC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yCAAyC,CAAC,CAAC;SACtE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAc,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAmB,EAAE,kBAAkB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAC9G,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,KAAwB;QAC5C,8DAA8D;QAC9D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,0BAA0B;QAC1B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;SAC5F;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;SACnG;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;eACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,aAAa,EAAE;YACrE,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;SACnH;QAED,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC3B,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACtH;YACD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,EAAE;gBAC1B,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACxC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE5C,cAAc;QACd,wEAAwE;QACxE,IAAI;QACJ,wBAAwB;QACxB,iDAAiD;QACjD,oBAAoB;QACpB,uCAAuC;QACvC,WAAW;QACX,2CAA2C;QAC3C,IAAI;QACJ,qGAAqG;QACrG,oFAAoF;QACpF,oCAAoC;QACpC,IAAI,oBAAoB,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0FAA0F,EAAE;gBACzG,oBAAoB;aACvB,CAAC,CAAC;YACH,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE;gBAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACrC;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,8BAA8B;QAC1C,IAAI,KAAwB,CAAC;QAC7B,IAAI,iBAA0B,CAAC;QAE/B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC/D;aAAM,IAAI,IAAI,CAAC,eAAe,EAAE;YAC7B,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5D,IAAI,KAAK;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;SAC7F;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC3E,IAAI,iBAAiB;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;SAC/G;QAED,OAAO,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAChD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,SAAS;gBACzC,CAAC,CAAC,IAAI;YACV,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC/E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChF,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACvD,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC/B;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,OAAO,OAAO,CAAC;SAClB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,MAAsB,EAAE,eAAiC;QACzF,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QAE5D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY;YACZ,aAAa;YACb,cAAc,EAAE,YAAY,GAAG,aAAa;YAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;IACP,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAI,GAAW;QAC7C,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAG,CAAC,CAAC;QAEhD,OAAO,KAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;SACxF;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;SAClF;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;SACb;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACO,WAAW,CAAC,MAAc;QAChC,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC;QAC3B,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,MAAgB,EAAE,CAAC,CAAC;SACpD;aAAM,IAAI,MAAM,YAAY,iBAAO,EAAE;YAClC,OAAO,GAAG,MAAM,CAAC;SACpB;aAAM,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE;YACpC,OAAO,GAAG,IAAI,iBAAO,CAAC,MAAwB,CAAC,CAAC;SACnD;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC;QAE9D,sDAAsD;QACtD,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,YAAY,EAAE;YACzC,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACnD;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtC,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,EAAE,4CAA4C;YAChG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC/B;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,SAAS,qFAAqF,CAAC,CAAC,CAAC,8BAA8B;SAC3K;IACL,CAAC;IAED;;;OAGG;IACO,qBAAqB,CAAC,SAAiB;QAC7C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;SAC7E;IACL,CAAC;IAED;;OAEG;IACO,gCAAgC,CAAC,SAAiB;QACxD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;SACnF;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,GAAG,CAAC,CAAC;SAClF;IACL,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;OAEG;IACH,MAAM;QACF,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,YAAY;QACR,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAuB,EAAE,OAAiB,EAAE,UAA8B,EAAE;QAC1F,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;QACtB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpD,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;YACvB,GAAG,OAAO;YACV,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,6BAAqB,EAAE,CAAC,CAAC,CAAC,SAAS;YAC9E,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,gCAAwB,EAAE,CAAC,CAAC,CAAC,SAAS;YACpF,OAAO;SACV,CAAC,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEtB,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA4C;QAC1E,OAAO,IAAA,0BAAkB,EAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACJ;AA/lBD,kCA+lBC"}
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
import { LruCache } from '@apify/datastructures';
|
|
2
|
+
import { Request, RequestOptions } from '../request';
|
|
3
|
+
import { StorageClient, RequestQueueClient, RequestQueueInfo, BatchAddRequestsResult } from './storage';
|
|
4
|
+
/**
|
|
5
|
+
* When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare const QUERY_HEAD_MIN_LENGTH = 100;
|
|
9
|
+
/** @internal */
|
|
10
|
+
export declare const QUERY_HEAD_BUFFER = 3;
|
|
11
|
+
/**
|
|
12
|
+
* If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS
|
|
13
|
+
* then we assume the get head operation to be consistent.
|
|
14
|
+
* @internal
|
|
15
|
+
*/
|
|
16
|
+
export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
|
|
17
|
+
/**
|
|
18
|
+
* How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
|
|
19
|
+
* @internal
|
|
20
|
+
*/
|
|
21
|
+
export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
|
|
22
|
+
/**
|
|
23
|
+
* Indicates how long it usually takes for the underlying storage to propagate all writes
|
|
24
|
+
* to be available to subsequent reads.
|
|
25
|
+
* @internal
|
|
26
|
+
*/
|
|
27
|
+
export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
|
|
28
|
+
/**
|
|
29
|
+
* Helper function that creates ID from uniqueKey for local emulation of request queue.
|
|
30
|
+
* It's also used for local cache of remote request queue.
|
|
31
|
+
*
|
|
32
|
+
* This function may not exactly match how requestId is created server side.
|
|
33
|
+
* So we never pass requestId created by this to server and use it only for local cache.
|
|
34
|
+
*
|
|
35
|
+
* @internal
|
|
36
|
+
*/
|
|
37
|
+
export declare function getRequestId(uniqueKey: string): string;
|
|
38
|
+
/**
|
|
39
|
+
* A helper class that is used to report results from various
|
|
40
|
+
* {@link RequestQueue} functions as well as {@link enqueueLinks}.
|
|
41
|
+
*/
|
|
42
|
+
export interface QueueOperationInfo {
|
|
43
|
+
/** Indicates if request was already present in the queue. */
|
|
44
|
+
wasAlreadyPresent: boolean;
|
|
45
|
+
/** Indicates if request was already marked as handled. */
|
|
46
|
+
wasAlreadyHandled: boolean;
|
|
47
|
+
/** The ID of the added request */
|
|
48
|
+
requestId: string;
|
|
49
|
+
uniqueKey: string;
|
|
50
|
+
}
|
|
51
|
+
export interface RequestQueueOperationOptions {
|
|
52
|
+
/**
|
|
53
|
+
* If set to `true`:
|
|
54
|
+
* - while adding the request to the queue: the request will be added to the foremost position in the queue.
|
|
55
|
+
* - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
|
|
56
|
+
* in the next call to {@link RequestQueue.fetchNextRequest}.
|
|
57
|
+
* By default, it's put to the end of the queue.
|
|
58
|
+
* @default false
|
|
59
|
+
*/
|
|
60
|
+
forefront?: boolean;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Represents a queue of URLs to crawl, which is used for deep crawling of websites
|
|
64
|
+
* where you start with several URLs and then recursively
|
|
65
|
+
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
66
|
+
*
|
|
67
|
+
* Each URL is represented using an instance of the {@link Request} class.
|
|
68
|
+
* The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
|
|
69
|
+
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
70
|
+
* To add a single URL multiple times to the queue,
|
|
71
|
+
* corresponding {@link Request} objects will need to have different `uniqueKey` properties.
|
|
72
|
+
*
|
|
73
|
+
* Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
|
|
74
|
+
*
|
|
75
|
+
* `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
|
|
76
|
+
* and {@link PlaywrightCrawler} as a source of URLs to crawl.
|
|
77
|
+
* Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
78
|
+
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
79
|
+
*
|
|
80
|
+
* `RequestQueue` stores its data either on local disk or in the Apify Cloud,
|
|
81
|
+
* depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
|
|
82
|
+
*
|
|
83
|
+
* If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
|
|
84
|
+
* that directory in an SQLite database file.
|
|
85
|
+
*
|
|
86
|
+
* If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
|
|
87
|
+
* [Apify Request Queue](https://docs.apify.com/storage/request-queue)
|
|
88
|
+
* cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
|
|
89
|
+
* option to {@link RequestQueue.open} function,
|
|
90
|
+
* even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
|
|
91
|
+
*
|
|
92
|
+
* **Example usage:**
|
|
93
|
+
*
|
|
94
|
+
* ```javascript
|
|
95
|
+
* // Open the default request queue associated with the actor run
|
|
96
|
+
* const queue = await RequestQueue.open();
|
|
97
|
+
*
|
|
98
|
+
* // Open a named request queue
|
|
99
|
+
* const queueWithName = await RequestQueue.open('some-name');
|
|
100
|
+
*
|
|
101
|
+
* // Enqueue few requests
|
|
102
|
+
* await queue.addRequest({ url: 'http://example.com/aaa' });
|
|
103
|
+
* await queue.addRequest({ url: 'http://example.com/bbb' });
|
|
104
|
+
* await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
|
|
105
|
+
* ```
|
|
106
|
+
* @category Sources
|
|
107
|
+
*/
|
|
108
|
+
export declare class RequestQueue {
|
|
109
|
+
log: import("@apify/log/log").Log;
|
|
110
|
+
id: string;
|
|
111
|
+
name?: string;
|
|
112
|
+
timeoutSecs: number;
|
|
113
|
+
clientKey: string;
|
|
114
|
+
client: RequestQueueClient;
|
|
115
|
+
/**
|
|
116
|
+
* Contains a cached list of request IDs from the head of the queue,
|
|
117
|
+
* as obtained in the last query. Both key and value is the request ID.
|
|
118
|
+
* Need to apply a type here to the generated TS types don't try to use types-apify
|
|
119
|
+
*/
|
|
120
|
+
private queueHeadDict;
|
|
121
|
+
queryQueueHeadPromise?: Promise<{
|
|
122
|
+
wasLimitReached: boolean;
|
|
123
|
+
prevLimit: number;
|
|
124
|
+
queueModifiedAt: Date;
|
|
125
|
+
queryStartedAt: Date;
|
|
126
|
+
hadMultipleClients: boolean;
|
|
127
|
+
}> | null;
|
|
128
|
+
inProgress: Set<unknown>;
|
|
129
|
+
recentlyHandled: LruCache<any>;
|
|
130
|
+
assumedTotalCount: number;
|
|
131
|
+
assumedHandledCount: number;
|
|
132
|
+
requestsCache: LruCache<Pick<QueueOperationInfo, "uniqueKey" | "wasAlreadyHandled"> & {
|
|
133
|
+
isHandled: boolean;
|
|
134
|
+
id: string;
|
|
135
|
+
}>;
|
|
136
|
+
/**
|
|
137
|
+
* @internal
|
|
138
|
+
*/
|
|
139
|
+
constructor(options: RequestQueueOptions);
|
|
140
|
+
/**
|
|
141
|
+
* @ignore
|
|
142
|
+
*/
|
|
143
|
+
inProgressCount(): number;
|
|
144
|
+
/**
|
|
145
|
+
* Adds a request to the queue.
|
|
146
|
+
*
|
|
147
|
+
* If a request with the same `uniqueKey` property is already present in the queue,
|
|
148
|
+
* it will not be updated. You can find out whether this happened from the resulting
|
|
149
|
+
* {@link QueueOperationInfo} object.
|
|
150
|
+
*
|
|
151
|
+
* To add multiple requests to the queue by extracting links from a webpage,
|
|
152
|
+
* see the {@link enqueueLinks} helper function.
|
|
153
|
+
*
|
|
154
|
+
* @param requestLike {@link Request} object or vanilla object with request data.
|
|
155
|
+
* Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
|
|
156
|
+
* @param [options] Request queue operation options.
|
|
157
|
+
*/
|
|
158
|
+
addRequest(requestLike: Request | RequestOptions, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo>;
|
|
159
|
+
/**
|
|
160
|
+
* Adds requests to the queue in batches of 25.
|
|
161
|
+
*
|
|
162
|
+
* If a request that is passed in is already present due to its `uniqueKey` property being the same,
|
|
163
|
+
* it will not be updated. You can find out whether this happened by finding the request in the resulting
|
|
164
|
+
* {@link BatchAddRequestsResult} object.
|
|
165
|
+
*
|
|
166
|
+
* @experimental
|
|
167
|
+
* @param requestsLike {@link Request} objects or vanilla objects with request data.
|
|
168
|
+
* Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
|
|
169
|
+
* @param [options] Request queue operation options.
|
|
170
|
+
*/
|
|
171
|
+
addRequests(requestsLike: (Request | RequestOptions)[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
172
|
+
/**
|
|
173
|
+
* Gets the request from the queue specified by ID.
|
|
174
|
+
*
|
|
175
|
+
* @param id ID of the request.
|
|
176
|
+
* @returns Returns the request object, or `null` if it was not found.
|
|
177
|
+
*/
|
|
178
|
+
getRequest(id: string): Promise<Request | null>;
|
|
179
|
+
/**
|
|
180
|
+
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
|
|
181
|
+
*
|
|
182
|
+
* Once you successfully finish processing of the request, you need to call
|
|
183
|
+
* {@link RequestQueue.markRequestHandled}
|
|
184
|
+
* to mark the request as handled in the queue. If there was some error in processing the request,
|
|
185
|
+
* call {@link RequestQueue.reclaimRequest} instead,
|
|
186
|
+
* so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
|
|
187
|
+
*
|
|
188
|
+
* Note that the `null` return value doesn't mean the queue processing finished,
|
|
189
|
+
* it means there are currently no pending requests.
|
|
190
|
+
* To check whether all requests in queue were finished,
|
|
191
|
+
* use {@link RequestQueue.isFinished} instead.
|
|
192
|
+
*
|
|
193
|
+
* @returns
|
|
194
|
+
* Returns the request object or `null` if there are no more pending requests.
|
|
195
|
+
*/
|
|
196
|
+
fetchNextRequest(): Promise<Request | null>;
|
|
197
|
+
/**
|
|
198
|
+
* Marks a request that was previously returned by the
|
|
199
|
+
* {@link RequestQueue.fetchNextRequest}
|
|
200
|
+
* function as handled after successful processing.
|
|
201
|
+
* Handled requests will never again be returned by the `fetchNextRequest` function.
|
|
202
|
+
*/
|
|
203
|
+
markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
|
|
204
|
+
/**
|
|
205
|
+
* Reclaims a failed request back to the queue, so that it can be returned for processing later again
|
|
206
|
+
* by another call to {@link RequestQueue.fetchNextRequest}.
|
|
207
|
+
* The request record in the queue is updated using the provided `request` parameter.
|
|
208
|
+
* For example, this lets you store the number of retries or error messages for the request.
|
|
209
|
+
*/
|
|
210
|
+
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | null>;
|
|
211
|
+
/**
|
|
212
|
+
* Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
|
|
213
|
+
* would return `null`, otherwise it resolves to `false`.
|
|
214
|
+
* Note that even if the queue is empty, there might be some pending requests currently being processed.
|
|
215
|
+
* If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
|
|
216
|
+
*/
|
|
217
|
+
isEmpty(): Promise<boolean>;
|
|
218
|
+
/**
|
|
219
|
+
* Resolves to `true` if all requests were already handled and there are no more left.
|
|
220
|
+
* Due to the nature of distributed storage used by the queue,
|
|
221
|
+
* the function might occasionally return a false negative,
|
|
222
|
+
* but it will never return a false positive.
|
|
223
|
+
*/
|
|
224
|
+
isFinished(): Promise<boolean>;
|
|
225
|
+
/**
|
|
226
|
+
* Caches information about request to beware of unneeded addRequest() calls.
|
|
227
|
+
*/
|
|
228
|
+
protected _cacheRequest(cacheKey: string, queueOperationInfo: QueueOperationInfo): void;
|
|
229
|
+
/**
|
|
230
|
+
* We always request more items than is in progress to ensure that something falls into head.
|
|
231
|
+
*
|
|
232
|
+
* @param [ensureConsistency] If true then query for queue head is retried until queueModifiedAt
|
|
233
|
+
* is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
|
|
234
|
+
* head is consistent.
|
|
235
|
+
* @default false
|
|
236
|
+
* @param [limit] How many queue head items will be fetched.
|
|
237
|
+
* @param [iteration] Used when this function is called recursively to limit the recursion.
|
|
238
|
+
* @returns Indicates if queue head is consistent (true) or inconsistent (false).
|
|
239
|
+
*/
|
|
240
|
+
protected _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): Promise<boolean>;
|
|
241
|
+
/**
|
|
242
|
+
* Adds a request straight to the queueHeadDict, to improve performance.
|
|
243
|
+
*/
|
|
244
|
+
private _maybeAddRequestToQueueHead;
|
|
245
|
+
/**
|
|
246
|
+
* Removes the queue either from the Apify Cloud storage or from the local database,
|
|
247
|
+
* depending on the mode of operation.
|
|
248
|
+
*/
|
|
249
|
+
drop(): Promise<void>;
|
|
250
|
+
/**
|
|
251
|
+
* Returns the number of handled requests.
|
|
252
|
+
*
|
|
253
|
+
* This function is just a convenient shortcut for:
|
|
254
|
+
*
|
|
255
|
+
* ```javascript
|
|
256
|
+
* const { handledRequestCount } = await queue.getInfo();
|
|
257
|
+
* ```
|
|
258
|
+
*/
|
|
259
|
+
handledCount(): Promise<number>;
|
|
260
|
+
/**
|
|
261
|
+
* Returns an object containing general information about the request queue.
|
|
262
|
+
*
|
|
263
|
+
* The function returns the same object as the Apify API Client's
|
|
264
|
+
* [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
|
|
265
|
+
* function, which in turn calls the
|
|
266
|
+
* [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
|
|
267
|
+
* API endpoint.
|
|
268
|
+
*
|
|
269
|
+
* **Example:**
|
|
270
|
+
* ```
|
|
271
|
+
* {
|
|
272
|
+
* id: "WkzbQMuFYuamGv3YF",
|
|
273
|
+
* name: "my-queue",
|
|
274
|
+
* userId: "wRsJZtadYvn4mBZmm",
|
|
275
|
+
* createdAt: new Date("2015-12-12T07:34:14.202Z"),
|
|
276
|
+
* modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
|
|
277
|
+
* accessedAt: new Date("2015-12-14T08:36:13.202Z"),
|
|
278
|
+
* totalRequestCount: 25,
|
|
279
|
+
* handledRequestCount: 5,
|
|
280
|
+
* pendingRequestCount: 20,
|
|
281
|
+
* }
|
|
282
|
+
* ```
|
|
283
|
+
*/
|
|
284
|
+
getInfo(): Promise<RequestQueueInfo | undefined>;
|
|
285
|
+
/**
|
|
286
|
+
* Opens a request queue and returns a promise resolving to an instance
|
|
287
|
+
* of the {@link RequestQueue} class.
|
|
288
|
+
*
|
|
289
|
+
* {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
290
|
+
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
291
|
+
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
292
|
+
* and depth-first crawling orders.
|
|
293
|
+
*
|
|
294
|
+
* For more details and code examples, see the {@link RequestQueue} class.
|
|
295
|
+
*
|
|
296
|
+
* @param [queueIdOrName]
|
|
297
|
+
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
298
|
+
* the function returns the default request queue associated with the actor run.
|
|
299
|
+
* @param [options] Open Request Queue options.
|
|
300
|
+
*/
|
|
301
|
+
static open(queueIdOrName?: string | null): Promise<RequestQueue>;
|
|
302
|
+
}
|
|
303
|
+
export interface RequestQueueOptions {
|
|
304
|
+
id: string;
|
|
305
|
+
name?: string;
|
|
306
|
+
client: StorageClient;
|
|
307
|
+
}
|
|
308
|
+
export interface QueueOperationInfoOptions {
|
|
309
|
+
requestId: string;
|
|
310
|
+
wasAlreadyHandled: boolean;
|
|
311
|
+
}
|
|
312
|
+
//# sourceMappingURL=request_queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAQjE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAIxG;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAS7C;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAQ7C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAE/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,qBAAa,YAAY;IACrB,GAAG,+BAAyC;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,aAAa,CAAgC;IACrD,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAC5B,eAAe,EAAE,OAAO,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,EAAE,IAAI,CAAC;QACtB,cAAc,EAAE,IAAI,CAAC;QACrB,kBAAkB,EAAE,OAAO,CAAC;KAC/B,CAAC,GAAG,IAAI,CAAQ;IAIjB,UAAU,eAAa;IAKvB,eAAe,gBAA4D;IAI3E,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAMxB,aAAa;mBACkE,OAAO;YAAM,MAAM;OAC5D;IAEtC;;OAEG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;OAEG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,cAAc,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAgDhI;;;;;;;;;;;OAWG;IACG,WAAW,CACb,YAAY,EAAE,CAAC,OAAO,GAAG,cAAc,CAAC,EAAE,EAC1C,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAwFlC;;;;;OAKG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAoBrD;;;;;;;;;;;;;;;;OAgBG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAyDjD;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IA6B9E;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAuCtH;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAOpC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,kBAAkB,GAAG,IAAI;IASvF;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmFnB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAQnC;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAItD;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;CAK1E;AAED,MAAM,WAAW,mBAAmB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;CAC9B"}
|