@crawlee/core 3.0.0-alpha.0 → 3.0.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/dist/LICENSE.md +201 -0
  2. package/{README.md → dist/README.md} +0 -0
  3. package/dist/autoscaling/autoscaled_pool.d.ts +286 -0
  4. package/dist/autoscaling/autoscaled_pool.d.ts.map +1 -0
  5. package/dist/autoscaling/autoscaled_pool.js +624 -0
  6. package/dist/autoscaling/autoscaled_pool.js.map +1 -0
  7. package/dist/autoscaling/index.d.ts +4 -0
  8. package/dist/autoscaling/index.d.ts.map +1 -0
  9. package/dist/autoscaling/index.js +7 -0
  10. package/dist/autoscaling/index.js.map +1 -0
  11. package/dist/autoscaling/snapshotter.d.ts +193 -0
  12. package/dist/autoscaling/snapshotter.d.ts.map +1 -0
  13. package/dist/autoscaling/snapshotter.js +377 -0
  14. package/dist/autoscaling/snapshotter.js.map +1 -0
  15. package/dist/autoscaling/system_status.d.ts +181 -0
  16. package/dist/autoscaling/system_status.d.ts.map +1 -0
  17. package/dist/autoscaling/system_status.js +194 -0
  18. package/dist/autoscaling/system_status.js.map +1 -0
  19. package/dist/cache_container.d.ts +18 -0
  20. package/dist/cache_container.d.ts.map +1 -0
  21. package/dist/cache_container.js +42 -0
  22. package/dist/cache_container.js.map +1 -0
  23. package/dist/configuration.d.ts +150 -0
  24. package/dist/configuration.d.ts.map +1 -0
  25. package/dist/configuration.js +284 -0
  26. package/dist/configuration.js.map +1 -0
  27. package/dist/constants.d.ts +3 -0
  28. package/dist/constants.d.ts.map +1 -0
  29. package/dist/constants.js +6 -0
  30. package/dist/constants.js.map +1 -0
  31. package/dist/crawlers/crawler_commons.d.ts +23 -0
  32. package/dist/crawlers/crawler_commons.d.ts.map +1 -0
  33. package/dist/crawlers/crawler_commons.js +3 -0
  34. package/dist/crawlers/crawler_commons.js.map +1 -0
  35. package/dist/crawlers/crawler_extension.d.ts +11 -0
  36. package/dist/crawlers/crawler_extension.d.ts.map +1 -0
  37. package/dist/crawlers/crawler_extension.js +30 -0
  38. package/dist/crawlers/crawler_extension.js.map +1 -0
  39. package/dist/crawlers/crawler_utils.d.ts +25 -0
  40. package/dist/crawlers/crawler_utils.d.ts.map +1 -0
  41. package/dist/crawlers/crawler_utils.js +72 -0
  42. package/dist/crawlers/crawler_utils.js.map +1 -0
  43. package/dist/crawlers/index.d.ts +5 -0
  44. package/dist/crawlers/index.d.ts.map +1 -0
  45. package/dist/crawlers/index.js +8 -0
  46. package/dist/crawlers/index.js.map +1 -0
  47. package/dist/crawlers/statistics.d.ts +147 -0
  48. package/dist/crawlers/statistics.d.ts.map +1 -0
  49. package/dist/crawlers/statistics.js +360 -0
  50. package/dist/crawlers/statistics.js.map +1 -0
  51. package/dist/enqueue_links/enqueue_links.d.ts +138 -0
  52. package/dist/enqueue_links/enqueue_links.d.ts.map +1 -0
  53. package/dist/enqueue_links/enqueue_links.js +123 -0
  54. package/dist/enqueue_links/enqueue_links.js.map +1 -0
  55. package/dist/enqueue_links/index.d.ts +3 -0
  56. package/dist/enqueue_links/index.d.ts.map +1 -0
  57. package/dist/enqueue_links/index.js +6 -0
  58. package/dist/enqueue_links/index.js.map +1 -0
  59. package/dist/enqueue_links/shared.d.ts +60 -0
  60. package/dist/enqueue_links/shared.d.ts.map +1 -0
  61. package/dist/enqueue_links/shared.js +148 -0
  62. package/dist/enqueue_links/shared.js.map +1 -0
  63. package/dist/errors.d.ts +8 -0
  64. package/dist/errors.d.ts.map +1 -0
  65. package/dist/errors.js +12 -0
  66. package/dist/errors.js.map +1 -0
  67. package/dist/events/event_manager.d.ts +41 -0
  68. package/dist/events/event_manager.d.ts.map +1 -0
  69. package/dist/events/event_manager.js +103 -0
  70. package/dist/events/event_manager.js.map +1 -0
  71. package/dist/events/index.d.ts +3 -0
  72. package/dist/events/index.d.ts.map +1 -0
  73. package/dist/events/index.js +6 -0
  74. package/dist/events/index.js.map +1 -0
  75. package/dist/events/local_event_manager.d.ts +32 -0
  76. package/dist/events/local_event_manager.d.ts.map +1 -0
  77. package/dist/events/local_event_manager.js +114 -0
  78. package/dist/events/local_event_manager.js.map +1 -0
  79. package/dist/index.d.ts +14 -0
  80. package/dist/index.d.ts.map +1 -0
  81. package/dist/index.js +17 -0
  82. package/dist/index.js.map +1 -0
  83. package/dist/index.mjs +63 -0
  84. package/dist/log.d.ts +3 -0
  85. package/dist/log.d.ts.map +1 -0
  86. package/dist/log.js +12 -0
  87. package/dist/log.js.map +1 -0
  88. package/dist/package.json +82 -0
  89. package/dist/proxy_configuration.d.ts +181 -0
  90. package/dist/proxy_configuration.d.ts.map +1 -0
  91. package/dist/proxy_configuration.js +209 -0
  92. package/dist/proxy_configuration.js.map +1 -0
  93. package/dist/request.d.ts +174 -0
  94. package/dist/request.d.ts.map +1 -0
  95. package/dist/request.js +270 -0
  96. package/dist/request.js.map +1 -0
  97. package/dist/serialization.d.ts +33 -0
  98. package/dist/serialization.d.ts.map +1 -0
  99. package/dist/serialization.js +146 -0
  100. package/dist/serialization.js.map +1 -0
  101. package/dist/session_pool/errors.d.ts +8 -0
  102. package/dist/session_pool/errors.d.ts.map +1 -0
  103. package/dist/session_pool/errors.js +20 -0
  104. package/dist/session_pool/errors.js.map +1 -0
  105. package/dist/session_pool/events.d.ts +3 -0
  106. package/dist/session_pool/events.d.ts.map +1 -0
  107. package/dist/session_pool/events.js +6 -0
  108. package/dist/session_pool/events.js.map +1 -0
  109. package/dist/session_pool/index.d.ts +6 -0
  110. package/dist/session_pool/index.d.ts.map +1 -0
  111. package/dist/session_pool/index.js +9 -0
  112. package/dist/session_pool/index.js.map +1 -0
  113. package/dist/session_pool/session.d.ts +212 -0
  114. package/dist/session_pool/session.d.ts.map +1 -0
  115. package/dist/session_pool/session.js +360 -0
  116. package/dist/session_pool/session.js.map +1 -0
  117. package/dist/session_pool/session_pool.d.ts +222 -0
  118. package/dist/session_pool/session_pool.d.ts.map +1 -0
  119. package/dist/session_pool/session_pool.js +389 -0
  120. package/dist/session_pool/session_pool.js.map +1 -0
  121. package/dist/session_pool/session_utils.d.ts +9 -0
  122. package/dist/session_pool/session_utils.d.ts.map +1 -0
  123. package/dist/session_pool/session_utils.js +19 -0
  124. package/dist/session_pool/session_utils.js.map +1 -0
  125. package/dist/storages/dataset.d.ts +358 -0
  126. package/dist/storages/dataset.d.ts.map +1 -0
  127. package/dist/storages/dataset.js +415 -0
  128. package/dist/storages/dataset.js.map +1 -0
  129. package/dist/storages/index.d.ts +7 -0
  130. package/dist/storages/index.d.ts.map +1 -0
  131. package/dist/storages/index.js +11 -0
  132. package/dist/storages/index.js.map +1 -0
  133. package/dist/storages/key_value_store.d.ts +242 -0
  134. package/dist/storages/key_value_store.d.ts.map +1 -0
  135. package/dist/storages/key_value_store.js +319 -0
  136. package/dist/storages/key_value_store.js.map +1 -0
  137. package/dist/storages/request_list.d.ts +461 -0
  138. package/dist/storages/request_list.d.ts.map +1 -0
  139. package/dist/storages/request_list.js +715 -0
  140. package/dist/storages/request_list.js.map +1 -0
  141. package/dist/storages/request_queue.d.ts +312 -0
  142. package/dist/storages/request_queue.d.ts.map +1 -0
  143. package/dist/storages/request_queue.js +708 -0
  144. package/dist/storages/request_queue.js.map +1 -0
  145. package/dist/storages/storage.d.ts +260 -0
  146. package/dist/storages/storage.d.ts.map +1 -0
  147. package/dist/storages/storage.js +3 -0
  148. package/dist/storages/storage.js.map +1 -0
  149. package/dist/storages/storage_manager.d.ts +40 -0
  150. package/dist/storages/storage_manager.d.ts.map +1 -0
  151. package/dist/storages/storage_manager.js +119 -0
  152. package/dist/storages/storage_manager.js.map +1 -0
  153. package/dist/tsconfig.build.tsbuildinfo +1 -0
  154. package/dist/typedefs.d.ts +12 -0
  155. package/dist/typedefs.d.ts.map +1 -0
  156. package/dist/typedefs.js +15 -0
  157. package/dist/typedefs.js.map +1 -0
  158. package/dist/validators.d.ts +21 -0
  159. package/dist/validators.d.ts.map +1 -0
  160. package/dist/validators.js +26 -0
  161. package/dist/validators.js.map +1 -0
  162. package/package.json +11 -10
@@ -0,0 +1 @@
1
+ {"version":3,"file":"request_list.js","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";;;;AAAA,0CAAoD;AACpD,iDAAuC;AAEvC,oDAAiD;AACjD,gCAA6B;AAC7B,wCAAqD;AACrD,oDAAqE;AACrE,iEAA4D;AAG5D,gBAAgB;AACH,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAE1D,gBAAgB;AACH,QAAA,wBAAwB,GAAG,uBAAuB,CAAC;AAEhE,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAiJvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,MAAa,WAAW;IAmDpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QArD5C;;;;mBAAc,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;WAAC;QAEnD;;;;WAIG;QACH;;;;mBAAsB,EAAE;WAAC;QAEzB,kHAAkH;QAClH;;;;mBAAoB,CAAC;WAAC;QAEtB,gGAAgG;QAChG;;;;mBAAmD,EAAE;WAAC;QAEtD;;;WAGG;QACH;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;QAE/B;;;WAGG;QACH;;;;mBAAY,IAAI,GAAG,EAAU;WAAC;QAE9B;;;WAGG;QACH;;;;mBAAmB,IAAI;WAAC;QAExB;;;WAGG;QACH;;;;mBAAuB,KAAK;WAAC;QAE7B;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;;WAAiC;QACjC;;;;;WAAoC;QACpC;;;;;WAAwC;QACxC;;;;;WAA8B;QAC9B;;;;;WAAmC;QACnC;;;;;WAA0B;QAC1B;;;;;WAAqD;QACrD;;;;;WAA6B;QAMzB,MAAM,EACF,OAAO,EACP,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,KAAK,EACL,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,EAAE;YAC/B,MAAM,IAAI,kBAAa,CAAC,kEAAkE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACjH;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC1B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YACrC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,YAAE,CAAC,MAAM;gBACpB,aAAa,EAAE,YAAE,CAAC,MAAM;gBACxB,UAAU,EAAE,YAAE,CAAC,MAAM;aACxB,CAAC;YACF,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACzC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,eAAe,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACpF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,kBAAkB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,0HAA0H;QAC1H,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,qDAAqD;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;SAC9E;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAE/E,mFAAmF;QACnF,8CAA8C;QAC9C,IAAI,iBAAiB,EAAE;YACnB,MAAM,IAAI,CAAC,qBAAqB,CAAC,iBAA2B,CAAC,CAAC;SACjE;aAAM;YACH,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACxC;QAED,IAAI,CAAC,aAAa,CAAC,KAAyB,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzF,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,EAAE,qCAA0B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SACzE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,qBAAqB,CAAC,iBAAyB;QAC3D,gDAAgD;QAChD,uCAAuC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1B;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,MAAM,aAAa,GAAG,IAAA,iCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,aAAa,EAAE;YACvC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;SAC7B;IACL,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,uBAAuB;QACnC,wFAAwF;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,oEAAoE;YACpE,mDAAmD;YACnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAK,MAAqB,CAAC,eAAe,EAAE;gBACtE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAwB,CAAC,CAAC;gBACnF,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAwB,EAAE,eAAe,CAAC,CAAC;aAC7E;iBAAM;gBACH,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC5B;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI;gBACA,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,wBAAwB,GAAG,mBAAmB,CAAC,MAAM,CAAC;gBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,EAAE,CAAC,EAAE,EAAE;oBAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;oBAC3C,IAAI,CAAC,WAAW,CAAC,MAAO,CAAC,CAAC;iBAC7B;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC3F;SACJ;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;SAChF;QACD,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI;YACA,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;YAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;SAChC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yCAAyC,CAAC,CAAC;SACtE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAc,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAmB,EAAE,kBAAkB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAC9G,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,KAAwB;QAC5C,8DAA8D;QAC9D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,0BAA0B;QAC1B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;SAC5F;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;SACnG;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;eACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,aAAa,EAAE;YACrE,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;SACnH;QAED,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC3B,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACtH;YACD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,EAAE;gBAC1B,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACxC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE5C,cAAc;QACd,wEAAwE;QACxE,IAAI;QACJ,wBAAwB;QACxB,iDAAiD;QACjD,oBAAoB;QACpB,uCAAuC;QACvC,WAAW;QACX,2CAA2C;QAC3C,IAAI;QACJ,qGAAqG;QACrG,oFAAoF;QACpF,oCAAoC;QACpC,IAAI,oBAAoB,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0FAA0F,EAAE;gBACzG,oBAAoB;aACvB,CAAC,CAAC;YACH,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE;gBAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACrC;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,8BAA8B;QAC1C,IAAI,KAAwB,CAAC;QAC7B,IAAI,iBAA0B,CAAC;QAE/B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC/D;aAAM,IAAI,IAAI,CAAC,eAAe,EAAE;YAC7B,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5D,IAAI,KAAK;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;SAC7F;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC3E,IAAI,iBAAiB;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;SAC/G;QAED,OAAO,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAChD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,SAAS;gBACzC,CAAC,CAAC,IAAI;YACV,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC/E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChF,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACvD,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC/B;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,OAAO,OAAO,CAAC;SAClB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,MAAsB,EAAE,eAAiC;QACzF,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QAE5D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY;YACZ,aAAa;YACb,cAAc,EAAE,YAAY,GAAG,aAAa;YAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;IACP,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAI,GAAW;QAC7C,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAG,CAAC,CAAC;QAEhD,OAAO,KAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;SACxF;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;SAClF;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;SACb;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACO,WAAW,CAAC,MAAc;QAChC,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC;QAC3B,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,MAAgB,EAAE,CAAC,CAAC;SACpD;aAAM,IAAI,MAAM,YAAY,iBAAO,EAAE;YAClC,OAAO,GAAG,MAAM,CAAC;SACpB;aAAM,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE;YACpC,OAAO,GAAG,IAAI,iBAAO,CAAC,MAAwB,CAAC,CAAC;SACnD;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC;QAE9D,sDAAsD;QACtD,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,YAAY,EAAE;YACzC,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACnD;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtC,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,EAAE,4CAA4C;YAChG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC/B;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,SAAS,qFAAqF,CAAC,CAAC,CAAC,8BAA8B;SAC3K;IACL,CAAC;IAED;;;OAGG;IACO,qBAAqB,CAAC,SAAiB;QAC7C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;SAC7E;IACL,CAAC;IAED;;OAEG;IACO,gCAAgC,CAAC,SAAiB;QACxD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;SACnF;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,GAAG,CAAC,CAAC;SAClF;IACL,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;OAEG;IACH,MAAM;QACF,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,YAAY;QACR,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAuB,EAAE,OAAiB,EAAE,UAA8B,EAAE;QAC1F,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;QACtB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpD,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;YACvB,GAAG,OAAO;YACV,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,6BAAqB,EAAE,CAAC,CAAC,CAAC,SAAS;YAC9E,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,gCAAwB,EAAE,CAAC,CAAC,CAAC,SAAS;YACpF,OAAO;SACV,CAAC,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEtB,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA4C;QAC1E,OAAO,IAAA,0BAAkB,EAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACJ;AA/lBD,kCA+lBC"}
@@ -0,0 +1,312 @@
1
+ import { LruCache } from '@apify/datastructures';
2
+ import { Request, RequestOptions } from '../request';
3
+ import { StorageClient, RequestQueueClient, RequestQueueInfo, BatchAddRequestsResult } from './storage';
4
+ /**
5
+ * When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
6
+ * @internal
7
+ */
8
+ export declare const QUERY_HEAD_MIN_LENGTH = 100;
9
+ /** @internal */
10
+ export declare const QUERY_HEAD_BUFFER = 3;
11
+ /**
12
+ * If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS
13
+ * then we assume the get head operation to be consistent.
14
+ * @internal
15
+ */
16
+ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
17
+ /**
18
+ * How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
19
+ * @internal
20
+ */
21
+ export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
22
+ /**
23
+ * Indicates how long it usually takes for the underlying storage to propagate all writes
24
+ * to be available to subsequent reads.
25
+ * @internal
26
+ */
27
+ export declare const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
28
+ /**
29
+ * Helper function that creates ID from uniqueKey for local emulation of request queue.
30
+ * It's also used for local cache of remote request queue.
31
+ *
32
+ * This function may not exactly match how requestId is created server side.
33
+ * So we never pass requestId created by this to server and use it only for local cache.
34
+ *
35
+ * @internal
36
+ */
37
+ export declare function getRequestId(uniqueKey: string): string;
38
+ /**
39
+ * A helper class that is used to report results from various
40
+ * {@link RequestQueue} functions as well as {@link enqueueLinks}.
41
+ */
42
+ export interface QueueOperationInfo {
43
+ /** Indicates if request was already present in the queue. */
44
+ wasAlreadyPresent: boolean;
45
+ /** Indicates if request was already marked as handled. */
46
+ wasAlreadyHandled: boolean;
47
+ /** The ID of the added request */
48
+ requestId: string;
49
+ uniqueKey: string;
50
+ }
51
+ export interface RequestQueueOperationOptions {
52
+ /**
53
+ * If set to `true`:
54
+ * - while adding the request to the queue: the request will be added to the foremost position in the queue.
55
+ * - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
56
+ * in the next call to {@link RequestQueue.fetchNextRequest}.
57
+ * By default, it's put to the end of the queue.
58
+ * @default false
59
+ */
60
+ forefront?: boolean;
61
+ }
62
+ /**
63
+ * Represents a queue of URLs to crawl, which is used for deep crawling of websites
64
+ * where you start with several URLs and then recursively
65
+ * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
66
+ *
67
+ * Each URL is represented using an instance of the {@link Request} class.
68
+ * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
69
+ * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
70
+ * To add a single URL multiple times to the queue,
71
+ * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
72
+ *
73
+ * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
74
+ *
75
+ * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
76
+ * and {@link PlaywrightCrawler} as a source of URLs to crawl.
77
+ * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
78
+ * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
79
+ *
80
+ * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
81
+ * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
82
+ *
83
+ * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
84
+ * that directory in an SQLite database file.
85
+ *
86
+ * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
87
+ * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
88
+ * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
89
+ * option to {@link RequestQueue.open} function,
90
+ * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
91
+ *
92
+ * **Example usage:**
93
+ *
94
+ * ```javascript
95
+ * // Open the default request queue associated with the actor run
96
+ * const queue = await RequestQueue.open();
97
+ *
98
+ * // Open a named request queue
99
+ * const queueWithName = await RequestQueue.open('some-name');
100
+ *
101
+ * // Enqueue few requests
102
+ * await queue.addRequest({ url: 'http://example.com/aaa' });
103
+ * await queue.addRequest({ url: 'http://example.com/bbb' });
104
+ * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
105
+ * ```
106
+ * @category Sources
107
+ */
108
+ export declare class RequestQueue {
109
+ log: import("@apify/log/log").Log;
110
+ id: string;
111
+ name?: string;
112
+ timeoutSecs: number;
113
+ clientKey: string;
114
+ client: RequestQueueClient;
115
+ /**
116
+ * Contains a cached list of request IDs from the head of the queue,
117
+ * as obtained in the last query. Both key and value is the request ID.
118
+ * Need to apply a type here to the generated TS types don't try to use types-apify
119
+ */
120
+ private queueHeadDict;
121
+ queryQueueHeadPromise?: Promise<{
122
+ wasLimitReached: boolean;
123
+ prevLimit: number;
124
+ queueModifiedAt: Date;
125
+ queryStartedAt: Date;
126
+ hadMultipleClients: boolean;
127
+ }> | null;
128
+ inProgress: Set<unknown>;
129
+ recentlyHandled: LruCache<any>;
130
+ assumedTotalCount: number;
131
+ assumedHandledCount: number;
132
+ requestsCache: LruCache<Pick<QueueOperationInfo, "uniqueKey" | "wasAlreadyHandled"> & {
133
+ isHandled: boolean;
134
+ id: string;
135
+ }>;
136
+ /**
137
+ * @internal
138
+ */
139
+ constructor(options: RequestQueueOptions);
140
+ /**
141
+ * @ignore
142
+ */
143
+ inProgressCount(): number;
144
+ /**
145
+ * Adds a request to the queue.
146
+ *
147
+ * If a request with the same `uniqueKey` property is already present in the queue,
148
+ * it will not be updated. You can find out whether this happened from the resulting
149
+ * {@link QueueOperationInfo} object.
150
+ *
151
+ * To add multiple requests to the queue by extracting links from a webpage,
152
+ * see the {@link enqueueLinks} helper function.
153
+ *
154
+ * @param requestLike {@link Request} object or vanilla object with request data.
155
+ * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
156
+ * @param [options] Request queue operation options.
157
+ */
158
+ addRequest(requestLike: Request | RequestOptions, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo>;
159
+ /**
160
+ * Adds requests to the queue in batches of 25.
161
+ *
162
+ * If a request that is passed in is already present due to its `uniqueKey` property being the same,
163
+ * it will not be updated. You can find out whether this happened by finding the request in the resulting
164
+ * {@link BatchAddRequestsResult} object.
165
+ *
166
+ * @experimental
167
+ * @param requestsLike {@link Request} objects or vanilla objects with request data.
168
+ * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
169
+ * @param [options] Request queue operation options.
170
+ */
171
+ addRequests(requestsLike: (Request | RequestOptions)[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
172
+ /**
173
+ * Gets the request from the queue specified by ID.
174
+ *
175
+ * @param id ID of the request.
176
+ * @returns Returns the request object, or `null` if it was not found.
177
+ */
178
+ getRequest(id: string): Promise<Request | null>;
179
+ /**
180
+ * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
181
+ *
182
+ * Once you successfully finish processing of the request, you need to call
183
+ * {@link RequestQueue.markRequestHandled}
184
+ * to mark the request as handled in the queue. If there was some error in processing the request,
185
+ * call {@link RequestQueue.reclaimRequest} instead,
186
+ * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
187
+ *
188
+ * Note that the `null` return value doesn't mean the queue processing finished,
189
+ * it means there are currently no pending requests.
190
+ * To check whether all requests in queue were finished,
191
+ * use {@link RequestQueue.isFinished} instead.
192
+ *
193
+ * @returns
194
+ * Returns the request object or `null` if there are no more pending requests.
195
+ */
196
+ fetchNextRequest(): Promise<Request | null>;
197
+ /**
198
+ * Marks a request that was previously returned by the
199
+ * {@link RequestQueue.fetchNextRequest}
200
+ * function as handled after successful processing.
201
+ * Handled requests will never again be returned by the `fetchNextRequest` function.
202
+ */
203
+ markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
204
+ /**
205
+ * Reclaims a failed request back to the queue, so that it can be returned for processing later again
206
+ * by another call to {@link RequestQueue.fetchNextRequest}.
207
+ * The request record in the queue is updated using the provided `request` parameter.
208
+ * For example, this lets you store the number of retries or error messages for the request.
209
+ */
210
+ reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | null>;
211
+ /**
212
+ * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
213
+ * would return `null`, otherwise it resolves to `false`.
214
+ * Note that even if the queue is empty, there might be some pending requests currently being processed.
215
+ * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
216
+ */
217
+ isEmpty(): Promise<boolean>;
218
+ /**
219
+ * Resolves to `true` if all requests were already handled and there are no more left.
220
+ * Due to the nature of distributed storage used by the queue,
221
+ * the function might occasionally return a false negative,
222
+ * but it will never return a false positive.
223
+ */
224
+ isFinished(): Promise<boolean>;
225
+ /**
226
+ * Caches information about request to beware of unneeded addRequest() calls.
227
+ */
228
+ protected _cacheRequest(cacheKey: string, queueOperationInfo: QueueOperationInfo): void;
229
+ /**
230
+ * We always request more items than is in progress to ensure that something falls into head.
231
+ *
232
+ * @param [ensureConsistency] If true then query for queue head is retried until queueModifiedAt
233
+ * is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
234
+ * head is consistent.
235
+ * @default false
236
+ * @param [limit] How many queue head items will be fetched.
237
+ * @param [iteration] Used when this function is called recursively to limit the recursion.
238
+ * @returns Indicates if queue head is consistent (true) or inconsistent (false).
239
+ */
240
+ protected _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): Promise<boolean>;
241
+ /**
242
+ * Adds a request straight to the queueHeadDict, to improve performance.
243
+ */
244
+ private _maybeAddRequestToQueueHead;
245
+ /**
246
+ * Removes the queue either from the Apify Cloud storage or from the local database,
247
+ * depending on the mode of operation.
248
+ */
249
+ drop(): Promise<void>;
250
+ /**
251
+ * Returns the number of handled requests.
252
+ *
253
+ * This function is just a convenient shortcut for:
254
+ *
255
+ * ```javascript
256
+ * const { handledRequestCount } = await queue.getInfo();
257
+ * ```
258
+ */
259
+ handledCount(): Promise<number>;
260
+ /**
261
+ * Returns an object containing general information about the request queue.
262
+ *
263
+ * The function returns the same object as the Apify API Client's
264
+ * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
265
+ * function, which in turn calls the
266
+ * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
267
+ * API endpoint.
268
+ *
269
+ * **Example:**
270
+ * ```
271
+ * {
272
+ * id: "WkzbQMuFYuamGv3YF",
273
+ * name: "my-queue",
274
+ * userId: "wRsJZtadYvn4mBZmm",
275
+ * createdAt: new Date("2015-12-12T07:34:14.202Z"),
276
+ * modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
277
+ * accessedAt: new Date("2015-12-14T08:36:13.202Z"),
278
+ * totalRequestCount: 25,
279
+ * handledRequestCount: 5,
280
+ * pendingRequestCount: 20,
281
+ * }
282
+ * ```
283
+ */
284
+ getInfo(): Promise<RequestQueueInfo | undefined>;
285
+ /**
286
+ * Opens a request queue and returns a promise resolving to an instance
287
+ * of the {@link RequestQueue} class.
288
+ *
289
+ * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
290
+ * The queue is used for deep crawling of websites, where you start with several URLs and then
291
+ * recursively follow links to other pages. The data structure supports both breadth-first
292
+ * and depth-first crawling orders.
293
+ *
294
+ * For more details and code examples, see the {@link RequestQueue} class.
295
+ *
296
+ * @param [queueIdOrName]
297
+ * ID or name of the request queue to be opened. If `null` or `undefined`,
298
+ * the function returns the default request queue associated with the actor run.
299
+ * @param [options] Open Request Queue options.
300
+ */
301
+ static open(queueIdOrName?: string | null): Promise<RequestQueue>;
302
+ }
303
+ export interface RequestQueueOptions {
304
+ id: string;
305
+ name?: string;
306
+ client: StorageClient;
307
+ }
308
+ export interface QueueOperationInfoOptions {
309
+ requestId: string;
310
+ wasAlreadyHandled: boolean;
311
+ }
312
+ //# sourceMappingURL=request_queue.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAQjE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAIxG;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAS7C;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAQ7C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAE/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,qBAAa,YAAY;IACrB,GAAG,+BAAyC;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,aAAa,CAAgC;IACrD,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAC5B,eAAe,EAAE,OAAO,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,EAAE,IAAI,CAAC;QACtB,cAAc,EAAE,IAAI,CAAC;QACrB,kBAAkB,EAAE,OAAO,CAAC;KAC/B,CAAC,GAAG,IAAI,CAAQ;IAIjB,UAAU,eAAa;IAKvB,eAAe,gBAA4D;IAI3E,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAMxB,aAAa;mBACkE,OAAO;YAAM,MAAM;OAC5D;IAEtC;;OAEG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;OAEG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,cAAc,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAgDhI;;;;;;;;;;;OAWG;IACG,WAAW,CACb,YAAY,EAAE,CAAC,OAAO,GAAG,cAAc,CAAC,EAAE,EAC1C,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAwFlC;;;;;OAKG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAoBrD;;;;;;;;;;;;;;;;OAgBG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAyDjD;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IA6B9E;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAuCtH;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAOpC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,kBAAkB,GAAG,IAAI;IASvF;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmFnB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAQnC;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAItD;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;CAK1E;AAED,MAAM,WAAW,mBAAmB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;CAC9B"}