@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/README.md +14 -14
  2. package/autoscaling/autoscaled_pool.d.ts +67 -172
  3. package/autoscaling/autoscaled_pool.js +182 -329
  4. package/autoscaling/concurrency_system.d.ts +269 -0
  5. package/autoscaling/concurrency_system.js +365 -0
  6. package/autoscaling/cpu_load_signal.d.ts +43 -0
  7. package/autoscaling/cpu_load_signal.js +47 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  9. package/autoscaling/event_loop_load_signal.js +60 -0
  10. package/autoscaling/index.d.ts +6 -1
  11. package/autoscaling/index.js +6 -1
  12. package/autoscaling/load_signal.d.ts +100 -0
  13. package/autoscaling/load_signal.js +105 -0
  14. package/autoscaling/memory_load_signal.d.ts +47 -0
  15. package/autoscaling/memory_load_signal.js +106 -0
  16. package/autoscaling/snapshotter.d.ts +58 -163
  17. package/autoscaling/snapshotter.js +45 -263
  18. package/autoscaling/storage_backend_load_signal.d.ts +56 -0
  19. package/autoscaling/storage_backend_load_signal.js +73 -0
  20. package/autoscaling/system_status.d.ts +67 -89
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +3 -2
  29. package/cookie_utils.js +18 -7
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +90 -126
  33. package/crawlers/crawler_commons.js +1 -108
  34. package/crawlers/error_snapshotter.d.ts +2 -5
  35. package/crawlers/error_snapshotter.js +7 -8
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +187 -63
  43. package/crawlers/statistics.js +354 -164
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +60 -153
  47. package/enqueue_links/enqueue_links.js +38 -229
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +49 -30
  51. package/enqueue_links/shared.js +94 -68
  52. package/errors.d.ts +72 -4
  53. package/errors.js +89 -5
  54. package/events/event_manager.d.ts +35 -9
  55. package/events/event_manager.js +12 -13
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +39 -13
  60. package/http.d.ts +9 -0
  61. package/http.js +28 -0
  62. package/index.d.ts +7 -5
  63. package/index.js +8 -4
  64. package/iterables.d.ts +79 -0
  65. package/iterables.js +134 -0
  66. package/log.d.ts +78 -1
  67. package/log.js +106 -1
  68. package/memory-storage/consts.d.ts +4 -0
  69. package/memory-storage/consts.js +4 -0
  70. package/memory-storage/index.d.ts +1 -0
  71. package/memory-storage/index.js +1 -0
  72. package/memory-storage/memory-storage.d.ts +44 -0
  73. package/memory-storage/memory-storage.js +160 -0
  74. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  75. package/memory-storage/resource-clients/common/base-client.js +6 -0
  76. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  77. package/memory-storage/resource-clients/dataset.js +106 -0
  78. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  79. package/memory-storage/resource-clients/key-value-store.js +199 -0
  80. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  81. package/memory-storage/resource-clients/request-queue.js +407 -0
  82. package/memory-storage/utils.d.ts +16 -0
  83. package/memory-storage/utils.js +41 -0
  84. package/owned_or_injected.d.ts +58 -0
  85. package/owned_or_injected.js +98 -0
  86. package/package.json +13 -12
  87. package/proxy_configuration.d.ts +22 -128
  88. package/proxy_configuration.js +32 -144
  89. package/recoverable_state.d.ts +83 -51
  90. package/recoverable_state.js +163 -72
  91. package/request.d.ts +57 -16
  92. package/request.js +130 -69
  93. package/router.d.ts +193 -21
  94. package/router.js +188 -43
  95. package/serialization.d.ts +0 -1
  96. package/serialization.js +15 -15
  97. package/service_locator.d.ts +165 -0
  98. package/service_locator.js +253 -0
  99. package/session_pool/consts.d.ts +1 -2
  100. package/session_pool/consts.js +1 -2
  101. package/session_pool/errors.d.ts +0 -1
  102. package/session_pool/errors.js +0 -1
  103. package/session_pool/fingerprint.d.ts +9 -0
  104. package/session_pool/fingerprint.js +30 -0
  105. package/session_pool/index.d.ts +0 -2
  106. package/session_pool/index.js +0 -2
  107. package/session_pool/session.d.ts +35 -88
  108. package/session_pool/session.js +101 -159
  109. package/session_pool/session_pool.d.ts +74 -91
  110. package/session_pool/session_pool.js +175 -165
  111. package/storages/batched_adds.d.ts +37 -0
  112. package/storages/batched_adds.js +73 -0
  113. package/storages/dataset.d.ts +109 -56
  114. package/storages/dataset.js +283 -149
  115. package/storages/index.d.ts +9 -9
  116. package/storages/index.js +7 -9
  117. package/storages/key_value_store.d.ts +183 -48
  118. package/storages/key_value_store.js +445 -171
  119. package/storages/key_value_store_codec.d.ts +32 -0
  120. package/storages/key_value_store_codec.js +113 -0
  121. package/storages/request_dedup_cache.d.ts +22 -0
  122. package/storages/request_dedup_cache.js +48 -0
  123. package/storages/request_list.d.ts +52 -109
  124. package/storages/request_list.js +183 -152
  125. package/storages/request_loader.d.ts +101 -0
  126. package/storages/request_loader.js +1 -0
  127. package/storages/request_manager.d.ts +33 -0
  128. package/storages/request_manager.js +1 -0
  129. package/storages/request_manager_tandem.d.ts +48 -19
  130. package/storages/request_manager_tandem.js +118 -45
  131. package/storages/request_queue.d.ts +290 -47
  132. package/storages/request_queue.js +762 -216
  133. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +45 -89
  134. package/storages/sitemap_request_loader.js +438 -0
  135. package/storages/storage_instance_manager.d.ts +87 -0
  136. package/storages/storage_instance_manager.js +256 -0
  137. package/storages/storage_stats.d.ts +48 -0
  138. package/storages/storage_stats.js +29 -0
  139. package/storages/throttling_request_manager.d.ts +239 -0
  140. package/storages/throttling_request_manager.js +646 -0
  141. package/storages/transaction.d.ts +252 -0
  142. package/storages/transaction.js +251 -0
  143. package/storages/utils.d.ts +59 -11
  144. package/storages/utils.js +75 -15
  145. package/system-info/cpu-info.d.ts +67 -0
  146. package/system-info/cpu-info.js +216 -0
  147. package/system-info/memory-info.d.ts +31 -0
  148. package/system-info/memory-info.js +115 -0
  149. package/system-info/ps-tree.d.ts +17 -0
  150. package/system-info/ps-tree.js +144 -0
  151. package/system-info/runtime.d.ts +14 -0
  152. package/system-info/runtime.js +80 -0
  153. package/typedefs.d.ts +0 -6
  154. package/typedefs.js +0 -1
  155. package/url.d.ts +9 -0
  156. package/url.js +11 -0
  157. package/validators.d.ts +22 -18
  158. package/validators.js +13 -18
  159. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  160. package/autoscaling/autoscaled_pool.js.map +0 -1
  161. package/autoscaling/index.d.ts.map +0 -1
  162. package/autoscaling/index.js.map +0 -1
  163. package/autoscaling/snapshotter.d.ts.map +0 -1
  164. package/autoscaling/snapshotter.js.map +0 -1
  165. package/autoscaling/system_status.d.ts.map +0 -1
  166. package/autoscaling/system_status.js.map +0 -1
  167. package/configuration.d.ts.map +0 -1
  168. package/configuration.js.map +0 -1
  169. package/cookie_utils.d.ts.map +0 -1
  170. package/cookie_utils.js.map +0 -1
  171. package/crawlers/context_pipeline.d.ts.map +0 -1
  172. package/crawlers/context_pipeline.js.map +0 -1
  173. package/crawlers/crawler_commons.d.ts.map +0 -1
  174. package/crawlers/crawler_commons.js.map +0 -1
  175. package/crawlers/crawler_utils.d.ts +0 -10
  176. package/crawlers/crawler_utils.d.ts.map +0 -1
  177. package/crawlers/crawler_utils.js +0 -12
  178. package/crawlers/crawler_utils.js.map +0 -1
  179. package/crawlers/error_snapshotter.d.ts.map +0 -1
  180. package/crawlers/error_snapshotter.js.map +0 -1
  181. package/crawlers/error_tracker.d.ts.map +0 -1
  182. package/crawlers/error_tracker.js.map +0 -1
  183. package/crawlers/index.d.ts.map +0 -1
  184. package/crawlers/index.js.map +0 -1
  185. package/crawlers/internals/types.d.ts.map +0 -1
  186. package/crawlers/internals/types.js.map +0 -1
  187. package/crawlers/statistics.d.ts.map +0 -1
  188. package/crawlers/statistics.js.map +0 -1
  189. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  190. package/enqueue_links/enqueue_links.js.map +0 -1
  191. package/enqueue_links/index.d.ts.map +0 -1
  192. package/enqueue_links/index.js.map +0 -1
  193. package/enqueue_links/shared.d.ts.map +0 -1
  194. package/enqueue_links/shared.js.map +0 -1
  195. package/errors.d.ts.map +0 -1
  196. package/errors.js.map +0 -1
  197. package/events/event_manager.d.ts.map +0 -1
  198. package/events/event_manager.js.map +0 -1
  199. package/events/index.d.ts.map +0 -1
  200. package/events/index.js.map +0 -1
  201. package/events/local_event_manager.d.ts.map +0 -1
  202. package/events/local_event_manager.js.map +0 -1
  203. package/http_clients/base-http-client.d.ts +0 -140
  204. package/http_clients/base-http-client.d.ts.map +0 -1
  205. package/http_clients/base-http-client.js +0 -40
  206. package/http_clients/base-http-client.js.map +0 -1
  207. package/http_clients/form-data-like.d.ts +0 -67
  208. package/http_clients/form-data-like.d.ts.map +0 -1
  209. package/http_clients/form-data-like.js +0 -5
  210. package/http_clients/form-data-like.js.map +0 -1
  211. package/http_clients/got-scraping-http-client.d.ts +0 -20
  212. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  213. package/http_clients/got-scraping-http-client.js +0 -85
  214. package/http_clients/got-scraping-http-client.js.map +0 -1
  215. package/http_clients/index.d.ts +0 -3
  216. package/http_clients/index.d.ts.map +0 -1
  217. package/http_clients/index.js +0 -3
  218. package/http_clients/index.js.map +0 -1
  219. package/index.d.ts.map +0 -1
  220. package/index.js.map +0 -1
  221. package/log.d.ts.map +0 -1
  222. package/log.js.map +0 -1
  223. package/proxy_configuration.d.ts.map +0 -1
  224. package/proxy_configuration.js.map +0 -1
  225. package/recoverable_state.d.ts.map +0 -1
  226. package/recoverable_state.js.map +0 -1
  227. package/request.d.ts.map +0 -1
  228. package/request.js.map +0 -1
  229. package/router.d.ts.map +0 -1
  230. package/router.js.map +0 -1
  231. package/serialization.d.ts.map +0 -1
  232. package/serialization.js.map +0 -1
  233. package/session_pool/consts.d.ts.map +0 -1
  234. package/session_pool/consts.js.map +0 -1
  235. package/session_pool/errors.d.ts.map +0 -1
  236. package/session_pool/errors.js.map +0 -1
  237. package/session_pool/events.d.ts +0 -3
  238. package/session_pool/events.d.ts.map +0 -1
  239. package/session_pool/events.js +0 -3
  240. package/session_pool/events.js.map +0 -1
  241. package/session_pool/index.d.ts.map +0 -1
  242. package/session_pool/index.js.map +0 -1
  243. package/session_pool/session.d.ts.map +0 -1
  244. package/session_pool/session.js.map +0 -1
  245. package/session_pool/session_pool.d.ts.map +0 -1
  246. package/session_pool/session_pool.js.map +0 -1
  247. package/storages/access_checking.d.ts +0 -13
  248. package/storages/access_checking.d.ts.map +0 -1
  249. package/storages/access_checking.js +0 -14
  250. package/storages/access_checking.js.map +0 -1
  251. package/storages/dataset.d.ts.map +0 -1
  252. package/storages/dataset.js.map +0 -1
  253. package/storages/index.d.ts.map +0 -1
  254. package/storages/index.js.map +0 -1
  255. package/storages/key_value_store.d.ts.map +0 -1
  256. package/storages/key_value_store.js.map +0 -1
  257. package/storages/request_list.d.ts.map +0 -1
  258. package/storages/request_list.js.map +0 -1
  259. package/storages/request_list_adapter.d.ts +0 -58
  260. package/storages/request_list_adapter.d.ts.map +0 -1
  261. package/storages/request_list_adapter.js +0 -81
  262. package/storages/request_list_adapter.js.map +0 -1
  263. package/storages/request_manager_tandem.d.ts.map +0 -1
  264. package/storages/request_manager_tandem.js.map +0 -1
  265. package/storages/request_provider.d.ts +0 -371
  266. package/storages/request_provider.d.ts.map +0 -1
  267. package/storages/request_provider.js +0 -585
  268. package/storages/request_provider.js.map +0 -1
  269. package/storages/request_queue.d.ts.map +0 -1
  270. package/storages/request_queue.js.map +0 -1
  271. package/storages/request_queue_v2.d.ts +0 -87
  272. package/storages/request_queue_v2.d.ts.map +0 -1
  273. package/storages/request_queue_v2.js +0 -438
  274. package/storages/request_queue_v2.js.map +0 -1
  275. package/storages/sitemap_request_list.d.ts.map +0 -1
  276. package/storages/sitemap_request_list.js +0 -430
  277. package/storages/sitemap_request_list.js.map +0 -1
  278. package/storages/storage_manager.d.ts +0 -58
  279. package/storages/storage_manager.d.ts.map +0 -1
  280. package/storages/storage_manager.js +0 -105
  281. package/storages/storage_manager.js.map +0 -1
  282. package/storages/utils.d.ts.map +0 -1
  283. package/storages/utils.js.map +0 -1
  284. package/typedefs.d.ts.map +0 -1
  285. package/typedefs.js.map +0 -1
  286. package/validators.d.ts.map +0 -1
  287. package/validators.js.map +0 -1
@@ -1,371 +0,0 @@
1
- import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
2
- import type { ReadonlyDeep } from 'type-fest';
3
- import { ListDictionary, LruCache } from '@apify/datastructures';
4
- import type { Log } from '@apify/log';
5
- import { Configuration } from '../configuration.js';
6
- import type { ProxyConfiguration } from '../proxy_configuration.js';
7
- import type { InternalSource, RequestOptions, Source } from '../request.js';
8
- import { Request } from '../request.js';
9
- import type { IStorage, StorageManagerOptions } from './storage_manager.js';
10
- export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
11
- /**
12
- * Represents a provider of requests/URLs to crawl.
13
- */
14
- export interface IRequestManager {
15
- /**
16
- * Returns `true` if all requests were already handled and there are no more left.
17
- */
18
- isFinished(): Promise<boolean>;
19
- /**
20
- * Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
21
- * would return `null`, otherwise it resolves to `false`.
22
- * Note that even if the provider is empty, there might be some pending requests currently being processed.
23
- */
24
- isEmpty(): Promise<boolean>;
25
- /**
26
- * Returns number of handled requests.
27
- */
28
- handledCount(): Promise<number>;
29
- /**
30
- * Get the total number of requests known to the request manager.
31
- */
32
- getTotalCount(): number;
33
- /**
34
- * Get an offline approximation of the number of pending requests.
35
- */
36
- getPendingCount(): number;
37
- /**
38
- * Gets the next {@link Request} to process.
39
- *
40
- * The function's `Promise` resolves to `null` if there are no more
41
- * requests to process.
42
- */
43
- fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
44
- /**
45
- * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
46
- * Provides an alternative for the repeated use of `fetchNextRequest`.
47
- */
48
- [Symbol.asyncIterator](): AsyncGenerator<Request>;
49
- /**
50
- * Marks request as handled after successful processing.
51
- */
52
- markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
53
- /**
54
- * Reclaims request to the provider if its processing failed.
55
- * The request will become available in the next `fetchNextRequest()`.
56
- */
57
- reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
58
- addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
59
- addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
60
- }
61
- export declare abstract class RequestProvider implements IStorage, IRequestManager {
62
- readonly config: Configuration;
63
- id: string;
64
- name?: string;
65
- timeoutSecs: number;
66
- clientKey: string;
67
- client: RequestQueueClient;
68
- protected proxyConfiguration?: ProxyConfiguration;
69
- log: Log;
70
- internalTimeoutMillis: number;
71
- requestLockSecs: number;
72
- assumedTotalCount: number;
73
- assumedHandledCount: number;
74
- private initialCount;
75
- private initialHandledCount;
76
- protected queueHeadIds: ListDictionary<string>;
77
- protected requestCache: LruCache<RequestLruItem>;
78
- protected recentlyHandledRequestsCache: LruCache<boolean>;
79
- protected queuePausedForMigration: boolean;
80
- protected lastActivity: Date;
81
- protected isFinishedCalledWhileHeadWasNotEmpty: number;
82
- protected inProgressRequestBatchCount: number;
83
- constructor(options: InternalRequestProviderOptions, config?: Configuration);
84
- /**
85
- * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
86
- *
87
- * Survives restarts and actor migrations.
88
- */
89
- getTotalCount(): number;
90
- /**
91
- * Returns an offline approximation of the total number of pending requests in the queue.
92
- *
93
- * Survives restarts and Actor migrations.
94
- */
95
- getPendingCount(): number;
96
- /**
97
- * Adds a request to the queue.
98
- *
99
- * If a request with the same `uniqueKey` property is already present in the queue,
100
- * it will not be updated. You can find out whether this happened from the resulting
101
- * {@link QueueOperationInfo} object.
102
- *
103
- * To add multiple requests to the queue by extracting links from a webpage,
104
- * see the {@link enqueueLinks} helper function.
105
- *
106
- * @param requestLike {@link Request} object or vanilla object with request data.
107
- * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
108
- * @param [options] Request queue operation options.
109
- */
110
- addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
111
- /**
112
- * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
113
- * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
114
- * if you don't want to block the processing, as those methods will only wait for the initial 1000 requests,
115
- * start processing right after that happens, and continue adding more in the background.
116
- *
117
- * If a request passed in is already present due to its `uniqueKey` property being the same,
118
- * it will not be updated. You can find out whether this happened by finding the request in the resulting
119
- * {@link BatchAddRequestsResult} object.
120
- *
121
- * @param requestsLike {@link Request} objects or vanilla objects with request data.
122
- * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
123
- * @param [options] Request queue operation options.
124
- */
125
- addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
126
- /**
127
- * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
128
- * adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
129
- * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
130
- * the `waitForAllRequestsToBeAdded` promise you get in the response object.
131
- *
132
- * @param requests The requests to add
133
- * @param options Options for the request queue
134
- */
135
- addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
136
- /**
137
- * Gets the request from the queue specified by ID.
138
- *
139
- * @param id ID of the request.
140
- * @returns Returns the request object, or `null` if it was not found.
141
- */
142
- getRequest<T extends Dictionary = Dictionary>(id: string): Promise<Request<T> | null>;
143
- /**
144
- * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
145
- *
146
- * Once you successfully finish processing of the request, you need to call
147
- * {@link RequestQueue.markRequestHandled}
148
- * to mark the request as handled in the queue. If there was some error in processing the request,
149
- * call {@link RequestQueue.reclaimRequest} instead,
150
- * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
151
- *
152
- * Note that the `null` return value doesn't mean the queue processing finished,
153
- * it means there are currently no pending requests.
154
- * To check whether all requests in queue were finished,
155
- * use {@link RequestQueue.isFinished} instead.
156
- *
157
- * @returns
158
- * Returns the request object or `null` if there are no more pending requests.
159
- */
160
- abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
161
- /**
162
- * Marks a request that was previously returned by the
163
- * {@link RequestQueue.fetchNextRequest}
164
- * function as handled after successful processing.
165
- * Handled requests will never again be returned by the `fetchNextRequest` function.
166
- */
167
- markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
168
- /**
169
- * Reclaims a failed request back to the queue, so that it can be returned for processing later again
170
- * by another call to {@link RequestQueue.fetchNextRequest}.
171
- * The request record in the queue is updated using the provided `request` parameter.
172
- * For example, this lets you store the number of retries or error messages for the request.
173
- */
174
- reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
175
- protected abstract ensureHeadIsNonEmpty(): Promise<void>;
176
- /**
177
- * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
178
- * would return `null`, otherwise it resolves to `false`.
179
- * Note that even if the queue is empty, there might be some pending requests currently being processed.
180
- * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
181
- */
182
- isEmpty(): Promise<boolean>;
183
- /**
184
- * Resolves to `true` if all requests were already handled and there are no more left.
185
- * Due to the nature of distributed storage used by the queue,
186
- * the function may occasionally return a false negative,
187
- * but it shall never return a false positive.
188
- */
189
- abstract isFinished(): Promise<boolean>;
190
- protected _reset(): void;
191
- /**
192
- * Caches information about request to beware of unneeded addRequest() calls.
193
- */
194
- protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
195
- /**
196
- * Adds a request straight to the queueHeadDict, to improve performance.
197
- */
198
- protected _maybeAddRequestToQueueHead(requestId: string, forefront: boolean): void;
199
- /**
200
- * Removes the queue either from the Apify Cloud storage or from the local database,
201
- * depending on the mode of operation.
202
- */
203
- drop(): Promise<void>;
204
- /**
205
- * @inheritdoc
206
- */
207
- [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
208
- /**
209
- * Returns the number of handled requests.
210
- *
211
- * This function is just a convenient shortcut for:
212
- *
213
- * ```javascript
214
- * const { handledRequestCount } = await queue.getInfo();
215
- * ```
216
- * @inheritdoc
217
- */
218
- handledCount(): Promise<number>;
219
- /**
220
- * Returns an object containing general information about the request queue.
221
- *
222
- * The function returns the same object as the Apify API Client's
223
- * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
224
- * function, which in turn calls the
225
- * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
226
- * API endpoint.
227
- *
228
- * **Example:**
229
- * ```
230
- * {
231
- * id: "WkzbQMuFYuamGv3YF",
232
- * name: "my-queue",
233
- * userId: "wRsJZtadYvn4mBZmm",
234
- * createdAt: new Date("2015-12-12T07:34:14.202Z"),
235
- * modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
236
- * accessedAt: new Date("2015-12-14T08:36:13.202Z"),
237
- * totalRequestCount: 25,
238
- * handledRequestCount: 5,
239
- * pendingRequestCount: 20,
240
- * }
241
- * ```
242
- */
243
- getInfo(): Promise<RequestQueueInfo | undefined>;
244
- /**
245
- * Fetches URLs from requestsFromUrl and returns them in format of list of requests
246
- */
247
- protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
248
- /**
249
- * Adds all fetched requests from a URL from a remote resource.
250
- */
251
- protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;
252
- /**
253
- * @internal wraps public utility for mocking purposes
254
- */
255
- private _downloadListOfUrls;
256
- /**
257
- * Opens a request queue and returns a promise resolving to an instance
258
- * of the {@link RequestQueue} class.
259
- *
260
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
261
- * The queue is used for deep crawling of websites, where you start with several URLs and then
262
- * recursively follow links to other pages. The data structure supports both breadth-first
263
- * and depth-first crawling orders.
264
- *
265
- * For more details and code examples, see the {@link RequestQueue} class.
266
- *
267
- * @param [queueIdOrName]
268
- * ID or name of the request queue to be opened. If `null` or `undefined`,
269
- * the function returns the default request queue associated with the crawler run.
270
- * @param [options] Open Request Queue options.
271
- */
272
- static open(queueIdOrName?: string | null, options?: StorageManagerOptions): Promise<RequestProvider>;
273
- }
274
- interface RequestLruItem {
275
- uniqueKey: string;
276
- isHandled: boolean;
277
- id: string;
278
- hydrated: Request | null;
279
- lockExpiresAt: number | null;
280
- forefront: boolean;
281
- }
282
- export interface RequestProviderOptions {
283
- id: string;
284
- name?: string;
285
- client: StorageClient;
286
- /**
287
- * Used to pass the proxy configuration for the `requestsFromUrl` objects.
288
- * Takes advantage of the internal address rotation and authentication process.
289
- * If undefined, the `requestsFromUrl` requests will be made without proxy.
290
- */
291
- proxyConfiguration?: ProxyConfiguration;
292
- }
293
- /**
294
- * @deprecated Use {@link RequestProviderOptions} instead.
295
- */
296
- export interface RequestQueueOptions extends RequestProviderOptions {
297
- }
298
- /**
299
- * @internal
300
- */
301
- export interface InternalRequestProviderOptions extends RequestProviderOptions {
302
- logPrefix: string;
303
- requestCacheMaxSize: number;
304
- recentlyHandledRequestsMaxSize: number;
305
- }
306
- export interface RequestQueueOperationOptions {
307
- /**
308
- * If set to `true`:
309
- * - while adding the request to the queue: the request will be added to the foremost position in the queue.
310
- * - while reclaiming the request: the request will be placed to the beginning of the queue, so that it's returned
311
- * in the next call to {@link RequestQueue.fetchNextRequest}.
312
- * By default, it's put to the end of the queue.
313
- *
314
- * In case the request is already present in the queue, this option has no effect.
315
- *
316
- * If more requests are added with this option at once, their order in the following `fetchNextRequest` call
317
- * is arbitrary.
318
- * @default false
319
- */
320
- forefront?: boolean;
321
- /**
322
- * Should the requests be added to the local LRU cache?
323
- * @default false
324
- * @internal
325
- */
326
- cache?: boolean;
327
- }
328
- /**
329
- * @internal
330
- */
331
- export interface RequestQueueOperationInfo extends QueueOperationInfo {
332
- uniqueKey: string;
333
- forefront: boolean;
334
- }
335
- export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {
336
- /**
337
- * Whether to wait for all the provided requests to be added, instead of waiting just for the initial batch of up to `batchSize`.
338
- * @default false
339
- */
340
- waitForAllRequestsToBeAdded?: boolean;
341
- /**
342
- * @default 1000
343
- */
344
- batchSize?: number;
345
- /**
346
- * @default 1000
347
- */
348
- waitBetweenBatchesMillis?: number;
349
- }
350
- export interface AddRequestsBatchedResult {
351
- addedRequests: ProcessedRequest[];
352
- /**
353
- * A promise which will resolve with the rest of the requests that were added to the queue.
354
- *
355
- * Alternatively, we can set {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} to `true`
356
- * in the {@link BasicCrawler.addRequests|`crawler.addRequests()`} options.
357
- *
358
- * **Example:**
359
- *
360
- * ```ts
361
- * // Assuming `requests` is a list of requests.
362
- * const result = await crawler.addRequests(requests);
363
- *
364
- * // If we want to wait for the rest of the requests to be added to the queue:
365
- * await result.waitForAllRequestsToBeAdded;
366
- * ```
367
- */
368
- waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
369
- }
370
- export {};
371
- //# sourceMappingURL=request_provider.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI5E,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAmClE,QAAQ,CAAC,MAAM;IAlCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAMtD;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAQjC;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC;CA6BlH;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}