@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +75 -167
  5. package/configuration.js +164 -140
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +40 -19
  9. package/errors.js +52 -30
  10. package/events/{event_manager.d.ts → event-manager.d.ts} +29 -6
  11. package/events/{event_manager.js → event-manager.js} +10 -9
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/{local_event_manager.d.ts → local-event-manager.d.ts} +3 -10
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -13
  19. package/index.js +10 -12
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +108 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +23 -23
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -101
  50. package/request.js +139 -249
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/{service_locator.d.ts → service-locator.d.ts} +66 -37
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +39 -0
  56. package/storages/batched-adds.js +81 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -151
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +32 -171
  68. package/storages/{request_list.js → request-list.js} +276 -253
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +34 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +127 -161
  76. package/storages/request-queue.js +871 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -17
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -298
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -527
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -179
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -279
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -178
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -168
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -204
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -350
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -254
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -268
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -84
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -219
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js +0 -93
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/proxy_configuration.d.ts +0 -169
  179. package/proxy_configuration.d.ts.map +0 -1
  180. package/proxy_configuration.js +0 -247
  181. package/proxy_configuration.js.map +0 -1
  182. package/recoverable_state.d.ts +0 -122
  183. package/recoverable_state.d.ts.map +0 -1
  184. package/recoverable_state.js +0 -138
  185. package/recoverable_state.js.map +0 -1
  186. package/request.d.ts.map +0 -1
  187. package/request.js.map +0 -1
  188. package/router.d.ts +0 -134
  189. package/router.d.ts.map +0 -1
  190. package/router.js +0 -164
  191. package/router.js.map +0 -1
  192. package/serialization.d.ts.map +0 -1
  193. package/serialization.js.map +0 -1
  194. package/service_locator.d.ts.map +0 -1
  195. package/service_locator.js +0 -252
  196. package/service_locator.js.map +0 -1
  197. package/session_pool/consts.d.ts +0 -4
  198. package/session_pool/consts.d.ts.map +0 -1
  199. package/session_pool/consts.js +0 -4
  200. package/session_pool/consts.js.map +0 -1
  201. package/session_pool/errors.d.ts +0 -8
  202. package/session_pool/errors.d.ts.map +0 -1
  203. package/session_pool/errors.js +0 -12
  204. package/session_pool/errors.js.map +0 -1
  205. package/session_pool/events.d.ts +0 -3
  206. package/session_pool/events.d.ts.map +0 -1
  207. package/session_pool/events.js +0 -3
  208. package/session_pool/events.js.map +0 -1
  209. package/session_pool/index.d.ts +0 -6
  210. package/session_pool/index.d.ts.map +0 -1
  211. package/session_pool/index.js +0 -6
  212. package/session_pool/index.js.map +0 -1
  213. package/session_pool/session.d.ts +0 -186
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js +0 -278
  216. package/session_pool/session.js.map +0 -1
  217. package/session_pool/session_pool.d.ts +0 -256
  218. package/session_pool/session_pool.d.ts.map +0 -1
  219. package/session_pool/session_pool.js +0 -386
  220. package/session_pool/session_pool.js.map +0 -1
  221. package/storages/access_checking.d.ts +0 -13
  222. package/storages/access_checking.d.ts.map +0 -1
  223. package/storages/access_checking.js +0 -14
  224. package/storages/access_checking.js.map +0 -1
  225. package/storages/dataset.d.ts.map +0 -1
  226. package/storages/dataset.js.map +0 -1
  227. package/storages/index.d.ts.map +0 -1
  228. package/storages/index.js.map +0 -1
  229. package/storages/key_value_store.d.ts.map +0 -1
  230. package/storages/key_value_store.js +0 -513
  231. package/storages/key_value_store.js.map +0 -1
  232. package/storages/request_list.d.ts.map +0 -1
  233. package/storages/request_list.js.map +0 -1
  234. package/storages/request_list_adapter.d.ts +0 -58
  235. package/storages/request_list_adapter.d.ts.map +0 -1
  236. package/storages/request_list_adapter.js +0 -81
  237. package/storages/request_list_adapter.js.map +0 -1
  238. package/storages/request_manager_tandem.d.ts +0 -68
  239. package/storages/request_manager_tandem.d.ts.map +0 -1
  240. package/storages/request_manager_tandem.js +0 -124
  241. package/storages/request_manager_tandem.js.map +0 -1
  242. package/storages/request_provider.d.ts.map +0 -1
  243. package/storages/request_provider.js +0 -592
  244. package/storages/request_provider.js.map +0 -1
  245. package/storages/request_queue.d.ts +0 -128
  246. package/storages/request_queue.d.ts.map +0 -1
  247. package/storages/request_queue.js +0 -320
  248. package/storages/request_queue.js.map +0 -1
  249. package/storages/request_queue_v2.d.ts +0 -87
  250. package/storages/request_queue_v2.d.ts.map +0 -1
  251. package/storages/request_queue_v2.js +0 -437
  252. package/storages/request_queue_v2.js.map +0 -1
  253. package/storages/sitemap_request_list.d.ts +0 -245
  254. package/storages/sitemap_request_list.d.ts.map +0 -1
  255. package/storages/sitemap_request_list.js +0 -431
  256. package/storages/sitemap_request_list.js.map +0 -1
  257. package/storages/storage_manager.d.ts +0 -60
  258. package/storages/storage_manager.d.ts.map +0 -1
  259. package/storages/storage_manager.js +0 -95
  260. package/storages/storage_manager.js.map +0 -1
  261. package/storages/utils.d.ts.map +0 -1
  262. package/storages/utils.js.map +0 -1
  263. package/typedefs.d.ts.map +0 -1
  264. package/typedefs.js.map +0 -1
  265. package/validators.d.ts.map +0 -1
  266. package/validators.js.map +0 -1
@@ -0,0 +1,127 @@
1
+ import type { Dictionary } from '@crawlee/types';
2
+ import type { Request } from '../request.js';
3
+ import type { IRequestManager } from './request-manager.js';
4
+ import type { RequestQueueOperationInfo } from './request-queue.js';
5
+ /**
6
+ * A request source's own availability, in a single answer.
7
+ *
8
+ * - `ready` — the next {@link IRequestLoader.fetchNextRequest} is expected to hand something over.
9
+ * - `waiting` — nothing to fetch right now, but the source is not done: requests are in progress, are being
10
+ * added in the background, or are held back until `readyAt`.
11
+ * - `stalled` — the source holds requests it cannot make progress on. Only a manager that paces its own
12
+ * dispatch can reach this; see {@link ThrottlingRequestManager}.
13
+ * - `finished` — everything has been handled.
14
+ */
15
+ export type RequestSourceStatus = {
16
+ status: 'ready';
17
+ } | {
18
+ status: 'waiting';
19
+ /**
20
+ * A `Date.now()` timestamp at which the source expects to become `ready`. Absent when the wait has
21
+ * no clock (an in-progress request, a background add), leaving a consumer to poll.
22
+ */
23
+ readyAt?: number;
24
+ } | {
25
+ status: 'stalled';
26
+ reason: string;
27
+ } | {
28
+ status: 'finished';
29
+ };
30
+ /** Loaders never stall — only a manager that paces its own dispatch can. */
31
+ export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
32
+ status: 'stalled';
33
+ }>;
34
+ /**
35
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
36
+ *
37
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
38
+ * nothing.
39
+ *
40
+ * @internal
41
+ */
42
+ export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourceStatus): RequestSourceStatus;
43
+ /**
44
+ * An abstract interface defining a read-only stream of requests to crawl.
45
+ *
46
+ * Request loaders are used to manage and provide access to a storage of crawling requests.
47
+ *
48
+ * Key responsibilities:
49
+ * - Fetching the next request to be processed.
50
+ * - Marking requests as handled once they are no longer in progress.
51
+ * - Managing state information such as the total and handled request counts.
52
+ *
53
+ * ## Request lifecycle contract
54
+ *
55
+ * Every request returned by {@link IRequestLoader.fetchNextRequest} is considered **in progress**
56
+ * until it is passed to {@link IRequestLoader.markRequestAsHandled}. Once you fetch a request, you are
57
+ * obligated to eventually mark it as handled — there is no way to hand a request back to a loader
58
+ * (only an {@link IRequestManager} can reclaim requests for a retry). "Handled" therefore means
59
+ * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
60
+ *
61
+ * Honoring this contract matters for three reasons:
62
+ * - **Restarts and migrations:** loaders that persist their state (such as {@link RequestList})
63
+ * treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
64
+ * but never marked handled will be crawled again.
65
+ * - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
66
+ * in progress. Leaving a request unmarked keeps the crawler running indefinitely.
67
+ * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
68
+ * skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
69
+ * and {@link IRequestLoader.getPendingCount}.
70
+ *
71
+ * Concrete implementations such as {@link RequestList} or {@link SitemapRequestLoader} build on this interface.
72
+ * The {@link IRequestManager} interface extends it with the capability to enqueue and reclaim requests.
73
+ */
74
+ export interface IRequestLoader {
75
+ /**
76
+ * Returns an approximation of the total number of requests in the loader (i.e. pending + handled).
77
+ */
78
+ getTotalCount(): Promise<number>;
79
+ /**
80
+ * Returns an approximation of the number of pending requests in the loader.
81
+ */
82
+ getPendingCount(): Promise<number>;
83
+ /**
84
+ * Returns the number of requests in the loader that have been handled.
85
+ */
86
+ getHandledCount(): Promise<number>;
87
+ /**
88
+ * Reports whether the loader has a request to hand over, is waiting on one, or is done — see
89
+ * {@link RequestSourceStatus}.
90
+ *
91
+ * A consumer's task loop is gated on this, so implementations MUST answer `ready` before evaluating
92
+ * anything else. `finished` may arrive late behind distributed storage, but it is never wrong.
93
+ */
94
+ checkReadiness(): Promise<RequestSourceStatus>;
95
+ /**
96
+ * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
97
+ *
98
+ * The returned request is marked as **in progress** and remains so until it is passed to
99
+ * {@link IRequestLoader.markRequestAsHandled}. The caller is responsible for eventually marking
100
+ * every fetched request as handled; otherwise the loader never considers itself finished and the
101
+ * request may be re-served after a restart. See the request lifecycle contract on {@link IRequestLoader}.
102
+ */
103
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
104
+ /**
105
+ * Can be used to iterate over the loader instance in a `for await .. of` loop.
106
+ * Provides an alternative for the repeated use of `fetchNextRequest`.
107
+ */
108
+ [Symbol.asyncIterator](): AsyncGenerator<Request>;
109
+ /**
110
+ * Marks a request previously returned by {@link IRequestLoader.fetchNextRequest} as handled,
111
+ * removing it from the set of in-progress requests.
112
+ *
113
+ * Call this once you are done with the request — whether processing succeeded or was abandoned after
114
+ * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
115
+ * signal completion; failing to do so prevents {@link IRequestLoader.checkReadiness} from ever reporting
116
+ * `finished` and skews the handled and pending counts. See the request lifecycle contract on
117
+ * {@link IRequestLoader}.
118
+ */
119
+ markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
120
+ /**
121
+ * Combines the loader with a request manager to support adding and reclaiming requests.
122
+ *
123
+ * @param requestManager Request manager to combine the loader with. If not provided, the default
124
+ * {@link RequestQueue} is used.
125
+ */
126
+ toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
127
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
3
+ *
4
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
5
+ * nothing.
6
+ *
7
+ * @internal
8
+ */
9
+ export function joinRequestSourceStatuses(a, b) {
10
+ if (a.status === 'ready') {
11
+ return a;
12
+ }
13
+ if (b.status === 'ready') {
14
+ return b;
15
+ }
16
+ // `ready` and `waiting` outranking `stalled` mask a stalled source while the other still has work, now or
17
+ // later. The crawler turns `stalled` into a `PersistentRateLimitError`, so "fixing" the masking turns a crawl
18
+ // that can still progress elsewhere into a failure. The stall surfaces once the other source runs out of work.
19
+ if (a.status === 'waiting') {
20
+ // The earlier of the two known wake-up times - unknown only if neither source announced one.
21
+ if (b.status !== 'waiting' || b.readyAt === undefined) {
22
+ return a;
23
+ }
24
+ return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
25
+ }
26
+ if (b.status === 'waiting') {
27
+ return b;
28
+ }
29
+ if (a.status === 'stalled') {
30
+ return a;
31
+ }
32
+ // `a` is finished, so `b` decides.
33
+ return b;
34
+ }
@@ -0,0 +1,82 @@
1
+ import type { Dictionary } from '@crawlee/types';
2
+ import type { Request, Source } from '../request.js';
3
+ import type { IRequestLoader, RequestSourceStatus } from './request-loader.js';
4
+ import type { IRequestManager, PacingSignal, RequestsLike } from './request-manager.js';
5
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request-queue.js';
6
+ /**
7
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
8
+ * {@link IRequestManager} (such as a `RequestQueue`).
9
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
10
+ */
11
+ export declare class RequestManagerTandem implements IRequestManager {
12
+ #private;
13
+ /**
14
+ * @param requestLoader The read-only loader to read requests from first.
15
+ * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
16
+ * factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
17
+ * (e.g. a lazily-opened default {@link RequestQueue}).
18
+ */
19
+ constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
20
+ /**
21
+ * Fetches the next request, transferring one from the loader first if the loader still has work.
22
+ * @inheritdoc
23
+ */
24
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
25
+ /**
26
+ * The loader and the manager read as one source.
27
+ * @inheritdoc
28
+ */
29
+ checkReadiness(): Promise<RequestSourceStatus>;
30
+ /**
31
+ * @inheritdoc
32
+ */
33
+ getHandledCount(): Promise<number>;
34
+ /**
35
+ * @inheritdoc
36
+ */
37
+ getTotalCount(): Promise<number>;
38
+ /**
39
+ * @inheritdoc
40
+ */
41
+ getPendingCount(): Promise<number>;
42
+ /**
43
+ * @inheritdoc
44
+ */
45
+ [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
46
+ /**
47
+ * @inheritdoc
48
+ */
49
+ markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
50
+ /**
51
+ * @inheritdoc
52
+ */
53
+ reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
54
+ /**
55
+ * @inheritdoc
56
+ */
57
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
58
+ /**
59
+ * @inheritdoc
60
+ */
61
+ addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
62
+ /**
63
+ * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
64
+ * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
65
+ * @inheritdoc
66
+ */
67
+ purge(): Promise<void>;
68
+ /**
69
+ * Forwards the hint to the writable request manager — that is where requests are fetched from and
70
+ * reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.
71
+ * @inheritdoc
72
+ */
73
+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
74
+ /**
75
+ * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
76
+ * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
77
+ * pacing.
78
+ * @inheritdoc
79
+ */
80
+ recordPacingSignal(signal: PacingSignal): boolean;
81
+ extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
82
+ }
@@ -0,0 +1,200 @@
1
+ import { serviceLocator } from '../service-locator.js';
2
+ import { joinRequestSourceStatuses } from './request-loader.js';
3
+ /**
4
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
5
+ * {@link IRequestManager} (such as a `RequestQueue`).
6
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
7
+ */
8
+ export class RequestManagerTandem {
9
+ #log;
10
+ #requestLoader;
11
+ #requestManagerPromise;
12
+ #resolvedRequestManager;
13
+ #requestManagerFactory;
14
+ /**
15
+ * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTime}.
16
+ * Remembered so it can be applied to the writable manager once it is lazily resolved.
17
+ */
18
+ #expectedRequestProcessingSecs;
19
+ /**
20
+ * @param requestLoader The read-only loader to read requests from first.
21
+ * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
22
+ * factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
23
+ * (e.g. a lazily-opened default {@link RequestQueue}).
24
+ */
25
+ constructor(requestLoader, requestManager) {
26
+ this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
27
+ this.#requestLoader = requestLoader;
28
+ if (typeof requestManager === 'function') {
29
+ this.#requestManagerFactory = requestManager;
30
+ }
31
+ else {
32
+ // Nothing to open, so mark it resolved up front - synchronous pacing signals can then reach it.
33
+ this.#resolvedRequestManager = requestManager;
34
+ this.#requestManagerFactory = () => requestManager;
35
+ }
36
+ }
37
+ /**
38
+ * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
39
+ */
40
+ async #getRequestManager() {
41
+ if (this.#resolvedRequestManager === undefined) {
42
+ this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
43
+ this.#resolvedRequestManager = await this.#requestManagerPromise;
44
+ // Apply any hint received before the manager was resolved.
45
+ if (this.#expectedRequestProcessingSecs !== undefined) {
46
+ await this.#resolvedRequestManager.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
47
+ }
48
+ }
49
+ return this.#resolvedRequestManager;
50
+ }
51
+ /**
52
+ * Transfers a single request from the read-only loader to the writable manager.
53
+ * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
54
+ *
55
+ * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
56
+ * transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
57
+ */
58
+ async #transferNextRequestToQueue() {
59
+ const request = await this.#requestLoader.fetchNextRequest();
60
+ if (request === null) {
61
+ return true;
62
+ }
63
+ const requestManager = await this.#getRequestManager();
64
+ try {
65
+ await requestManager.addRequest(request, { forefront: true });
66
+ return true;
67
+ }
68
+ catch (error) {
69
+ this.#log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
70
+ return false;
71
+ }
72
+ finally {
73
+ // Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
74
+ await this.#requestLoader.markRequestAsHandled(request);
75
+ }
76
+ }
77
+ /**
78
+ * Fetches the next request, transferring one from the loader first if the loader still has work.
79
+ * @inheritdoc
80
+ */
81
+ async fetchNextRequest() {
82
+ // Only the loader's own state decides this: a manager waiting out a backoff must not freeze the
83
+ // loader's unrelated requests for the length of it.
84
+ if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
85
+ // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
86
+ // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
87
+ if (!(await this.#transferNextRequestToQueue())) {
88
+ return null;
89
+ }
90
+ }
91
+ // Try to fetch from manager after the transfer
92
+ return (await this.#getRequestManager()).fetchNextRequest();
93
+ }
94
+ /**
95
+ * The loader and the manager read as one source.
96
+ * @inheritdoc
97
+ */
98
+ async checkReadiness() {
99
+ const requestManager = await this.#getRequestManager();
100
+ const [loaderStatus, managerStatus] = await Promise.all([
101
+ this.#requestLoader.checkReadiness(),
102
+ requestManager.checkReadiness(),
103
+ ]);
104
+ return joinRequestSourceStatuses(loaderStatus, managerStatus);
105
+ }
106
+ /**
107
+ * @inheritdoc
108
+ */
109
+ async getHandledCount() {
110
+ // Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
111
+ return (await this.#getRequestManager()).getHandledCount();
112
+ }
113
+ /**
114
+ * @inheritdoc
115
+ */
116
+ async getTotalCount() {
117
+ const requestManager = await this.#getRequestManager();
118
+ const [managerTotal, loaderTotal] = await Promise.all([
119
+ requestManager.getTotalCount(),
120
+ // count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
121
+ this.#requestLoader.getPendingCount(),
122
+ ]);
123
+ return managerTotal + loaderTotal;
124
+ }
125
+ /**
126
+ * @inheritdoc
127
+ */
128
+ async getPendingCount() {
129
+ const requestManager = await this.#getRequestManager();
130
+ const [managerPending, loaderPending] = await Promise.all([
131
+ requestManager.getPendingCount(),
132
+ this.#requestLoader.getPendingCount(),
133
+ ]);
134
+ return managerPending + loaderPending;
135
+ }
136
+ /**
137
+ * @inheritdoc
138
+ */
139
+ async *[Symbol.asyncIterator]() {
140
+ while (true) {
141
+ const req = await this.fetchNextRequest();
142
+ if (!req)
143
+ break;
144
+ yield req;
145
+ }
146
+ }
147
+ /**
148
+ * @inheritdoc
149
+ */
150
+ async markRequestAsHandled(request) {
151
+ return (await this.#getRequestManager()).markRequestAsHandled(request);
152
+ }
153
+ /**
154
+ * @inheritdoc
155
+ */
156
+ async reclaimRequest(request, options) {
157
+ return (await this.#getRequestManager()).reclaimRequest(request, options);
158
+ }
159
+ /**
160
+ * @inheritdoc
161
+ */
162
+ async addRequest(requestLike, options) {
163
+ return (await this.#getRequestManager()).addRequest(requestLike, options);
164
+ }
165
+ /**
166
+ * @inheritdoc
167
+ */
168
+ async addRequestsBatched(requests, options) {
169
+ return (await this.#getRequestManager()).addRequestsBatched(requests, options);
170
+ }
171
+ /**
172
+ * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
173
+ * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
174
+ * @inheritdoc
175
+ */
176
+ async purge() {
177
+ await (await this.#getRequestManager()).purge?.();
178
+ }
179
+ /**
180
+ * Forwards the hint to the writable request manager — that is where requests are fetched from and
181
+ * reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.
182
+ * @inheritdoc
183
+ */
184
+ async setExpectedRequestProcessingTime(processingTimeSecs) {
185
+ this.#expectedRequestProcessingSecs = processingTimeSecs;
186
+ await this.#resolvedRequestManager?.setExpectedRequestProcessingTime?.(processingTimeSecs);
187
+ }
188
+ /**
189
+ * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
190
+ * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
191
+ * pacing.
192
+ * @inheritdoc
193
+ */
194
+ recordPacingSignal(signal) {
195
+ return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
196
+ }
197
+ async extendRequestProcessingTime(request, extensionSecs) {
198
+ return (await this.#getRequestManager()).extendRequestProcessingTime?.(request, extensionSecs) ?? false;
199
+ }
200
+ }
@@ -0,0 +1,119 @@
1
+ import type { LiteralUnion } from 'type-fest';
2
+ import type { Request, Source } from '../request.js';
3
+ import type { IRequestLoader } from './request-loader.js';
4
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request-queue.js';
5
+ export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
6
+ /**
7
+ * Extends the read-only {@link IRequestLoader} interface with the capability to enqueue new requests
8
+ * and reclaim failed ones.
9
+ */
10
+ export interface IRequestManager extends IRequestLoader {
11
+ /**
12
+ * Reclaims request to the provider if its processing failed.
13
+ * The request will be returned by some subsequent `fetchNextRequest()` call.
14
+ */
15
+ reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
16
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
17
+ addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
18
+ /**
19
+ * Remove all requests from the queue but keep the queue itself, resetting it
20
+ * so it can be reused (e.g. across multiple `crawler.run()` calls).
21
+ *
22
+ * Implementations that do not support purging may leave this `undefined`.
23
+ */
24
+ purge?(): Promise<void>;
25
+ /**
26
+ * Tells the manager how long a consumer expects to hold a request fetched via `fetchNextRequest()`
27
+ * before marking it handled or reclaiming it (typically the request-handler timeout plus padding).
28
+ *
29
+ * Managers backed by a storage backend that reserves requests via locking use this to avoid handing
30
+ * the same request out again while it is still being processed. Implementations that do not need
31
+ * this hint may leave it `undefined`.
32
+ */
33
+ setExpectedRequestProcessingTime?(processingTimeSecs: number): Promise<void>;
34
+ /**
35
+ * Records something said about the pace requests should go out at, so that a manager which paces its own
36
+ * dispatch can hold requests back.
37
+ *
38
+ * Required rather than optional, so that a wrapping manager always forwards it and a pacer nested in a
39
+ * composition still receives it; a manager that does not pace returns `false`.
40
+ *
41
+ * @returns `true` if anything in the composition took responsibility for the signal.
42
+ */
43
+ recordPacingSignal(signal: PacingSignal): boolean;
44
+ /**
45
+ * Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
46
+ * processed, on storage backends that reserve requests via locking (e.g. via
47
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`}).
48
+ *
49
+ * @returns `true` when the lock was prolonged, `false` when this manager does not lock
50
+ * requests or no longer holds this one; non-locking implementations may leave it `undefined`,
51
+ * which callers treat as `false`.
52
+ */
53
+ extendRequestProcessingTime?(request: Request, extensionSecs: number): Promise<boolean>;
54
+ }
55
+ /**
56
+ * How much of the URL space a {@link PacingSignal} covers.
57
+ *
58
+ * Open on purpose: `'hostname'` and `'registrableDomain'` are what Crawlee's own reporters send and what
59
+ * {@link ThrottlingRequestManager} understands, but any string is accepted, so a pacer keyed on something
60
+ * else can be reported to in its own vocabulary.
61
+ */
62
+ export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;
63
+ /**
64
+ * Something said about the pace requests should go out at, reported to a request manager through
65
+ * {@link IRequestManager.recordPacingSignal}.
66
+ *
67
+ * One shape rather than a method per channel: a pacing manager switches on `reason`, a wrapping one forwards the
68
+ * value without knowing what is in it, and a new kind of signal costs the interface nothing. The `url` travels
69
+ * inside the value because the crawl-wide variant has none. Nothing here names the mechanism a signal came
70
+ * from - status codes, headers and robots.txt are the crawler's business - and every delay is in milliseconds.
71
+ *
72
+ * ## Scope
73
+ *
74
+ * A manager may apply a signal to a **wider** scope than it was given - a floor that holds for one host still
75
+ * holds when a whole site is paced by it - but never to a narrower one, which would leave some of the URLs the
76
+ * signal covers running unpaced. A manager that can only do the latter, or that does not recognise the scope at
77
+ * all, MUST throw rather than quietly under-apply it.
78
+ */
79
+ export type PacingSignal = {
80
+ /**
81
+ * The source turned a request away because we were going too fast — an HTTP 429 or 503, an exhausted
82
+ * quota. Reactive and transient: a pacer typically backs off while refusals continue, and lets that
83
+ * decay once they stop.
84
+ */
85
+ reason: 'rateLimited';
86
+ /** The URL that was turned away. */
87
+ url: string;
88
+ /** How long the source asked us to wait before trying again, if it said. */
89
+ waitMs?: number;
90
+ /**
91
+ * How far this refusal reaches, if the reporter can tell — a 429 rarely says. Left out, it asks the
92
+ * manager to apply the signal however it happens to group requests.
93
+ */
94
+ scope?: PacingScope;
95
+ } | {
96
+ /**
97
+ * The source declared a standing floor on how often it may be requested — a robots.txt `Crawl-delay`,
98
+ * a documented quota. A property of the source rather than of the run, so a pacer keeps it for the
99
+ * whole crawl.
100
+ */
101
+ reason: 'minInterval';
102
+ /** A URL of the source that declared the interval. */
103
+ url: string;
104
+ /** The declared minimum interval between two requests to the source. */
105
+ intervalMs: number;
106
+ /** Required, since whoever declares an interval knows what it applies to. */
107
+ scope: PacingScope;
108
+ } | {
109
+ /**
110
+ * A standing floor under the pace of **every** domain the manager dispatches to, declared by whoever
111
+ * owns the crawl rather than by a source — a crawler's `sameDomainDelaySecs`. A manager that paces only
112
+ * some of its domains MUST throw rather than under-apply it.
113
+ */
114
+ reason: 'minIntervalEverywhere';
115
+ /** The declared minimum interval between two requests to any one source. */
116
+ intervalMs: number;
117
+ /** At what granularity the floor applies. */
118
+ scope: PacingScope;
119
+ };
@@ -0,0 +1 @@
1
+ export {};