@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +1 -1
  2. package/configuration.d.ts +16 -47
  3. package/configuration.js +13 -25
  4. package/debug.js +4 -4
  5. package/errors.d.ts +28 -38
  6. package/errors.js +33 -47
  7. package/events/event_manager.d.ts +2 -2
  8. package/events/event_manager.js +7 -6
  9. package/events/index.d.ts +1 -0
  10. package/events/local_event_manager.d.ts +1 -8
  11. package/events/local_event_manager.js +13 -13
  12. package/events/system_info.d.ts +38 -0
  13. package/index.d.ts +2 -8
  14. package/index.js +4 -8
  15. package/internal.d.ts +8 -0
  16. package/internal.js +9 -0
  17. package/log.d.ts +10 -11
  18. package/log.js +52 -20
  19. package/memory-storage/memory-storage.d.ts +15 -18
  20. package/memory-storage/memory-storage.js +80 -58
  21. package/memory-storage/resource-clients/dataset.d.ts +1 -6
  22. package/memory-storage/resource-clients/dataset.js +23 -31
  23. package/memory-storage/resource-clients/key-value-store.d.ts +1 -10
  24. package/memory-storage/resource-clients/key-value-store.js +43 -67
  25. package/memory-storage/resource-clients/request-queue.d.ts +1 -42
  26. package/memory-storage/resource-clients/request-queue.js +109 -117
  27. package/owned_or_injected.d.ts +1 -3
  28. package/owned_or_injected.js +17 -17
  29. package/package.json +17 -20
  30. package/proxy_configuration.d.ts +21 -26
  31. package/proxy_configuration.js +35 -25
  32. package/recoverable_state.d.ts +104 -47
  33. package/recoverable_state.js +199 -74
  34. package/request.d.ts +20 -107
  35. package/request.js +78 -244
  36. package/serialization.js +17 -16
  37. package/service_locator.d.ts +22 -10
  38. package/service_locator.js +59 -48
  39. package/storages/batched_adds.d.ts +37 -0
  40. package/storages/batched_adds.js +73 -0
  41. package/storages/dataset.d.ts +13 -8
  42. package/storages/dataset.js +149 -40
  43. package/storages/index.d.ts +4 -4
  44. package/storages/index.js +2 -4
  45. package/storages/key_value_store.d.ts +16 -35
  46. package/storages/key_value_store.js +223 -110
  47. package/storages/key_value_store_codec.js +6 -11
  48. package/storages/request_dedup_cache.d.ts +1 -4
  49. package/storages/request_dedup_cache.js +15 -15
  50. package/storages/request_list.d.ts +9 -104
  51. package/storages/request_list.js +236 -233
  52. package/storages/request_loader.d.ts +49 -18
  53. package/storages/request_loader.js +36 -1
  54. package/storages/request_manager.d.ts +86 -0
  55. package/storages/request_manager_tandem.d.ts +14 -38
  56. package/storages/request_manager_tandem.js +67 -64
  57. package/storages/request_queue.d.ts +23 -50
  58. package/storages/request_queue.js +371 -226
  59. package/storages/storage_instance_manager.d.ts +2 -4
  60. package/storages/storage_instance_manager.js +21 -21
  61. package/storages/storage_stats.d.ts +1 -1
  62. package/storages/storage_stats.js +4 -4
  63. package/storages/transaction.d.ts +270 -0
  64. package/storages/transaction.js +296 -0
  65. package/storages/utils.d.ts +6 -3
  66. package/storages/utils.js +11 -2
  67. package/system-info/runtime.js +7 -7
  68. package/url.d.ts +9 -0
  69. package/url.js +11 -0
  70. package/validators.d.ts +23 -25
  71. package/validators.js +14 -25
  72. package/autoscaling/autoscaled_pool.d.ts +0 -213
  73. package/autoscaling/autoscaled_pool.js +0 -378
  74. package/autoscaling/client_load_signal.d.ts +0 -59
  75. package/autoscaling/client_load_signal.js +0 -73
  76. package/autoscaling/concurrency_system.d.ts +0 -283
  77. package/autoscaling/concurrency_system.js +0 -350
  78. package/autoscaling/cpu_load_signal.d.ts +0 -44
  79. package/autoscaling/cpu_load_signal.js +0 -46
  80. package/autoscaling/event_loop_load_signal.d.ts +0 -54
  81. package/autoscaling/event_loop_load_signal.js +0 -60
  82. package/autoscaling/index.d.ts +0 -9
  83. package/autoscaling/index.js +0 -9
  84. package/autoscaling/load_signal.d.ts +0 -99
  85. package/autoscaling/load_signal.js +0 -103
  86. package/autoscaling/memory_load_signal.d.ts +0 -56
  87. package/autoscaling/memory_load_signal.js +0 -106
  88. package/autoscaling/snapshotter.d.ts +0 -87
  89. package/autoscaling/snapshotter.js +0 -67
  90. package/autoscaling/system_status.d.ts +0 -161
  91. package/autoscaling/system_status.js +0 -139
  92. package/autoscaling/weighted_avg.d.ts +0 -5
  93. package/autoscaling/weighted_avg.js +0 -14
  94. package/cookie_utils.d.ts +0 -44
  95. package/cookie_utils.js +0 -122
  96. package/crawlers/context_pipeline.d.ts +0 -70
  97. package/crawlers/context_pipeline.js +0 -122
  98. package/crawlers/crawler_commons.d.ts +0 -257
  99. package/crawlers/crawler_commons.js +0 -107
  100. package/crawlers/error_snapshotter.d.ts +0 -59
  101. package/crawlers/error_snapshotter.js +0 -117
  102. package/crawlers/error_tracker.d.ts +0 -54
  103. package/crawlers/error_tracker.js +0 -308
  104. package/crawlers/index.d.ts +0 -5
  105. package/crawlers/index.js +0 -5
  106. package/crawlers/internals/types.d.ts +0 -7
  107. package/crawlers/statistics.d.ts +0 -209
  108. package/crawlers/statistics.js +0 -350
  109. package/enqueue_links/enqueue_links.d.ts +0 -264
  110. package/enqueue_links/enqueue_links.js +0 -271
  111. package/enqueue_links/index.d.ts +0 -2
  112. package/enqueue_links/index.js +0 -2
  113. package/enqueue_links/shared.d.ts +0 -83
  114. package/enqueue_links/shared.js +0 -221
  115. package/router.d.ts +0 -309
  116. package/router.js +0 -309
  117. package/session_pool/consts.d.ts +0 -3
  118. package/session_pool/consts.js +0 -3
  119. package/session_pool/errors.d.ts +0 -7
  120. package/session_pool/errors.js +0 -11
  121. package/session_pool/fingerprint.d.ts +0 -9
  122. package/session_pool/fingerprint.js +0 -30
  123. package/session_pool/index.d.ts +0 -4
  124. package/session_pool/index.js +0 -4
  125. package/session_pool/session.d.ts +0 -161
  126. package/session_pool/session.js +0 -218
  127. package/session_pool/session_pool.d.ts +0 -246
  128. package/session_pool/session_pool.js +0 -386
  129. package/storages/access_checking.d.ts +0 -12
  130. package/storages/access_checking.js +0 -17
  131. package/storages/sitemap_request_loader.d.ts +0 -249
  132. package/storages/sitemap_request_loader.js +0 -432
  133. /package/{crawlers/internals/types.js → events/system_info.js} +0 -0
@@ -2,6 +2,44 @@ import type { Dictionary } from '@crawlee/types';
2
2
  import type { Request } from '../request.js';
3
3
  import type { IRequestManager } from './request_manager.js';
4
4
  import type { RequestQueueOperationInfo } from './request_queue.js';
5
+ /**
6
+ * A request source's own availability, in a single answer.
7
+ *
8
+ * - `ready` — the next {@link IRequestLoader.fetchNextRequest} is expected to hand something over.
9
+ * - `waiting` — nothing to fetch right now, but the source is not done: requests are in progress, are being
10
+ * added in the background, or are held back until `readyAt`.
11
+ * - `stalled` — the source holds requests it cannot make progress on. Only a manager that paces its own
12
+ * dispatch can reach this; see {@link ThrottlingRequestManager}.
13
+ * - `finished` — everything has been handled.
14
+ */
15
+ export type RequestSourceStatus = {
16
+ status: 'ready';
17
+ } | {
18
+ status: 'waiting';
19
+ /**
20
+ * A `Date.now()` timestamp at which the source expects to become `ready`. Absent when the wait has
21
+ * no clock (an in-progress request, a background add), leaving a consumer to poll.
22
+ */
23
+ readyAt?: number;
24
+ } | {
25
+ status: 'stalled';
26
+ reason: string;
27
+ } | {
28
+ status: 'finished';
29
+ };
30
+ /** Loaders never stall — only a manager that paces its own dispatch can. */
31
+ export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
32
+ status: 'stalled';
33
+ }>;
34
+ /**
35
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
36
+ *
37
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
38
+ * nothing.
39
+ *
40
+ * @internal
41
+ */
42
+ export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourceStatus): RequestSourceStatus;
5
43
  /**
6
44
  * An abstract interface defining a read-only stream of requests to crawl.
7
45
  *
@@ -21,10 +59,10 @@ import type { RequestQueueOperationInfo } from './request_queue.js';
21
59
  * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
22
60
  *
23
61
  * Honoring this contract matters for three reasons:
24
- * - **Restarts and migrations:** loaders that persist their state (see {@link IRequestLoader.persistState})
62
+ * - **Restarts and migrations:** loaders that persist their state (such as {@link RequestList})
25
63
  * treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
26
64
  * but never marked handled will be crawled again.
27
- * - **Termination detection:** {@link IRequestLoader.isFinished} only resolves to `true` once nothing is
65
+ * - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
28
66
  * in progress. Leaving a request unmarked keeps the crawler running indefinitely.
29
67
  * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
30
68
  * skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
@@ -47,15 +85,13 @@ export interface IRequestLoader {
47
85
  */
48
86
  getHandledCount(): Promise<number>;
49
87
  /**
50
- * Returns `true` if all requests were already handled and there are no more left.
51
- */
52
- isFinished(): Promise<boolean>;
53
- /**
54
- * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
55
- * would return `null`, otherwise it resolves to `false`.
56
- * Note that even if the loader is empty, there might be some pending requests currently being processed.
88
+ * Reports whether the loader has a request to hand over, is waiting on one, or is done — see
89
+ * {@link RequestSourceStatus}.
90
+ *
91
+ * A consumer's task loop is gated on this, so implementations MUST answer `ready` before evaluating
92
+ * anything else. `finished` may arrive late behind distributed storage, but it is never wrong.
57
93
  */
58
- isEmpty(): Promise<boolean>;
94
+ checkReadiness(): Promise<RequestSourceStatus>;
59
95
  /**
60
96
  * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
61
97
  *
@@ -76,16 +112,11 @@ export interface IRequestLoader {
76
112
  *
77
113
  * Call this once you are done with the request — whether processing succeeded or was abandoned after
78
114
  * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
79
- * signal completion; failing to do so prevents {@link IRequestLoader.isFinished} from ever resolving to
80
- * `true` and skews the handled and pending counts. See the request lifecycle contract on {@link IRequestLoader}.
115
+ * signal completion; failing to do so prevents {@link IRequestLoader.checkReadiness} from ever reporting
116
+ * `finished` and skews the handled and pending counts. See the request lifecycle contract on
117
+ * {@link IRequestLoader}.
81
118
  */
82
119
  markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
83
- /**
84
- * Persists the current state of the loader into the default {@link KeyValueStore}.
85
- *
86
- * Not all loaders support persistence; implementations that do not should leave this `undefined`.
87
- */
88
- persistState?(): Promise<void>;
89
120
  /**
90
121
  * Combines the loader with a request manager to support adding and reclaiming requests.
91
122
  *
@@ -1 +1,36 @@
1
- export {};
1
+ /**
2
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
3
+ *
4
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
5
+ * nothing.
6
+ *
7
+ * @internal
8
+ */
9
+ export function joinRequestSourceStatuses(a, b) {
10
+ if (a.status === 'ready') {
11
+ return a;
12
+ }
13
+ if (b.status === 'ready') {
14
+ return b;
15
+ }
16
+ // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
17
+ // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
18
+ // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
19
+ // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
20
+ // `waiting` is the same parity - that call site fired regardless of other domains' clocks.
21
+ if (a.status === 'stalled') {
22
+ return a;
23
+ }
24
+ if (b.status === 'stalled') {
25
+ return b;
26
+ }
27
+ if (a.status === 'waiting') {
28
+ // The earlier of the two known wake-up times - unknown only if neither source announced one.
29
+ if (b.status !== 'waiting' || b.readyAt === undefined) {
30
+ return a;
31
+ }
32
+ return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
33
+ }
34
+ // `a` is finished, so `b` decides.
35
+ return b;
36
+ }
@@ -1,3 +1,4 @@
1
+ import type { LiteralUnion } from 'type-fest';
1
2
  import type { Request, Source } from '../request.js';
2
3
  import type { IRequestLoader } from './request_loader.js';
3
4
  import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
@@ -30,4 +31,89 @@ export interface IRequestManager extends IRequestLoader {
30
31
  * this hint may leave it `undefined`.
31
32
  */
32
33
  setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
34
+ /**
35
+ * Records something said about the pace requests should go out at, so that a manager which paces its own
36
+ * dispatch can hold requests back.
37
+ *
38
+ * Required rather than optional, so that a wrapping manager always forwards it and a pacer nested in a
39
+ * composition still receives it; a manager that does not pace returns `false`.
40
+ *
41
+ * @returns `true` if anything in the composition took responsibility for the signal.
42
+ */
43
+ recordPacingSignal(signal: PacingSignal): boolean;
44
+ /**
45
+ * Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
46
+ * processed, on storage backends that reserve requests via locking (e.g. via
47
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`}).
48
+ *
49
+ * @returns `true` when the lock was prolonged, `false` when this manager does not lock
50
+ * requests or no longer holds this one; non-locking implementations may leave it `undefined`,
51
+ * which callers treat as `false`.
52
+ */
53
+ extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;
33
54
  }
55
+ /**
56
+ * How much of the URL space a {@link PacingSignal} covers.
57
+ *
58
+ * Open on purpose: `'hostname'` and `'registrableDomain'` are what Crawlee's own reporters send and what
59
+ * {@link ThrottlingRequestManager} understands, but any string is accepted, so a pacer keyed on something
60
+ * else can be reported to in its own vocabulary.
61
+ */
62
+ export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;
63
+ /**
64
+ * Something said about the pace requests should go out at, reported to a request manager through
65
+ * {@link IRequestManager.recordPacingSignal}.
66
+ *
67
+ * One shape rather than a method per channel: a pacing manager switches on `reason`, a wrapping one forwards the
68
+ * value without knowing what is in it, and a new kind of signal costs the interface nothing. The `url` travels
69
+ * inside the value because the crawl-wide variant has none. Nothing here names the mechanism a signal came
70
+ * from - status codes, headers and robots.txt are the crawler's business - and every delay is in milliseconds.
71
+ *
72
+ * ## Scope
73
+ *
74
+ * A manager may apply a signal to a **wider** scope than it was given - a floor that holds for one host still
75
+ * holds when a whole site is paced by it - but never to a narrower one, which would leave some of the URLs the
76
+ * signal covers running unpaced. A manager that can only do the latter, or that does not recognise the scope at
77
+ * all, MUST throw rather than quietly under-apply it.
78
+ */
79
+ export type PacingSignal = {
80
+ /**
81
+ * The source turned a request away because we were going too fast — an HTTP 429 or 503, an exhausted
82
+ * quota. Reactive and transient: a pacer typically backs off while refusals continue, and lets that
83
+ * decay once they stop.
84
+ */
85
+ reason: 'rateLimited';
86
+ /** The URL that was turned away. */
87
+ url: string;
88
+ /** How long the source asked us to wait before trying again, if it said. */
89
+ waitMs?: number;
90
+ /**
91
+ * How far this refusal reaches, if the reporter can tell — a 429 rarely says. Left out, it asks the
92
+ * manager to apply the signal however it happens to group requests.
93
+ */
94
+ scope?: PacingScope;
95
+ } | {
96
+ /**
97
+ * The source declared a standing floor on how often it may be requested — a robots.txt `Crawl-delay`,
98
+ * a documented quota. A property of the source rather than of the run, so a pacer keeps it for the
99
+ * whole crawl.
100
+ */
101
+ reason: 'minInterval';
102
+ /** A URL of the source that declared the interval. */
103
+ url: string;
104
+ /** The declared minimum interval between two requests to the source. */
105
+ intervalMs: number;
106
+ /** Required, since whoever declares an interval knows what it applies to. */
107
+ scope: PacingScope;
108
+ } | {
109
+ /**
110
+ * A standing floor under the pace of **every** domain the manager dispatches to, declared by whoever
111
+ * owns the crawl rather than by a source — a crawler's `sameDomainDelaySecs`. A manager that paces only
112
+ * some of its domains MUST throw rather than under-apply it.
113
+ */
114
+ reason: 'minIntervalEverywhere';
115
+ /** The declared minimum interval between two requests to any one source. */
116
+ intervalMs: number;
117
+ /** At what granularity the floor applies. */
118
+ scope: PacingScope;
119
+ };
@@ -1,7 +1,7 @@
1
1
  import type { Dictionary } from '@crawlee/types';
2
2
  import type { Request, Source } from '../request.js';
3
- import type { IRequestLoader } from './request_loader.js';
4
- import type { IRequestManager, RequestsLike } from './request_manager.js';
3
+ import type { IRequestLoader, RequestSourceStatus } from './request_loader.js';
4
+ import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
5
5
  import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
6
6
  /**
7
7
  * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
@@ -9,16 +9,7 @@ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueO
9
9
  * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
10
10
  */
11
11
  export declare class RequestManagerTandem implements IRequestManager {
12
- private log;
13
- private requestLoader;
14
- private requestManagerPromise?;
15
- private resolvedRequestManager?;
16
- private requestManagerFactory;
17
- /**
18
- * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
19
- * Remembered so it can be applied to the writable manager once it is lazily resolved.
20
- */
21
- private expectedRequestProcessingSecs?;
12
+ #private;
22
13
  /**
23
14
  * @param requestLoader The read-only loader to read requests from first.
24
15
  * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
@@ -27,33 +18,15 @@ export declare class RequestManagerTandem implements IRequestManager {
27
18
  */
28
19
  constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
29
20
  /**
30
- * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
31
- * @private
32
- */
33
- private getRequestManager;
34
- /**
35
- * Transfers a single request from the read-only loader to the writable manager.
36
- * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
37
- *
38
- * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
39
- * transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
40
- * @private
41
- */
42
- private transferNextRequestToQueue;
43
- /**
44
- * Fetches the next request from the request manager. If the manager is empty and the loader
45
- * is not finished, it will transfer a request from the loader to the manager first.
21
+ * Fetches the next request, transferring one from the loader first if the loader still has work.
46
22
  * @inheritdoc
47
23
  */
48
24
  fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
49
25
  /**
26
+ * The loader and the manager read as one source.
50
27
  * @inheritdoc
51
28
  */
52
- isFinished(): Promise<boolean>;
53
- /**
54
- * @inheritdoc
55
- */
56
- isEmpty(): Promise<boolean>;
29
+ checkReadiness(): Promise<RequestSourceStatus>;
57
30
  /**
58
31
  * @inheritdoc
59
32
  */
@@ -86,11 +59,6 @@ export declare class RequestManagerTandem implements IRequestManager {
86
59
  * @inheritdoc
87
60
  */
88
61
  addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
89
- /**
90
- * Persists the state of the underlying read-only loader, if it supports persistence.
91
- * @inheritdoc
92
- */
93
- persistState(): Promise<void>;
94
62
  /**
95
63
  * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
96
64
  * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
@@ -103,4 +71,12 @@ export declare class RequestManagerTandem implements IRequestManager {
103
71
  * @inheritdoc
104
72
  */
105
73
  setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
74
+ /**
75
+ * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
76
+ * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
77
+ * pacing.
78
+ * @inheritdoc
79
+ */
80
+ recordPacingSignal(signal: PacingSignal): boolean;
81
+ extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
106
82
  }
@@ -1,20 +1,21 @@
1
1
  import { serviceLocator } from '../service_locator.js';
2
+ import { joinRequestSourceStatuses } from './request_loader.js';
2
3
  /**
3
4
  * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
4
5
  * {@link IRequestManager} (such as a `RequestQueue`).
5
6
  * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
6
7
  */
7
8
  export class RequestManagerTandem {
8
- log;
9
- requestLoader;
10
- requestManagerPromise;
11
- resolvedRequestManager;
12
- requestManagerFactory;
9
+ #log;
10
+ #requestLoader;
11
+ #requestManagerPromise;
12
+ #resolvedRequestManager;
13
+ #requestManagerFactory;
13
14
  /**
14
15
  * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
15
16
  * Remembered so it can be applied to the writable manager once it is lazily resolved.
16
17
  */
17
- expectedRequestProcessingSecs;
18
+ #expectedRequestProcessingSecs;
18
19
  /**
19
20
  * @param requestLoader The read-only loader to read requests from first.
20
21
  * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
@@ -22,24 +23,30 @@ export class RequestManagerTandem {
22
23
  * (e.g. a lazily-opened default {@link RequestQueue}).
23
24
  */
24
25
  constructor(requestLoader, requestManager) {
25
- this.log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
26
- this.requestLoader = requestLoader;
27
- this.requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;
26
+ this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
27
+ this.#requestLoader = requestLoader;
28
+ if (typeof requestManager === 'function') {
29
+ this.#requestManagerFactory = requestManager;
30
+ }
31
+ else {
32
+ // Nothing to open, so mark it resolved up front - synchronous pacing signals can then reach it.
33
+ this.#resolvedRequestManager = requestManager;
34
+ this.#requestManagerFactory = () => requestManager;
35
+ }
28
36
  }
29
37
  /**
30
38
  * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
31
- * @private
32
39
  */
33
- async getRequestManager() {
34
- if (this.resolvedRequestManager === undefined) {
35
- this.requestManagerPromise ??= Promise.resolve(this.requestManagerFactory());
36
- this.resolvedRequestManager = await this.requestManagerPromise;
40
+ async #getRequestManager() {
41
+ if (this.#resolvedRequestManager === undefined) {
42
+ this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
43
+ this.#resolvedRequestManager = await this.#requestManagerPromise;
37
44
  // Apply any hint received before the manager was resolved.
38
- if (this.expectedRequestProcessingSecs !== undefined) {
39
- await this.resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.expectedRequestProcessingSecs);
45
+ if (this.#expectedRequestProcessingSecs !== undefined) {
46
+ await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
40
47
  }
41
48
  }
42
- return this.resolvedRequestManager;
49
+ return this.#resolvedRequestManager;
43
50
  }
44
51
  /**
45
52
  * Transfers a single request from the read-only loader to the writable manager.
@@ -47,80 +54,71 @@ export class RequestManagerTandem {
47
54
  *
48
55
  * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
49
56
  * transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
50
- * @private
51
57
  */
52
- async transferNextRequestToQueue() {
53
- const request = await this.requestLoader.fetchNextRequest();
58
+ async #transferNextRequestToQueue() {
59
+ const request = await this.#requestLoader.fetchNextRequest();
54
60
  if (request === null) {
55
61
  return true;
56
62
  }
57
- const requestManager = await this.getRequestManager();
63
+ const requestManager = await this.#getRequestManager();
58
64
  try {
59
65
  await requestManager.addRequest(request, { forefront: true });
60
66
  return true;
61
67
  }
62
68
  catch (error) {
63
- this.log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
69
+ this.#log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
64
70
  return false;
65
71
  }
66
72
  finally {
67
73
  // Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
68
- await this.requestLoader.markRequestAsHandled(request);
74
+ await this.#requestLoader.markRequestAsHandled(request);
69
75
  }
70
76
  }
71
77
  /**
72
- * Fetches the next request from the request manager. If the manager is empty and the loader
73
- * is not finished, it will transfer a request from the loader to the manager first.
78
+ * Fetches the next request, transferring one from the loader first if the loader still has work.
74
79
  * @inheritdoc
75
80
  */
76
81
  async fetchNextRequest() {
77
- // First, try to transfer a request from the requestList
78
- const [listEmpty, listFinished] = await Promise.all([
79
- this.requestLoader.isEmpty(),
80
- this.requestLoader.isFinished(),
81
- ]);
82
- if (!listEmpty && !listFinished) {
82
+ // Only the loader's own state decides this: a manager waiting out a backoff must not freeze the
83
+ // loader's unrelated requests for the length of it.
84
+ if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
83
85
  // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
84
86
  // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
85
- if (!(await this.transferNextRequestToQueue())) {
87
+ if (!(await this.#transferNextRequestToQueue())) {
86
88
  return null;
87
89
  }
88
90
  }
89
91
  // Try to fetch from manager after the transfer
90
- return (await this.getRequestManager()).fetchNextRequest();
92
+ return (await this.#getRequestManager()).fetchNextRequest();
91
93
  }
92
94
  /**
95
+ * The loader and the manager read as one source.
93
96
  * @inheritdoc
94
97
  */
95
- async isFinished() {
96
- const requestManager = await this.getRequestManager();
97
- const storagesFinished = await Promise.all([this.requestLoader.isFinished(), requestManager.isFinished()]);
98
- return storagesFinished.every(Boolean);
99
- }
100
- /**
101
- * @inheritdoc
102
- */
103
- async isEmpty() {
104
- const requestManager = await this.getRequestManager();
105
- const storagesEmpty = await Promise.all([this.requestLoader.isEmpty(), requestManager.isEmpty()]);
106
- return storagesEmpty.every(Boolean);
98
+ async checkReadiness() {
99
+ const requestManager = await this.#getRequestManager();
100
+ const [loaderStatus, managerStatus] = await Promise.all([
101
+ this.#requestLoader.checkReadiness(),
102
+ requestManager.checkReadiness(),
103
+ ]);
104
+ return joinRequestSourceStatuses(loaderStatus, managerStatus);
107
105
  }
108
106
  /**
109
107
  * @inheritdoc
110
108
  */
111
109
  async getHandledCount() {
112
110
  // Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
113
- return (await this.getRequestManager()).getHandledCount();
111
+ return (await this.#getRequestManager()).getHandledCount();
114
112
  }
115
113
  /**
116
114
  * @inheritdoc
117
115
  */
118
116
  async getTotalCount() {
119
- const requestManager = await this.getRequestManager();
117
+ const requestManager = await this.#getRequestManager();
120
118
  const [managerTotal, loaderTotal] = await Promise.all([
121
119
  requestManager.getTotalCount(),
122
120
  // count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
123
- this.requestLoader.getPendingCount(),
121
+ this.#requestLoader.getPendingCount(),
124
122
  ]);
125
123
  return managerTotal + loaderTotal;
126
124
  }
@@ -128,10 +126,10 @@ export class RequestManagerTandem {
128
126
  * @inheritdoc
129
127
  */
130
128
  async getPendingCount() {
131
- const requestManager = await this.getRequestManager();
129
+ const requestManager = await this.#getRequestManager();
132
130
  const [managerPending, loaderPending] = await Promise.all([
133
131
  requestManager.getPendingCount(),
134
- this.requestLoader.getPendingCount(),
132
+ this.#requestLoader.getPendingCount(),
135
133
  ]);
136
134
  return managerPending + loaderPending;
137
135
  }
@@ -150,32 +148,25 @@ export class RequestManagerTandem {
150
148
  * @inheritdoc
151
149
  */
152
150
  async markRequestAsHandled(request) {
153
- return (await this.getRequestManager()).markRequestAsHandled(request);
151
+ return (await this.#getRequestManager()).markRequestAsHandled(request);
154
152
  }
155
153
  /**
156
154
  * @inheritdoc
157
155
  */
158
156
  async reclaimRequest(request, options) {
159
- return (await this.getRequestManager()).reclaimRequest(request, options);
157
+ return (await this.#getRequestManager()).reclaimRequest(request, options);
160
158
  }
161
159
  /**
162
160
  * @inheritdoc
163
161
  */
164
162
  async addRequest(requestLike, options) {
165
- return (await this.getRequestManager()).addRequest(requestLike, options);
163
+ return (await this.#getRequestManager()).addRequest(requestLike, options);
166
164
  }
167
165
  /**
168
166
  * @inheritdoc
169
167
  */
170
168
  async addRequestsBatched(requests, options) {
171
- return (await this.getRequestManager()).addRequestsBatched(requests, options);
172
- }
173
- /**
174
- * Persists the state of the underlying read-only loader, if it supports persistence.
175
- * @inheritdoc
176
- */
177
- async persistState() {
178
- await this.requestLoader.persistState?.();
169
+ return (await this.#getRequestManager()).addRequestsBatched(requests, options);
179
170
  }
180
171
  /**
181
172
  * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
@@ -183,7 +174,7 @@ export class RequestManagerTandem {
183
174
  * @inheritdoc
184
175
  */
185
176
  async purge() {
186
- await (await this.getRequestManager()).purge?.();
177
+ await (await this.#getRequestManager()).purge?.();
187
178
  }
188
179
  /**
189
180
  * Forwards the hint to the writable request manager — that is where requests are fetched from and
@@ -191,7 +182,19 @@ export class RequestManagerTandem {
191
182
  * @inheritdoc
192
183
  */
193
184
  async setExpectedRequestProcessingTimeSecs(secs) {
194
- this.expectedRequestProcessingSecs = secs;
195
- await this.resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
185
+ this.#expectedRequestProcessingSecs = secs;
186
+ await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
187
+ }
188
+ /**
189
+ * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
190
+ * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
191
+ * pacing.
192
+ * @inheritdoc
193
+ */
194
+ recordPacingSignal(signal) {
195
+ return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
196
+ }
197
+ async extendRequestProcessingTimeSecs(request, secs) {
198
+ return (await this.#getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
196
199
  }
197
200
  }