@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/configuration.d.ts +16 -47
- package/configuration.js +13 -25
- package/debug.js +4 -4
- package/errors.d.ts +28 -38
- package/errors.js +33 -47
- package/events/event_manager.d.ts +2 -2
- package/events/event_manager.js +7 -6
- package/events/index.d.ts +1 -0
- package/events/local_event_manager.d.ts +1 -8
- package/events/local_event_manager.js +13 -13
- package/events/system_info.d.ts +38 -0
- package/index.d.ts +2 -8
- package/index.js +4 -8
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/log.d.ts +10 -11
- package/log.js +52 -20
- package/memory-storage/memory-storage.d.ts +15 -18
- package/memory-storage/memory-storage.js +80 -58
- package/memory-storage/resource-clients/dataset.d.ts +1 -6
- package/memory-storage/resource-clients/dataset.js +23 -31
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -10
- package/memory-storage/resource-clients/key-value-store.js +43 -67
- package/memory-storage/resource-clients/request-queue.d.ts +1 -42
- package/memory-storage/resource-clients/request-queue.js +109 -117
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +17 -20
- package/proxy_configuration.d.ts +21 -26
- package/proxy_configuration.js +35 -25
- package/recoverable_state.d.ts +104 -47
- package/recoverable_state.js +199 -74
- package/request.d.ts +20 -107
- package/request.js +78 -244
- package/serialization.js +17 -16
- package/service_locator.d.ts +22 -10
- package/service_locator.js +59 -48
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +13 -8
- package/storages/dataset.js +149 -40
- package/storages/index.d.ts +4 -4
- package/storages/index.js +2 -4
- package/storages/key_value_store.d.ts +16 -35
- package/storages/key_value_store.js +223 -110
- package/storages/key_value_store_codec.js +6 -11
- package/storages/request_dedup_cache.d.ts +1 -4
- package/storages/request_dedup_cache.js +15 -15
- package/storages/request_list.d.ts +9 -104
- package/storages/request_list.js +236 -233
- package/storages/request_loader.d.ts +49 -18
- package/storages/request_loader.js +36 -1
- package/storages/request_manager.d.ts +86 -0
- package/storages/request_manager_tandem.d.ts +14 -38
- package/storages/request_manager_tandem.js +67 -64
- package/storages/request_queue.d.ts +23 -50
- package/storages/request_queue.js +371 -226
- package/storages/storage_instance_manager.d.ts +2 -4
- package/storages/storage_instance_manager.js +21 -21
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/transaction.d.ts +270 -0
- package/storages/transaction.js +296 -0
- package/storages/utils.d.ts +6 -3
- package/storages/utils.js +11 -2
- package/system-info/runtime.js +7 -7
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -25
- package/validators.js +14 -25
- package/autoscaling/autoscaled_pool.d.ts +0 -213
- package/autoscaling/autoscaled_pool.js +0 -378
- package/autoscaling/client_load_signal.d.ts +0 -59
- package/autoscaling/client_load_signal.js +0 -73
- package/autoscaling/concurrency_system.d.ts +0 -283
- package/autoscaling/concurrency_system.js +0 -350
- package/autoscaling/cpu_load_signal.d.ts +0 -44
- package/autoscaling/cpu_load_signal.js +0 -46
- package/autoscaling/event_loop_load_signal.d.ts +0 -54
- package/autoscaling/event_loop_load_signal.js +0 -60
- package/autoscaling/index.d.ts +0 -9
- package/autoscaling/index.js +0 -9
- package/autoscaling/load_signal.d.ts +0 -99
- package/autoscaling/load_signal.js +0 -103
- package/autoscaling/memory_load_signal.d.ts +0 -56
- package/autoscaling/memory_load_signal.js +0 -106
- package/autoscaling/snapshotter.d.ts +0 -87
- package/autoscaling/snapshotter.js +0 -67
- package/autoscaling/system_status.d.ts +0 -161
- package/autoscaling/system_status.js +0 -139
- package/autoscaling/weighted_avg.d.ts +0 -5
- package/autoscaling/weighted_avg.js +0 -14
- package/cookie_utils.d.ts +0 -44
- package/cookie_utils.js +0 -122
- package/crawlers/context_pipeline.d.ts +0 -70
- package/crawlers/context_pipeline.js +0 -122
- package/crawlers/crawler_commons.d.ts +0 -257
- package/crawlers/crawler_commons.js +0 -107
- package/crawlers/error_snapshotter.d.ts +0 -59
- package/crawlers/error_snapshotter.js +0 -117
- package/crawlers/error_tracker.d.ts +0 -54
- package/crawlers/error_tracker.js +0 -308
- package/crawlers/index.d.ts +0 -5
- package/crawlers/index.js +0 -5
- package/crawlers/internals/types.d.ts +0 -7
- package/crawlers/statistics.d.ts +0 -209
- package/crawlers/statistics.js +0 -350
- package/enqueue_links/enqueue_links.d.ts +0 -264
- package/enqueue_links/enqueue_links.js +0 -271
- package/enqueue_links/index.d.ts +0 -2
- package/enqueue_links/index.js +0 -2
- package/enqueue_links/shared.d.ts +0 -83
- package/enqueue_links/shared.js +0 -221
- package/router.d.ts +0 -309
- package/router.js +0 -309
- package/session_pool/consts.d.ts +0 -3
- package/session_pool/consts.js +0 -3
- package/session_pool/errors.d.ts +0 -7
- package/session_pool/errors.js +0 -11
- package/session_pool/fingerprint.d.ts +0 -9
- package/session_pool/fingerprint.js +0 -30
- package/session_pool/index.d.ts +0 -4
- package/session_pool/index.js +0 -4
- package/session_pool/session.d.ts +0 -161
- package/session_pool/session.js +0 -218
- package/session_pool/session_pool.d.ts +0 -246
- package/session_pool/session_pool.js +0 -386
- package/storages/access_checking.d.ts +0 -12
- package/storages/access_checking.js +0 -17
- package/storages/sitemap_request_loader.d.ts +0 -249
- package/storages/sitemap_request_loader.js +0 -432
- /package/{crawlers/internals/types.js → events/system_info.js} +0 -0
|
@@ -2,6 +2,44 @@ import type { Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import type { Request } from '../request.js';
|
|
3
3
|
import type { IRequestManager } from './request_manager.js';
|
|
4
4
|
import type { RequestQueueOperationInfo } from './request_queue.js';
|
|
5
|
+
/**
|
|
6
|
+
* A request source's own availability, in a single answer.
|
|
7
|
+
*
|
|
8
|
+
* - `ready` — the next {@link IRequestLoader.fetchNextRequest} is expected to hand something over.
|
|
9
|
+
* - `waiting` — nothing to fetch right now, but the source is not done: requests are in progress, are being
|
|
10
|
+
* added in the background, or are held back until `readyAt`.
|
|
11
|
+
* - `stalled` — the source holds requests it cannot make progress on. Only a manager that paces its own
|
|
12
|
+
* dispatch can reach this; see {@link ThrottlingRequestManager}.
|
|
13
|
+
* - `finished` — everything has been handled.
|
|
14
|
+
*/
|
|
15
|
+
export type RequestSourceStatus = {
|
|
16
|
+
status: 'ready';
|
|
17
|
+
} | {
|
|
18
|
+
status: 'waiting';
|
|
19
|
+
/**
|
|
20
|
+
* A `Date.now()` timestamp at which the source expects to become `ready`. Absent when the wait has
|
|
21
|
+
* no clock (an in-progress request, a background add), leaving a consumer to poll.
|
|
22
|
+
*/
|
|
23
|
+
readyAt?: number;
|
|
24
|
+
} | {
|
|
25
|
+
status: 'stalled';
|
|
26
|
+
reason: string;
|
|
27
|
+
} | {
|
|
28
|
+
status: 'finished';
|
|
29
|
+
};
|
|
30
|
+
/** Loaders never stall — only a manager that paces its own dispatch can. */
|
|
31
|
+
export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
|
|
32
|
+
status: 'stalled';
|
|
33
|
+
}>;
|
|
34
|
+
/**
|
|
35
|
+
* Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
|
|
36
|
+
*
|
|
37
|
+
* Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
|
|
38
|
+
* nothing.
|
|
39
|
+
*
|
|
40
|
+
* @internal
|
|
41
|
+
*/
|
|
42
|
+
export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourceStatus): RequestSourceStatus;
|
|
5
43
|
/**
|
|
6
44
|
* An abstract interface defining a read-only stream of requests to crawl.
|
|
7
45
|
*
|
|
@@ -21,10 +59,10 @@ import type { RequestQueueOperationInfo } from './request_queue.js';
|
|
|
21
59
|
* "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
|
|
22
60
|
*
|
|
23
61
|
* Honoring this contract matters for three reasons:
|
|
24
|
-
* - **Restarts and migrations:** loaders that persist their state (
|
|
62
|
+
* - **Restarts and migrations:** loaders that persist their state (such as {@link RequestList})
|
|
25
63
|
* treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
|
|
26
64
|
* but never marked handled will be crawled again.
|
|
27
|
-
* - **Termination detection:** {@link IRequestLoader.
|
|
65
|
+
* - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
|
|
28
66
|
* in progress. Leaving a request unmarked keeps the crawler running indefinitely.
|
|
29
67
|
* - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
|
|
30
68
|
* skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
|
|
@@ -47,15 +85,13 @@ export interface IRequestLoader {
|
|
|
47
85
|
*/
|
|
48
86
|
getHandledCount(): Promise<number>;
|
|
49
87
|
/**
|
|
50
|
-
*
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
*
|
|
55
|
-
* would return `null`, otherwise it resolves to `false`.
|
|
56
|
-
* Note that even if the loader is empty, there might be some pending requests currently being processed.
|
|
88
|
+
* Reports whether the loader has a request to hand over, is waiting on one, or is done — see
|
|
89
|
+
* {@link RequestSourceStatus}.
|
|
90
|
+
*
|
|
91
|
+
* A consumer's task loop is gated on this, so implementations MUST answer `ready` before evaluating
|
|
92
|
+
* anything else. `finished` may arrive late behind distributed storage, but it is never wrong.
|
|
57
93
|
*/
|
|
58
|
-
|
|
94
|
+
checkReadiness(): Promise<RequestSourceStatus>;
|
|
59
95
|
/**
|
|
60
96
|
* Gets the next {@link Request} to process, or `null` if there are no more pending requests.
|
|
61
97
|
*
|
|
@@ -76,16 +112,11 @@ export interface IRequestLoader {
|
|
|
76
112
|
*
|
|
77
113
|
* Call this once you are done with the request — whether processing succeeded or was abandoned after
|
|
78
114
|
* exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
|
|
79
|
-
* signal completion; failing to do so prevents {@link IRequestLoader.
|
|
80
|
-
* `
|
|
115
|
+
* signal completion; failing to do so prevents {@link IRequestLoader.checkReadiness} from ever reporting
|
|
116
|
+
* `finished` and skews the handled and pending counts. See the request lifecycle contract on
|
|
117
|
+
* {@link IRequestLoader}.
|
|
81
118
|
*/
|
|
82
119
|
markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
|
|
83
|
-
/**
|
|
84
|
-
* Persists the current state of the loader into the default {@link KeyValueStore}.
|
|
85
|
-
*
|
|
86
|
-
* Not all loaders support persistence; implementations that do not should leave this `undefined`.
|
|
87
|
-
*/
|
|
88
|
-
persistState?(): Promise<void>;
|
|
89
120
|
/**
|
|
90
121
|
* Combines the loader with a request manager to support adding and reclaiming requests.
|
|
91
122
|
*
|
|
@@ -1 +1,36 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
|
|
3
|
+
*
|
|
4
|
+
* Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
|
|
5
|
+
* nothing.
|
|
6
|
+
*
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export function joinRequestSourceStatuses(a, b) {
|
|
10
|
+
if (a.status === 'ready') {
|
|
11
|
+
return a;
|
|
12
|
+
}
|
|
13
|
+
if (b.status === 'ready') {
|
|
14
|
+
return b;
|
|
15
|
+
}
|
|
16
|
+
// `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
|
|
17
|
+
// the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
|
|
18
|
+
// which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
|
|
19
|
+
// turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
|
|
20
|
+
// `waiting` is the same parity - that call site fired regardless of other domains' clocks.
|
|
21
|
+
if (a.status === 'stalled') {
|
|
22
|
+
return a;
|
|
23
|
+
}
|
|
24
|
+
if (b.status === 'stalled') {
|
|
25
|
+
return b;
|
|
26
|
+
}
|
|
27
|
+
if (a.status === 'waiting') {
|
|
28
|
+
// The earlier of the two known wake-up times - unknown only if neither source announced one.
|
|
29
|
+
if (b.status !== 'waiting' || b.readyAt === undefined) {
|
|
30
|
+
return a;
|
|
31
|
+
}
|
|
32
|
+
return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
|
|
33
|
+
}
|
|
34
|
+
// `a` is finished, so `b` decides.
|
|
35
|
+
return b;
|
|
36
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LiteralUnion } from 'type-fest';
|
|
1
2
|
import type { Request, Source } from '../request.js';
|
|
2
3
|
import type { IRequestLoader } from './request_loader.js';
|
|
3
4
|
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
|
|
@@ -30,4 +31,89 @@ export interface IRequestManager extends IRequestLoader {
|
|
|
30
31
|
* this hint may leave it `undefined`.
|
|
31
32
|
*/
|
|
32
33
|
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
|
|
34
|
+
/**
|
|
35
|
+
* Records something said about the pace requests should go out at, so that a manager which paces its own
|
|
36
|
+
* dispatch can hold requests back.
|
|
37
|
+
*
|
|
38
|
+
* Required rather than optional, so that a wrapping manager always forwards it and a pacer nested in a
|
|
39
|
+
* composition still receives it; a manager that does not pace returns `false`.
|
|
40
|
+
*
|
|
41
|
+
* @returns `true` if anything in the composition took responsibility for the signal.
|
|
42
|
+
*/
|
|
43
|
+
recordPacingSignal(signal: PacingSignal): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
|
|
46
|
+
* processed, on storage backends that reserve requests via locking (e.g. via
|
|
47
|
+
* {@link CrawlingContext.extendTimeout|`context.extendTimeout`}).
|
|
48
|
+
*
|
|
49
|
+
* @returns `true` when the lock was prolonged, `false` when this manager does not lock
|
|
50
|
+
* requests or no longer holds this one; non-locking implementations may leave it `undefined`,
|
|
51
|
+
* which callers treat as `false`.
|
|
52
|
+
*/
|
|
53
|
+
extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;
|
|
33
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* How much of the URL space a {@link PacingSignal} covers.
|
|
57
|
+
*
|
|
58
|
+
* Open on purpose: `'hostname'` and `'registrableDomain'` are what Crawlee's own reporters send and what
|
|
59
|
+
* {@link ThrottlingRequestManager} understands, but any string is accepted, so a pacer keyed on something
|
|
60
|
+
* else can be reported to in its own vocabulary.
|
|
61
|
+
*/
|
|
62
|
+
export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;
|
|
63
|
+
/**
|
|
64
|
+
* Something said about the pace requests should go out at, reported to a request manager through
|
|
65
|
+
* {@link IRequestManager.recordPacingSignal}.
|
|
66
|
+
*
|
|
67
|
+
* One shape rather than a method per channel: a pacing manager switches on `reason`, a wrapping one forwards the
|
|
68
|
+
* value without knowing what is in it, and a new kind of signal costs the interface nothing. The `url` travels
|
|
69
|
+
* inside the value because the crawl-wide variant has none. Nothing here names the mechanism a signal came
|
|
70
|
+
* from - status codes, headers and robots.txt are the crawler's business - and every delay is in milliseconds.
|
|
71
|
+
*
|
|
72
|
+
* ## Scope
|
|
73
|
+
*
|
|
74
|
+
* A manager may apply a signal to a **wider** scope than it was given - a floor that holds for one host still
|
|
75
|
+
* holds when a whole site is paced by it - but never to a narrower one, which would leave some of the URLs the
|
|
76
|
+
* signal covers running unpaced. A manager that can only do the latter, or that does not recognise the scope at
|
|
77
|
+
* all, MUST throw rather than quietly under-apply it.
|
|
78
|
+
*/
|
|
79
|
+
export type PacingSignal = {
|
|
80
|
+
/**
|
|
81
|
+
* The source turned a request away because we were going too fast — an HTTP 429 or 503, an exhausted
|
|
82
|
+
* quota. Reactive and transient: a pacer typically backs off while refusals continue, and lets that
|
|
83
|
+
* decay once they stop.
|
|
84
|
+
*/
|
|
85
|
+
reason: 'rateLimited';
|
|
86
|
+
/** The URL that was turned away. */
|
|
87
|
+
url: string;
|
|
88
|
+
/** How long the source asked us to wait before trying again, if it said. */
|
|
89
|
+
waitMs?: number;
|
|
90
|
+
/**
|
|
91
|
+
* How far this refusal reaches, if the reporter can tell — a 429 rarely says. Left out, it asks the
|
|
92
|
+
* manager to apply the signal however it happens to group requests.
|
|
93
|
+
*/
|
|
94
|
+
scope?: PacingScope;
|
|
95
|
+
} | {
|
|
96
|
+
/**
|
|
97
|
+
* The source declared a standing floor on how often it may be requested — a robots.txt `Crawl-delay`,
|
|
98
|
+
* a documented quota. A property of the source rather than of the run, so a pacer keeps it for the
|
|
99
|
+
* whole crawl.
|
|
100
|
+
*/
|
|
101
|
+
reason: 'minInterval';
|
|
102
|
+
/** A URL of the source that declared the interval. */
|
|
103
|
+
url: string;
|
|
104
|
+
/** The declared minimum interval between two requests to the source. */
|
|
105
|
+
intervalMs: number;
|
|
106
|
+
/** Required, since whoever declares an interval knows what it applies to. */
|
|
107
|
+
scope: PacingScope;
|
|
108
|
+
} | {
|
|
109
|
+
/**
|
|
110
|
+
* A standing floor under the pace of **every** domain the manager dispatches to, declared by whoever
|
|
111
|
+
* owns the crawl rather than by a source — a crawler's `sameDomainDelaySecs`. A manager that paces only
|
|
112
|
+
* some of its domains MUST throw rather than under-apply it.
|
|
113
|
+
*/
|
|
114
|
+
reason: 'minIntervalEverywhere';
|
|
115
|
+
/** The declared minimum interval between two requests to any one source. */
|
|
116
|
+
intervalMs: number;
|
|
117
|
+
/** At what granularity the floor applies. */
|
|
118
|
+
scope: PacingScope;
|
|
119
|
+
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { Request, Source } from '../request.js';
|
|
3
|
-
import type { IRequestLoader } from './request_loader.js';
|
|
4
|
-
import type { IRequestManager, RequestsLike } from './request_manager.js';
|
|
3
|
+
import type { IRequestLoader, RequestSourceStatus } from './request_loader.js';
|
|
4
|
+
import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
|
|
5
5
|
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
|
|
6
6
|
/**
|
|
7
7
|
* A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
|
|
@@ -9,16 +9,7 @@ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueO
|
|
|
9
9
|
* It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
|
|
10
10
|
*/
|
|
11
11
|
export declare class RequestManagerTandem implements IRequestManager {
|
|
12
|
-
private
|
|
13
|
-
private requestLoader;
|
|
14
|
-
private requestManagerPromise?;
|
|
15
|
-
private resolvedRequestManager?;
|
|
16
|
-
private requestManagerFactory;
|
|
17
|
-
/**
|
|
18
|
-
* The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
|
|
19
|
-
* Remembered so it can be applied to the writable manager once it is lazily resolved.
|
|
20
|
-
*/
|
|
21
|
-
private expectedRequestProcessingSecs?;
|
|
12
|
+
#private;
|
|
22
13
|
/**
|
|
23
14
|
* @param requestLoader The read-only loader to read requests from first.
|
|
24
15
|
* @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
|
|
@@ -27,33 +18,15 @@ export declare class RequestManagerTandem implements IRequestManager {
|
|
|
27
18
|
*/
|
|
28
19
|
constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
|
|
29
20
|
/**
|
|
30
|
-
*
|
|
31
|
-
* @private
|
|
32
|
-
*/
|
|
33
|
-
private getRequestManager;
|
|
34
|
-
/**
|
|
35
|
-
* Transfers a single request from the read-only loader to the writable manager.
|
|
36
|
-
* If the transfer fails, the request is dropped (and logged) rather than reclaimed.
|
|
37
|
-
*
|
|
38
|
-
* @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
|
|
39
|
-
* transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
|
|
40
|
-
* @private
|
|
41
|
-
*/
|
|
42
|
-
private transferNextRequestToQueue;
|
|
43
|
-
/**
|
|
44
|
-
* Fetches the next request from the request manager. If the manager is empty and the loader
|
|
45
|
-
* is not finished, it will transfer a request from the loader to the manager first.
|
|
21
|
+
* Fetches the next request, transferring one from the loader first if the loader still has work.
|
|
46
22
|
* @inheritdoc
|
|
47
23
|
*/
|
|
48
24
|
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
49
25
|
/**
|
|
26
|
+
* The loader and the manager read as one source.
|
|
50
27
|
* @inheritdoc
|
|
51
28
|
*/
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* @inheritdoc
|
|
55
|
-
*/
|
|
56
|
-
isEmpty(): Promise<boolean>;
|
|
29
|
+
checkReadiness(): Promise<RequestSourceStatus>;
|
|
57
30
|
/**
|
|
58
31
|
* @inheritdoc
|
|
59
32
|
*/
|
|
@@ -86,11 +59,6 @@ export declare class RequestManagerTandem implements IRequestManager {
|
|
|
86
59
|
* @inheritdoc
|
|
87
60
|
*/
|
|
88
61
|
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
|
|
89
|
-
/**
|
|
90
|
-
* Persists the state of the underlying read-only loader, if it supports persistence.
|
|
91
|
-
* @inheritdoc
|
|
92
|
-
*/
|
|
93
|
-
persistState(): Promise<void>;
|
|
94
62
|
/**
|
|
95
63
|
* Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
|
|
96
64
|
* The read-only loader is immutable and cannot be purged, so only the manager side is reset.
|
|
@@ -103,4 +71,12 @@ export declare class RequestManagerTandem implements IRequestManager {
|
|
|
103
71
|
* @inheritdoc
|
|
104
72
|
*/
|
|
105
73
|
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
|
|
76
|
+
* its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
|
|
77
|
+
* pacing.
|
|
78
|
+
* @inheritdoc
|
|
79
|
+
*/
|
|
80
|
+
recordPacingSignal(signal: PacingSignal): boolean;
|
|
81
|
+
extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
|
|
106
82
|
}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
import { serviceLocator } from '../service_locator.js';
|
|
2
|
+
import { joinRequestSourceStatuses } from './request_loader.js';
|
|
2
3
|
/**
|
|
3
4
|
* A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
|
|
4
5
|
* {@link IRequestManager} (such as a `RequestQueue`).
|
|
5
6
|
* It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
|
|
6
7
|
*/
|
|
7
8
|
export class RequestManagerTandem {
|
|
8
|
-
log;
|
|
9
|
-
requestLoader;
|
|
10
|
-
requestManagerPromise;
|
|
11
|
-
resolvedRequestManager;
|
|
12
|
-
requestManagerFactory;
|
|
9
|
+
#log;
|
|
10
|
+
#requestLoader;
|
|
11
|
+
#requestManagerPromise;
|
|
12
|
+
#resolvedRequestManager;
|
|
13
|
+
#requestManagerFactory;
|
|
13
14
|
/**
|
|
14
15
|
* The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
|
|
15
16
|
* Remembered so it can be applied to the writable manager once it is lazily resolved.
|
|
16
17
|
*/
|
|
17
|
-
expectedRequestProcessingSecs;
|
|
18
|
+
#expectedRequestProcessingSecs;
|
|
18
19
|
/**
|
|
19
20
|
* @param requestLoader The read-only loader to read requests from first.
|
|
20
21
|
* @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
|
|
@@ -22,24 +23,30 @@ export class RequestManagerTandem {
|
|
|
22
23
|
* (e.g. a lazily-opened default {@link RequestQueue}).
|
|
23
24
|
*/
|
|
24
25
|
constructor(requestLoader, requestManager) {
|
|
25
|
-
this
|
|
26
|
-
this
|
|
27
|
-
|
|
26
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
|
|
27
|
+
this.#requestLoader = requestLoader;
|
|
28
|
+
if (typeof requestManager === 'function') {
|
|
29
|
+
this.#requestManagerFactory = requestManager;
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
// Nothing to open, so mark it resolved up front - synchronous pacing signals can then reach it.
|
|
33
|
+
this.#resolvedRequestManager = requestManager;
|
|
34
|
+
this.#requestManagerFactory = () => requestManager;
|
|
35
|
+
}
|
|
28
36
|
}
|
|
29
37
|
/**
|
|
30
38
|
* Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
|
|
31
|
-
* @private
|
|
32
39
|
*/
|
|
33
|
-
async getRequestManager() {
|
|
34
|
-
if (this
|
|
35
|
-
this
|
|
36
|
-
this
|
|
40
|
+
async #getRequestManager() {
|
|
41
|
+
if (this.#resolvedRequestManager === undefined) {
|
|
42
|
+
this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
|
|
43
|
+
this.#resolvedRequestManager = await this.#requestManagerPromise;
|
|
37
44
|
// Apply any hint received before the manager was resolved.
|
|
38
|
-
if (this
|
|
39
|
-
await this
|
|
45
|
+
if (this.#expectedRequestProcessingSecs !== undefined) {
|
|
46
|
+
await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
|
|
40
47
|
}
|
|
41
48
|
}
|
|
42
|
-
return this
|
|
49
|
+
return this.#resolvedRequestManager;
|
|
43
50
|
}
|
|
44
51
|
/**
|
|
45
52
|
* Transfers a single request from the read-only loader to the writable manager.
|
|
@@ -47,80 +54,71 @@ export class RequestManagerTandem {
|
|
|
47
54
|
*
|
|
48
55
|
* @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
|
|
49
56
|
* transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
|
|
50
|
-
* @private
|
|
51
57
|
*/
|
|
52
|
-
async transferNextRequestToQueue() {
|
|
53
|
-
const request = await this
|
|
58
|
+
async #transferNextRequestToQueue() {
|
|
59
|
+
const request = await this.#requestLoader.fetchNextRequest();
|
|
54
60
|
if (request === null) {
|
|
55
61
|
return true;
|
|
56
62
|
}
|
|
57
|
-
const requestManager = await this
|
|
63
|
+
const requestManager = await this.#getRequestManager();
|
|
58
64
|
try {
|
|
59
65
|
await requestManager.addRequest(request, { forefront: true });
|
|
60
66
|
return true;
|
|
61
67
|
}
|
|
62
68
|
catch (error) {
|
|
63
|
-
this
|
|
69
|
+
this.#log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
|
|
64
70
|
return false;
|
|
65
71
|
}
|
|
66
72
|
finally {
|
|
67
73
|
// Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
|
|
68
|
-
await this
|
|
74
|
+
await this.#requestLoader.markRequestAsHandled(request);
|
|
69
75
|
}
|
|
70
76
|
}
|
|
71
77
|
/**
|
|
72
|
-
* Fetches the next request from the
|
|
73
|
-
* is not finished, it will transfer a request from the loader to the manager first.
|
|
78
|
+
* Fetches the next request, transferring one from the loader first if the loader still has work.
|
|
74
79
|
* @inheritdoc
|
|
75
80
|
*/
|
|
76
81
|
async fetchNextRequest() {
|
|
77
|
-
//
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
this.requestLoader.isFinished(),
|
|
81
|
-
]);
|
|
82
|
-
if (!listEmpty && !listFinished) {
|
|
82
|
+
// Only the loader's own state decides this: a manager waiting out a backoff must not freeze the
|
|
83
|
+
// loader's unrelated requests for the length of it.
|
|
84
|
+
if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
|
|
83
85
|
// If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
|
|
84
86
|
// crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
|
|
85
|
-
if (!(await this
|
|
87
|
+
if (!(await this.#transferNextRequestToQueue())) {
|
|
86
88
|
return null;
|
|
87
89
|
}
|
|
88
90
|
}
|
|
89
91
|
// Try to fetch from manager after the transfer
|
|
90
|
-
return (await this
|
|
92
|
+
return (await this.#getRequestManager()).fetchNextRequest();
|
|
91
93
|
}
|
|
92
94
|
/**
|
|
95
|
+
* The loader and the manager read as one source.
|
|
93
96
|
* @inheritdoc
|
|
94
97
|
*/
|
|
95
|
-
async
|
|
96
|
-
const requestManager = await this
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
*/
|
|
103
|
-
async isEmpty() {
|
|
104
|
-
const requestManager = await this.getRequestManager();
|
|
105
|
-
const storagesEmpty = await Promise.all([this.requestLoader.isEmpty(), requestManager.isEmpty()]);
|
|
106
|
-
return storagesEmpty.every(Boolean);
|
|
98
|
+
async checkReadiness() {
|
|
99
|
+
const requestManager = await this.#getRequestManager();
|
|
100
|
+
const [loaderStatus, managerStatus] = await Promise.all([
|
|
101
|
+
this.#requestLoader.checkReadiness(),
|
|
102
|
+
requestManager.checkReadiness(),
|
|
103
|
+
]);
|
|
104
|
+
return joinRequestSourceStatuses(loaderStatus, managerStatus);
|
|
107
105
|
}
|
|
108
106
|
/**
|
|
109
107
|
* @inheritdoc
|
|
110
108
|
*/
|
|
111
109
|
async getHandledCount() {
|
|
112
110
|
// Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
|
|
113
|
-
return (await this
|
|
111
|
+
return (await this.#getRequestManager()).getHandledCount();
|
|
114
112
|
}
|
|
115
113
|
/**
|
|
116
114
|
* @inheritdoc
|
|
117
115
|
*/
|
|
118
116
|
async getTotalCount() {
|
|
119
|
-
const requestManager = await this
|
|
117
|
+
const requestManager = await this.#getRequestManager();
|
|
120
118
|
const [managerTotal, loaderTotal] = await Promise.all([
|
|
121
119
|
requestManager.getTotalCount(),
|
|
122
120
|
// count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
|
|
123
|
-
this
|
|
121
|
+
this.#requestLoader.getPendingCount(),
|
|
124
122
|
]);
|
|
125
123
|
return managerTotal + loaderTotal;
|
|
126
124
|
}
|
|
@@ -128,10 +126,10 @@ export class RequestManagerTandem {
|
|
|
128
126
|
* @inheritdoc
|
|
129
127
|
*/
|
|
130
128
|
async getPendingCount() {
|
|
131
|
-
const requestManager = await this
|
|
129
|
+
const requestManager = await this.#getRequestManager();
|
|
132
130
|
const [managerPending, loaderPending] = await Promise.all([
|
|
133
131
|
requestManager.getPendingCount(),
|
|
134
|
-
this
|
|
132
|
+
this.#requestLoader.getPendingCount(),
|
|
135
133
|
]);
|
|
136
134
|
return managerPending + loaderPending;
|
|
137
135
|
}
|
|
@@ -150,32 +148,25 @@ export class RequestManagerTandem {
|
|
|
150
148
|
* @inheritdoc
|
|
151
149
|
*/
|
|
152
150
|
async markRequestAsHandled(request) {
|
|
153
|
-
return (await this
|
|
151
|
+
return (await this.#getRequestManager()).markRequestAsHandled(request);
|
|
154
152
|
}
|
|
155
153
|
/**
|
|
156
154
|
* @inheritdoc
|
|
157
155
|
*/
|
|
158
156
|
async reclaimRequest(request, options) {
|
|
159
|
-
return (await this
|
|
157
|
+
return (await this.#getRequestManager()).reclaimRequest(request, options);
|
|
160
158
|
}
|
|
161
159
|
/**
|
|
162
160
|
* @inheritdoc
|
|
163
161
|
*/
|
|
164
162
|
async addRequest(requestLike, options) {
|
|
165
|
-
return (await this
|
|
163
|
+
return (await this.#getRequestManager()).addRequest(requestLike, options);
|
|
166
164
|
}
|
|
167
165
|
/**
|
|
168
166
|
* @inheritdoc
|
|
169
167
|
*/
|
|
170
168
|
async addRequestsBatched(requests, options) {
|
|
171
|
-
return (await this
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Persists the state of the underlying read-only loader, if it supports persistence.
|
|
175
|
-
* @inheritdoc
|
|
176
|
-
*/
|
|
177
|
-
async persistState() {
|
|
178
|
-
await this.requestLoader.persistState?.();
|
|
169
|
+
return (await this.#getRequestManager()).addRequestsBatched(requests, options);
|
|
179
170
|
}
|
|
180
171
|
/**
|
|
181
172
|
* Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
|
|
@@ -183,7 +174,7 @@ export class RequestManagerTandem {
|
|
|
183
174
|
* @inheritdoc
|
|
184
175
|
*/
|
|
185
176
|
async purge() {
|
|
186
|
-
await (await this
|
|
177
|
+
await (await this.#getRequestManager()).purge?.();
|
|
187
178
|
}
|
|
188
179
|
/**
|
|
189
180
|
* Forwards the hint to the writable request manager — that is where requests are fetched from and
|
|
@@ -191,7 +182,19 @@ export class RequestManagerTandem {
|
|
|
191
182
|
* @inheritdoc
|
|
192
183
|
*/
|
|
193
184
|
async setExpectedRequestProcessingTimeSecs(secs) {
|
|
194
|
-
this
|
|
195
|
-
await this
|
|
185
|
+
this.#expectedRequestProcessingSecs = secs;
|
|
186
|
+
await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
|
|
190
|
+
* its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
|
|
191
|
+
* pacing.
|
|
192
|
+
* @inheritdoc
|
|
193
|
+
*/
|
|
194
|
+
recordPacingSignal(signal) {
|
|
195
|
+
return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
|
|
196
|
+
}
|
|
197
|
+
async extendRequestProcessingTimeSecs(request, secs) {
|
|
198
|
+
return (await this.#getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
|
|
196
199
|
}
|
|
197
200
|
}
|