@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +11 -1
- package/index.js +11 -0
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +84 -312
- package/internals/basic-crawler.js +709 -598
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +6 -0
- package/internals/request-timeout.js +27 -2
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.js +17 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +165 -0
- package/internals/sitemap-request-loader.js +492 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +19 -15
|
@@ -1,16 +1,29 @@
|
|
|
1
1
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
|
|
4
4
|
import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
|
|
5
|
-
import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
|
|
5
|
+
import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
|
|
6
6
|
import { RobotsTxtFile } from '@crawlee/utils';
|
|
7
|
-
import { getDomain } from 'tldts';
|
|
8
7
|
import { z } from 'zod';
|
|
9
8
|
import { LruCache } from '@apify/datastructures';
|
|
10
|
-
import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
|
|
9
|
+
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
11
10
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
12
|
-
import {
|
|
11
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
12
|
+
import { mergeCookies } from './cookie-utils.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import { CrawlingRequest, RequestState } from './crawling-request.js';
|
|
15
|
+
import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
|
|
16
|
+
import { Statistics } from './crawlers/statistics.js';
|
|
17
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
|
|
18
|
+
import { parseRetryAfterHeader } from './http.js';
|
|
19
|
+
import { CrawlerRun } from './crawler-run.js';
|
|
20
|
+
import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
|
|
13
21
|
import { createSendRequest } from './send-request.js';
|
|
22
|
+
import { Router, validateUserData } from './router.js';
|
|
23
|
+
import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
|
|
24
|
+
import { Session } from './session-pool/session.js';
|
|
25
|
+
import { SessionPool } from './session-pool/session-pool.js';
|
|
26
|
+
import { ThrottlingRequestManager } from './throttling-request-manager.js';
|
|
14
27
|
class LazyDefaultHttpClient extends BaseHttpClient {
|
|
15
28
|
#delegatePromise;
|
|
16
29
|
constructor(options) {
|
|
@@ -40,7 +53,6 @@ class LazyDefaultHttpClient extends BaseHttpClient {
|
|
|
40
53
|
* @ignore
|
|
41
54
|
*/
|
|
42
55
|
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
|
|
43
|
-
const deferredCleanupKey = Symbol('deferredCleanup');
|
|
44
56
|
// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
|
|
45
57
|
export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
|
|
46
58
|
const urlPatternSchema = z.union([
|
|
@@ -71,14 +83,13 @@ const addRequestsOptionsSchema = z.looseObject({
|
|
|
71
83
|
onSkippedRequest: schemas.anyFunction.optional(),
|
|
72
84
|
});
|
|
73
85
|
export class BasicCrawler {
|
|
74
|
-
static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
|
|
86
|
+
static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
|
|
75
87
|
/**
|
|
76
88
|
* Tracks the number of crawler instances created. The first crawler uses the default
|
|
77
89
|
* request queue; subsequent ones get their own queue via a unique alias so they don't
|
|
78
90
|
* collide.
|
|
79
91
|
*/
|
|
80
|
-
|
|
81
|
-
static instanceCount = 0;
|
|
92
|
+
static #instanceCount = 0;
|
|
82
93
|
/**
|
|
83
94
|
* Tracks crawler instances that accessed shared state without having an explicit id.
|
|
84
95
|
* Used to detect and warn about multiple crawlers sharing the same state.
|
|
@@ -93,12 +104,15 @@ export class BasicCrawler {
|
|
|
93
104
|
get statistics() {
|
|
94
105
|
return this.#statisticsDep.value;
|
|
95
106
|
}
|
|
107
|
+
#requestManager;
|
|
96
108
|
/**
|
|
97
109
|
* The main request-handling component of the crawler. It manages the requests that the crawler processes,
|
|
98
110
|
* combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
|
|
99
111
|
* via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
|
|
100
112
|
*/
|
|
101
|
-
requestManager
|
|
113
|
+
get requestManager() {
|
|
114
|
+
return this.#requestManager;
|
|
115
|
+
}
|
|
102
116
|
/** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
|
|
103
117
|
#sessionPoolDep;
|
|
104
118
|
/**
|
|
@@ -108,12 +122,6 @@ export class BasicCrawler {
|
|
|
108
122
|
get sessionPool() {
|
|
109
123
|
return this.#sessionPoolDep.value;
|
|
110
124
|
}
|
|
111
|
-
/**
|
|
112
|
-
* Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
|
|
113
|
-
* around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
|
|
114
|
-
* through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
|
|
115
|
-
*/
|
|
116
|
-
#ownedRequestQueue = OwnedOrInjected.resolve();
|
|
117
125
|
/**
|
|
118
126
|
* Whether the request-processing-time hint has already been forwarded to the request manager. The hint
|
|
119
127
|
* derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
|
|
@@ -126,31 +134,36 @@ export class BasicCrawler {
|
|
|
126
134
|
* the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
|
|
127
135
|
*/
|
|
128
136
|
#resolveConcurrencySystem;
|
|
129
|
-
/**
|
|
130
|
-
#
|
|
137
|
+
/** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
|
|
138
|
+
#run;
|
|
139
|
+
#concurrencySystem;
|
|
140
|
+
/** The run in progress, if any — a finished run is history and cannot be driven. */
|
|
141
|
+
get #liveRun() {
|
|
142
|
+
return this.#run?.isLive === true ? this.#run : undefined;
|
|
143
|
+
}
|
|
144
|
+
/** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
|
|
145
|
+
get #dispatchingRun() {
|
|
146
|
+
return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
|
|
147
|
+
}
|
|
148
|
+
/** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
|
|
149
|
+
get running() {
|
|
150
|
+
return this.#liveRun !== undefined;
|
|
151
|
+
}
|
|
131
152
|
/**
|
|
132
|
-
* The concurrency governor
|
|
153
|
+
* The concurrency governor the run in progress is booking its requests against — either the
|
|
133
154
|
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
|
|
134
155
|
* crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
|
|
135
156
|
*
|
|
136
|
-
* > *NOTE:* `undefined`
|
|
137
|
-
*
|
|
157
|
+
* > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
|
|
158
|
+
* during a run rather than caching it across runs.
|
|
138
159
|
*
|
|
139
160
|
* {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
|
|
140
161
|
* owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
|
|
141
162
|
* `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
|
|
142
163
|
*/
|
|
143
164
|
get concurrencySystem() {
|
|
144
|
-
return this.#
|
|
165
|
+
return this.#concurrencySystem?.value;
|
|
145
166
|
}
|
|
146
|
-
/**
|
|
147
|
-
* The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
|
|
148
|
-
* no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
|
|
149
|
-
* with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
|
|
150
|
-
* {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
|
|
151
|
-
* {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
|
|
152
|
-
*/
|
|
153
|
-
#autoscaledPool;
|
|
154
167
|
/**
|
|
155
168
|
* A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
|
|
156
169
|
* Only available if used by the crawler.
|
|
@@ -168,40 +181,38 @@ export class BasicCrawler {
|
|
|
168
181
|
* pipelines expect the basic crawler fields to already be present in the context at runtime.
|
|
169
182
|
*
|
|
170
183
|
* Context built with this pipeline can be passed into multiple crawler pipelines at once.
|
|
171
|
-
* This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
|
|
172
184
|
*/
|
|
173
|
-
get
|
|
174
|
-
|
|
175
|
-
this.#basicContextPipeline = this.buildBasicContextPipeline();
|
|
176
|
-
}
|
|
185
|
+
get #basicPipeline() {
|
|
186
|
+
this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
|
|
177
187
|
return this.#basicContextPipeline;
|
|
178
188
|
}
|
|
179
189
|
#contextPipeline;
|
|
190
|
+
/** @internal */
|
|
180
191
|
get contextPipeline() {
|
|
181
192
|
if (this.#contextPipeline === undefined) {
|
|
182
|
-
this.#contextPipeline = this
|
|
193
|
+
this.#contextPipeline = this.#buildFinalContextPipeline();
|
|
183
194
|
}
|
|
184
195
|
return this.#contextPipeline;
|
|
185
196
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
197
|
+
#hasFinishedBefore = false;
|
|
198
|
+
/** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
|
|
199
|
+
get hasFinishedBefore() {
|
|
200
|
+
return this.#hasFinishedBefore;
|
|
201
|
+
}
|
|
189
202
|
#log;
|
|
190
203
|
get log() {
|
|
191
204
|
return this.#log;
|
|
192
205
|
}
|
|
193
206
|
requestHandler;
|
|
194
|
-
errorHandler;
|
|
195
|
-
failedRequestHandler;
|
|
196
|
-
|
|
197
|
-
requestHandlerTimeoutMillis;
|
|
207
|
+
#errorHandler;
|
|
208
|
+
#failedRequestHandler;
|
|
209
|
+
#requestHandlerTimeoutMillis;
|
|
198
210
|
internalTimeoutMillis;
|
|
199
|
-
maxRequestRetries;
|
|
200
|
-
maxCrawlDepth;
|
|
201
|
-
#
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
|
|
211
|
+
#maxRequestRetries;
|
|
212
|
+
#maxCrawlDepth;
|
|
213
|
+
#maxRequestsPerCrawl;
|
|
214
|
+
get #handledRequestsCount() {
|
|
215
|
+
return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
|
|
205
216
|
}
|
|
206
217
|
#statusMessageLoggingInterval;
|
|
207
218
|
#statusMessageCallback;
|
|
@@ -214,8 +225,7 @@ export class BasicCrawler {
|
|
|
214
225
|
* {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
|
|
215
226
|
* settable here.
|
|
216
227
|
*/
|
|
217
|
-
|
|
218
|
-
taskLoopOptions;
|
|
228
|
+
#taskLoopOptions;
|
|
219
229
|
httpClient;
|
|
220
230
|
retryOnBlocked;
|
|
221
231
|
#respectRobotsTxtFile;
|
|
@@ -223,12 +233,13 @@ export class BasicCrawler {
|
|
|
223
233
|
#transactionalStorageEnabled;
|
|
224
234
|
/** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
|
|
225
235
|
#storageWritePolicy;
|
|
226
|
-
onSkippedRequest;
|
|
227
|
-
#closeEvents;
|
|
228
|
-
#loggedPerRun = new Set();
|
|
236
|
+
#onSkippedRequest;
|
|
229
237
|
#robotsTxtFileCache;
|
|
230
|
-
identity;
|
|
238
|
+
#identity;
|
|
231
239
|
#contextPipelineOptions;
|
|
240
|
+
/**
|
|
241
|
+
* @internal
|
|
242
|
+
*/
|
|
232
243
|
static optionsShape = {
|
|
233
244
|
contextPipelineBuilder: schemas.anyObject.optional(),
|
|
234
245
|
extendContext: schemas.anyFunction.optional(),
|
|
@@ -270,6 +281,7 @@ export class BasicCrawler {
|
|
|
270
281
|
// AutoscaledPool shorthands
|
|
271
282
|
minConcurrency: schemas.anyNumber.optional(),
|
|
272
283
|
maxConcurrency: schemas.anyNumber.optional(),
|
|
284
|
+
initialConcurrency: schemas.anyNumber.optional(),
|
|
273
285
|
maxRequestsPerMinute: schemas.anyNumber
|
|
274
286
|
.refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
|
|
275
287
|
.refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
|
|
@@ -278,12 +290,12 @@ export class BasicCrawler {
|
|
|
278
290
|
statistics: schemas.anyObject.optional(),
|
|
279
291
|
id: z.string().optional(),
|
|
280
292
|
};
|
|
281
|
-
static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
|
|
293
|
+
static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
|
|
282
294
|
/**
|
|
283
295
|
* All `BasicCrawler` parameters are passed via an options object.
|
|
284
296
|
*/
|
|
285
297
|
constructor(options = {}) {
|
|
286
|
-
const parsedOptions = parseArgument(options, BasicCrawler
|
|
298
|
+
const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
|
|
287
299
|
const {
|
|
288
300
|
// oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
|
|
289
301
|
requestList,
|
|
@@ -292,15 +304,18 @@ export class BasicCrawler {
|
|
|
292
304
|
// Service locator options
|
|
293
305
|
configuration, storageBackend, eventManager, logger,
|
|
294
306
|
// AutoscaledPool shorthands
|
|
295
|
-
minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
|
|
307
|
+
minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
|
|
296
308
|
// All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
|
|
297
309
|
// one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
|
|
298
310
|
// hammering a site.
|
|
299
311
|
if (concurrencySystem !== undefined &&
|
|
300
|
-
(minConcurrency !== undefined ||
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
312
|
+
(minConcurrency !== undefined ||
|
|
313
|
+
maxConcurrency !== undefined ||
|
|
314
|
+
initialConcurrency !== undefined ||
|
|
315
|
+
maxRequestsPerMinute !== undefined)) {
|
|
316
|
+
throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
|
|
317
|
+
'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
|
|
318
|
+
'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
|
|
304
319
|
}
|
|
305
320
|
// Create per-crawler service locator if custom services were provided.
|
|
306
321
|
// This wraps every method on the crawler instance so that calls to the global `serviceLocator`
|
|
@@ -328,50 +343,58 @@ export class BasicCrawler {
|
|
|
328
343
|
this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
|
|
329
344
|
// Initialize the Configuration instance to avoid lazy loading in the components
|
|
330
345
|
serviceLocator.getConfiguration();
|
|
331
|
-
const instanceIndex = BasicCrawler
|
|
332
|
-
this
|
|
333
|
-
if (requestManager !== undefined) {
|
|
334
|
-
|
|
335
|
-
throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
|
|
336
|
-
}
|
|
337
|
-
// Both would pace the same domains, from different keys and with no idea of one another.
|
|
338
|
-
if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
|
|
339
|
-
throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
|
|
340
|
-
'per domain on its own. Configure the delay on the manager instead, via the ' +
|
|
341
|
-
'`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
|
|
342
|
-
}
|
|
343
|
-
this.requestManager = requestManager;
|
|
346
|
+
const instanceIndex = BasicCrawler.#instanceCount++;
|
|
347
|
+
this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
|
|
348
|
+
if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
|
|
349
|
+
throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
|
|
344
350
|
}
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
351
|
+
const suppliedManager = requestManager ?? requestQueue;
|
|
352
|
+
// Offered before building a pacer of our own: anything that paces takes the floor - through any
|
|
353
|
+
// number of wrappers, since they all forward - so no domain ends up with two clocks.
|
|
354
|
+
const floorTaken = sameDomainDelaySecs > 0 &&
|
|
355
|
+
(suppliedManager?.recordPacingSignal({
|
|
356
|
+
reason: 'minIntervalEverywhere',
|
|
357
|
+
intervalMs: sameDomainDelaySecs * 1000,
|
|
358
|
+
// What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
|
|
359
|
+
scope: 'registrableDomain',
|
|
360
|
+
}) ??
|
|
361
|
+
false);
|
|
362
|
+
const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
|
|
363
|
+
// Built here rather than at first use so it can sit *inside* the tandem below, which is where a
|
|
364
|
+
// loader's transferred requests pass through it.
|
|
365
|
+
const writableManager = pacerNeeded
|
|
366
|
+
? new ThrottlingRequestManager({
|
|
367
|
+
domains: 'all',
|
|
368
|
+
minCrawlDelaySecs: sameDomainDelaySecs,
|
|
369
|
+
throttleBy: 'registrableDomain',
|
|
370
|
+
persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
|
|
371
|
+
// A factory, because the default queue is only opened on first use.
|
|
372
|
+
inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
|
|
373
|
+
})
|
|
374
|
+
: suppliedManager;
|
|
375
|
+
if (requestList !== undefined) {
|
|
376
|
+
// The list is read first, while new requests still have somewhere writable to go.
|
|
377
|
+
this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
|
|
348
378
|
}
|
|
349
|
-
else if (
|
|
379
|
+
else if (writableManager !== undefined) {
|
|
350
380
|
// A RequestQueue is itself a request manager.
|
|
351
|
-
this
|
|
352
|
-
}
|
|
353
|
-
else if (requestList !== undefined) {
|
|
354
|
-
// A lone read-only `requestList` (deprecated option) is combined with a lazily-opened default queue
|
|
355
|
-
// into a tandem, so that its requests are read first and new ones can still be enqueued during the
|
|
356
|
-
// crawl. The queue is opened on first use; the tandem also forwards `persistState()` to the loader.
|
|
357
|
-
this.requestManager = new RequestManagerTandem(requestList, () => this.openOwnedRequestQueue());
|
|
381
|
+
this.#requestManager = writableManager;
|
|
358
382
|
}
|
|
359
383
|
this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
|
|
360
384
|
this.proxyConfiguration = proxyConfiguration;
|
|
361
385
|
this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
|
|
362
386
|
this.#statusMessageCallback = statusMessageCallback;
|
|
363
387
|
this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
|
|
364
|
-
this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
|
|
365
388
|
this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
|
|
366
389
|
this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
|
|
367
390
|
this.requestHandler = requestHandler ?? this.router;
|
|
368
|
-
this
|
|
369
|
-
this
|
|
391
|
+
this.#failedRequestHandler = failedRequestHandler;
|
|
392
|
+
this.#errorHandler = errorHandler;
|
|
370
393
|
if (requestHandlerTimeoutSecs) {
|
|
371
|
-
this
|
|
394
|
+
this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
|
|
372
395
|
}
|
|
373
396
|
else {
|
|
374
|
-
this
|
|
397
|
+
this.#requestHandlerTimeoutMillis = 60_000;
|
|
375
398
|
}
|
|
376
399
|
this.retryOnBlocked = retryOnBlocked;
|
|
377
400
|
this.#respectRobotsTxtFile = respectRobotsTxtFile;
|
|
@@ -379,14 +402,13 @@ export class BasicCrawler {
|
|
|
379
402
|
const transactionalStorageOption = transactionalStorage;
|
|
380
403
|
this.#transactionalStorageEnabled = transactionalStorageOption !== false;
|
|
381
404
|
this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
|
|
382
|
-
this
|
|
405
|
+
this.#onSkippedRequest = onSkippedRequest;
|
|
383
406
|
// allow at least 5min for internal timeouts
|
|
384
407
|
this.internalTimeoutMillis =
|
|
385
408
|
serviceLocator.getConfiguration().internalTimeoutMillis ??
|
|
386
|
-
Math.max(this
|
|
387
|
-
this
|
|
388
|
-
this
|
|
389
|
-
this.#sameDomainDelaySecs = sameDomainDelaySecs;
|
|
409
|
+
Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
|
|
410
|
+
this.#maxRequestRetries = maxRequestRetries;
|
|
411
|
+
this.#maxCrawlDepth = maxCrawlDepth;
|
|
390
412
|
this.#statisticsDep = OwnedOrInjected.resolve(statistics,
|
|
391
413
|
// A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
|
|
392
414
|
// only be satisfied by an injected instance carrying the matching `state`, so this branch does
|
|
@@ -394,7 +416,7 @@ export class BasicCrawler {
|
|
|
394
416
|
() => new Statistics({
|
|
395
417
|
logMessage: `${this.constructor.name} request statistics:`,
|
|
396
418
|
log: this.log,
|
|
397
|
-
id: this
|
|
419
|
+
id: this.#identity.id,
|
|
398
420
|
}));
|
|
399
421
|
if (sessionPool && proxyConfiguration) {
|
|
400
422
|
this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
|
|
@@ -405,19 +427,21 @@ export class BasicCrawler {
|
|
|
405
427
|
this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
|
|
406
428
|
createSessionFunction: async (opts) => new Session({
|
|
407
429
|
...opts?.sessionOptions,
|
|
408
|
-
proxyInfo:
|
|
430
|
+
proxyInfo: this.proxyConfiguration
|
|
431
|
+
? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
|
|
432
|
+
: opts?.sessionOptions?.proxyInfo,
|
|
409
433
|
}),
|
|
410
434
|
}));
|
|
411
435
|
this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
|
|
412
436
|
const maxSignedInteger = 2 ** 31 - 1;
|
|
413
|
-
if (this
|
|
414
|
-
this.log.warning(`requestHandlerTimeoutMillis ${this
|
|
437
|
+
if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
|
|
438
|
+
this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
|
|
415
439
|
` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
|
|
416
|
-
this
|
|
440
|
+
this.#requestHandlerTimeoutMillis = maxSignedInteger;
|
|
417
441
|
}
|
|
418
442
|
this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
|
|
419
|
-
this
|
|
420
|
-
const isMaxPagesExceeded = () => this
|
|
443
|
+
this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
|
|
444
|
+
const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
|
|
421
445
|
// eslint-disable-next-line prefer-const
|
|
422
446
|
let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
|
|
423
447
|
// override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
|
|
@@ -429,90 +453,74 @@ export class BasicCrawler {
|
|
|
429
453
|
const source = this.requestManager;
|
|
430
454
|
if (!source)
|
|
431
455
|
throw new Error('Request provider is not initialized!');
|
|
432
|
-
const request = await this
|
|
456
|
+
const request = await this.#resolveRequest();
|
|
433
457
|
if (!request) {
|
|
434
458
|
return;
|
|
435
459
|
}
|
|
436
460
|
// Started here, rather than in `handleRequest`, so that a failure during context pipeline
|
|
437
461
|
// initialization (e.g. a browser page timing out before the request handler ever runs) is
|
|
438
|
-
// still accounted for by `
|
|
439
|
-
|
|
462
|
+
// still accounted for by `recordRequestFailure` below - which is a no-op without a
|
|
463
|
+
// matching `recordRequestStart`.
|
|
464
|
+
this.statistics.recordRequestStart(request.id || request.uniqueKey);
|
|
440
465
|
const crawlingContext = { request };
|
|
441
466
|
try {
|
|
442
467
|
// The transaction spans the whole pipeline call, covering the navigation hooks
|
|
443
|
-
// and `extendContext` too; `handleRequest`
|
|
468
|
+
// and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
|
|
444
469
|
await this.runInStorageTransaction(async () =>
|
|
445
470
|
// Navigation, the navigation hooks and the request handler are timed individually, but the
|
|
446
471
|
// phases between them are not, so a request could still get stuck indefinitely. This is the
|
|
447
472
|
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
|
|
448
|
-
await this
|
|
449
|
-
.chain(this.contextPipeline)
|
|
450
|
-
.call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
|
|
473
|
+
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
|
|
451
474
|
}
|
|
452
475
|
catch (error) {
|
|
453
|
-
//
|
|
454
|
-
//
|
|
455
|
-
if (error instanceof
|
|
456
|
-
this
|
|
457
|
-
await this.
|
|
458
|
-
|
|
459
|
-
}
|
|
460
|
-
// If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
|
|
461
|
-
// i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
|
|
462
|
-
// here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
|
|
463
|
-
const isPipelineError = error instanceof ContextPipelineInitializationError ||
|
|
464
|
-
error instanceof SessionError ||
|
|
465
|
-
error instanceof TimeoutError;
|
|
466
|
-
if (isPipelineError) {
|
|
467
|
-
const unwrappedError = this.unwrapError(error);
|
|
468
|
-
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
|
|
469
|
-
// SessionError already retired the session in `requestFunctionErrorHandler`;
|
|
470
|
-
// skip `markBad` to avoid double-counting usage/error score.
|
|
471
|
-
if (!this.errorAbsolvesSession(unwrappedError)) {
|
|
476
|
+
// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
|
|
477
|
+
// take the normal error flow. Anything else is fatal.
|
|
478
|
+
if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
|
|
479
|
+
const unwrappedError = this.#unwrapError(error);
|
|
480
|
+
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
|
|
481
|
+
if (!this.#errorAbsolvesSession(unwrappedError)) {
|
|
472
482
|
crawlingContext.session?.markBad();
|
|
473
483
|
}
|
|
474
484
|
return;
|
|
475
485
|
}
|
|
476
|
-
throw this
|
|
477
|
-
}
|
|
478
|
-
finally {
|
|
479
|
-
// Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
|
|
480
|
-
const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
|
|
481
|
-
await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
|
|
486
|
+
throw this.#unwrapError(error);
|
|
482
487
|
}
|
|
483
488
|
},
|
|
484
489
|
isTaskReadyFunction: async () => {
|
|
485
490
|
if (isMaxPagesExceeded()) {
|
|
486
|
-
this
|
|
487
|
-
`${this
|
|
491
|
+
this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
|
|
492
|
+
`${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
|
|
488
493
|
return false;
|
|
489
494
|
}
|
|
490
|
-
if (this.#
|
|
491
|
-
this
|
|
495
|
+
if (this.#run?.stopRequested) {
|
|
496
|
+
this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
|
|
492
497
|
'Ongoing requests will be allowed to complete.');
|
|
493
498
|
return false;
|
|
494
499
|
}
|
|
495
|
-
return isTaskReadyFunction ? await isTaskReadyFunction() : await this
|
|
500
|
+
return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
|
|
496
501
|
},
|
|
497
502
|
isFinishedFunction: async () => {
|
|
498
503
|
if (isMaxPagesExceeded()) {
|
|
499
|
-
this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this
|
|
504
|
+
this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
|
|
500
505
|
'and all requests that were in progress at that time have now finished. ' +
|
|
501
|
-
`In total, the crawler processed ${this
|
|
506
|
+
`In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
|
|
502
507
|
return true;
|
|
503
508
|
}
|
|
504
|
-
if (this.#
|
|
509
|
+
if (this.#run?.stopRequested) {
|
|
505
510
|
this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
|
|
506
511
|
return true;
|
|
507
512
|
}
|
|
508
|
-
//
|
|
509
|
-
// crawl that cannot progress
|
|
510
|
-
|
|
511
|
-
|
|
513
|
+
// `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
|
|
514
|
+
// where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
|
|
515
|
+
// and the only place where throwing does not abandon requests mid-processing.
|
|
516
|
+
const state = await this.requestManager?.checkReadiness();
|
|
517
|
+
// Under `keepAlive`, outliving a domain that will not let us through is the whole point.
|
|
518
|
+
if (state?.status === 'stalled' && !keepAlive) {
|
|
519
|
+
throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
|
|
512
520
|
}
|
|
513
521
|
const isFinished = isFinishedFunction
|
|
514
522
|
? await isFinishedFunction()
|
|
515
|
-
:
|
|
523
|
+
: state === undefined || state.status === 'finished';
|
|
516
524
|
if (isFinished) {
|
|
517
525
|
const reason = isFinishedFunction
|
|
518
526
|
? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
|
|
@@ -523,11 +531,14 @@ export class BasicCrawler {
|
|
|
523
531
|
},
|
|
524
532
|
log: this.log,
|
|
525
533
|
};
|
|
526
|
-
this
|
|
534
|
+
this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
|
|
527
535
|
this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
|
|
528
536
|
minConcurrency,
|
|
529
537
|
maxConcurrency,
|
|
530
538
|
maxTasksPerMinute: maxRequestsPerMinute,
|
|
539
|
+
// Spread conditionally - an explicit `undefined` would clobber a subclass default, see
|
|
540
|
+
// `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
|
|
541
|
+
...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
|
|
531
542
|
log: this.log,
|
|
532
543
|
}));
|
|
533
544
|
}
|
|
@@ -537,7 +548,7 @@ export class BasicCrawler {
|
|
|
537
548
|
}
|
|
538
549
|
/**
|
|
539
550
|
* Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
|
|
540
|
-
* `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
|
|
551
|
+
* `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
|
|
541
552
|
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
|
|
542
553
|
*
|
|
543
554
|
* Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
|
|
@@ -561,35 +572,21 @@ export class BasicCrawler {
|
|
|
561
572
|
* Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
|
|
562
573
|
* This handles base context creation, session resolution, and context helpers.
|
|
563
574
|
*/
|
|
564
|
-
buildBasicContextPipeline() {
|
|
575
|
+
#buildBasicContextPipeline() {
|
|
565
576
|
return ContextPipeline.create()
|
|
566
|
-
.compose(
|
|
567
|
-
.compose(
|
|
568
|
-
.compose(
|
|
569
|
-
.compose(
|
|
577
|
+
.compose(this.#checkRobotsTxt.bind(this))
|
|
578
|
+
.compose(this.#createBaseContext.bind(this))
|
|
579
|
+
.compose(this.#resolveSession.bind(this))
|
|
580
|
+
.compose(this.#createContextHelpers.bind(this));
|
|
570
581
|
}
|
|
571
|
-
async checkRobotsTxt({ request }) {
|
|
572
|
-
if (!(await this
|
|
582
|
+
async #checkRobotsTxt({ request }) {
|
|
583
|
+
if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
|
|
573
584
|
this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
|
|
574
|
-
|
|
575
|
-
request.noRetry = true;
|
|
576
|
-
await this.handleSkippedRequest({
|
|
577
|
-
url: request.url,
|
|
578
|
-
reason: 'robotsTxt',
|
|
579
|
-
});
|
|
580
|
-
throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
|
|
585
|
+
throw new ContextPipelineInterruptedError('robotsTxt');
|
|
581
586
|
}
|
|
582
587
|
return {};
|
|
583
588
|
}
|
|
584
|
-
|
|
585
|
-
* Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
|
|
586
|
-
* Subclasses should override this to add their own pipeline stages.
|
|
587
|
-
*/
|
|
588
|
-
buildContextPipeline() {
|
|
589
|
-
return ContextPipeline.create();
|
|
590
|
-
}
|
|
591
|
-
createBaseContext(context) {
|
|
592
|
-
const deferredCleanup = [];
|
|
589
|
+
#createBaseContext(context, onCleanup) {
|
|
593
590
|
return {
|
|
594
591
|
id: cryptoRandomObjectId(10),
|
|
595
592
|
log: this.log,
|
|
@@ -597,7 +594,23 @@ export class BasicCrawler {
|
|
|
597
594
|
useState: this.useState.bind(this),
|
|
598
595
|
getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
|
|
599
596
|
registerDeferredCleanup: (cleanup) => {
|
|
600
|
-
|
|
597
|
+
onCleanup(async () => {
|
|
598
|
+
try {
|
|
599
|
+
await cleanup();
|
|
600
|
+
}
|
|
601
|
+
catch (error) {
|
|
602
|
+
this.log.debug('Error in deferred cleanup', { error });
|
|
603
|
+
}
|
|
604
|
+
});
|
|
605
|
+
},
|
|
606
|
+
afterStorageCommit: (callback) => {
|
|
607
|
+
const transaction = currentStorageTransaction();
|
|
608
|
+
if (transaction === undefined) {
|
|
609
|
+
throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
|
|
610
|
+
"With `transactionalStorage: false` the request's writes are applied as they are " +
|
|
611
|
+
'made, so they throw at the call site instead of at commit time.');
|
|
612
|
+
}
|
|
613
|
+
transaction.afterCommit(callback);
|
|
601
614
|
},
|
|
602
615
|
extendTimeout: (secs) => {
|
|
603
616
|
const extraMillis = secs * 1000;
|
|
@@ -610,29 +623,43 @@ export class BasicCrawler {
|
|
|
610
623
|
if (context[navigationDeadlineKey] !== undefined) {
|
|
611
624
|
context[navigationDeadlineKey] += extraMillis;
|
|
612
625
|
}
|
|
626
|
+
// Extension failure must not fail the request that asked for more time.
|
|
627
|
+
this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
|
|
628
|
+
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
|
|
629
|
+
});
|
|
630
|
+
},
|
|
631
|
+
skipRequest: (message) => {
|
|
632
|
+
this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
|
|
633
|
+
throw new ContextPipelineInterruptedError('manual', message);
|
|
613
634
|
},
|
|
614
|
-
[deferredCleanupKey]: deferredCleanup,
|
|
615
635
|
};
|
|
616
636
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
637
|
+
/**
|
|
638
|
+
* Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
|
|
639
|
+
* instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
|
|
640
|
+
* `markRequestAsHandled`, so nothing observes the difference.
|
|
641
|
+
*/
|
|
642
|
+
async #resolveRequest() {
|
|
643
|
+
const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
644
|
+
if (!fetched) {
|
|
645
|
+
return null;
|
|
622
646
|
}
|
|
647
|
+
const request = CrawlingRequest.fromSchema(fetched);
|
|
648
|
+
// Reset loadedUrl so an old one is not carried over to retries.
|
|
649
|
+
request.loadedUrl = undefined;
|
|
623
650
|
return request;
|
|
624
651
|
}
|
|
625
|
-
async resolveSession({ request }) {
|
|
626
|
-
const session = await this
|
|
652
|
+
async #resolveSession({ request }) {
|
|
653
|
+
const session = await this.#timeoutAndRetry(async () => {
|
|
627
654
|
const existingSession = await this.sessionPool.getSession(request.sessionId);
|
|
628
655
|
if (!existingSession) {
|
|
629
|
-
throw new
|
|
656
|
+
throw new MissingSessionError(request.sessionId);
|
|
630
657
|
}
|
|
631
658
|
return existingSession;
|
|
632
659
|
}, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
633
660
|
return { session, proxyInfo: session?.proxyInfo };
|
|
634
661
|
}
|
|
635
|
-
async createContextHelpers({ request, session }) {
|
|
662
|
+
async #createContextHelpers({ request, session }) {
|
|
636
663
|
const addRequests = async (requests, options = {}) => {
|
|
637
664
|
const newCrawlDepth = request.crawlDepth + 1;
|
|
638
665
|
const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
|
|
@@ -641,9 +668,9 @@ export class BasicCrawler {
|
|
|
641
668
|
const sendRequest = createSendRequest(this.httpClient, request, session);
|
|
642
669
|
return { addRequests, sendRequest };
|
|
643
670
|
}
|
|
644
|
-
buildFinalContextPipeline() {
|
|
671
|
+
#buildFinalContextPipeline() {
|
|
645
672
|
const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
|
|
646
|
-
|
|
673
|
+
ContextPipeline.create());
|
|
647
674
|
// `extendContext` runs *before* the subclass navigation pipeline (which includes the
|
|
648
675
|
// pre/post-navigation hooks). This makes the extension visible to those hooks and to the
|
|
649
676
|
// request handler alike. The trade-off is that `extendContext` cannot access
|
|
@@ -658,26 +685,21 @@ export class BasicCrawler {
|
|
|
658
685
|
let contextPipeline;
|
|
659
686
|
if (extendContext !== undefined) {
|
|
660
687
|
contextPipeline = ContextPipeline.create()
|
|
661
|
-
.compose(
|
|
688
|
+
.compose(extendContext)
|
|
662
689
|
.chain(subclassPipeline);
|
|
663
690
|
}
|
|
664
691
|
else {
|
|
665
692
|
contextPipeline = subclassPipeline;
|
|
666
693
|
}
|
|
667
|
-
contextPipeline = contextPipeline.compose({
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
|
|
677
|
-
throw new ContextPipelineInterruptedError(message);
|
|
678
|
-
}
|
|
679
|
-
return context;
|
|
680
|
-
},
|
|
694
|
+
contextPipeline = contextPipeline.compose(async (context) => {
|
|
695
|
+
const { request } = context;
|
|
696
|
+
if (request && !this.#requestMatchesEnqueueStrategy(request)) {
|
|
697
|
+
// eslint-disable-next-line dot-notation
|
|
698
|
+
const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
|
|
699
|
+
this.log.debug(message);
|
|
700
|
+
throw new ContextPipelineInterruptedError('redirect');
|
|
701
|
+
}
|
|
702
|
+
return context;
|
|
681
703
|
});
|
|
682
704
|
return contextPipeline;
|
|
683
705
|
}
|
|
@@ -688,7 +710,7 @@ export class BasicCrawler {
|
|
|
688
710
|
* @param error The error to check.
|
|
689
711
|
*/
|
|
690
712
|
isProxyError(error) {
|
|
691
|
-
return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)
|
|
713
|
+
return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
|
|
692
714
|
}
|
|
693
715
|
/**
|
|
694
716
|
* Sets the status message for the current crawler run.
|
|
@@ -701,19 +723,21 @@ export class BasicCrawler {
|
|
|
701
723
|
*/
|
|
702
724
|
setStatusMessage(message, options = {}) {
|
|
703
725
|
const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
|
|
704
|
-
|
|
726
|
+
// Each allowed level has its own method on the logger, so this goes through them rather than through
|
|
727
|
+
// `logWithLevel`, which is abstract and therefore cannot be instrumented.
|
|
728
|
+
this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
|
|
705
729
|
// Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
|
|
706
730
|
// subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
|
|
707
731
|
// Setting the status message is not a storage concern, so we intentionally don't route it
|
|
708
732
|
// through the storage client anymore.
|
|
709
733
|
serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
|
|
710
|
-
crawlerId: this
|
|
734
|
+
crawlerId: this.#identity.id,
|
|
711
735
|
message,
|
|
712
736
|
isStatusMessageTerminal: options.isStatusMessageTerminal,
|
|
713
737
|
level: options.level,
|
|
714
738
|
});
|
|
715
739
|
}
|
|
716
|
-
getPeriodicLogger() {
|
|
740
|
+
#getPeriodicLogger() {
|
|
717
741
|
let previousState = { ...this.statistics.state };
|
|
718
742
|
const getOperationMode = () => {
|
|
719
743
|
const { requestsFailed } = this.statistics.state;
|
|
@@ -733,7 +757,7 @@ export class BasicCrawler {
|
|
|
733
757
|
}
|
|
734
758
|
else {
|
|
735
759
|
const total = await this.requestManager?.getTotalCount();
|
|
736
|
-
message = `Crawled ${this.statistics.state.
|
|
760
|
+
message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
|
|
737
761
|
}
|
|
738
762
|
if (this.#statusMessageCallback) {
|
|
739
763
|
await this.#statusMessageCallback({
|
|
@@ -757,120 +781,141 @@ export class BasicCrawler {
|
|
|
757
781
|
* We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
|
|
758
782
|
* running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
|
|
759
783
|
*
|
|
784
|
+
* Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
|
|
785
|
+
* run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
|
|
786
|
+
* if that is what you want.
|
|
787
|
+
*
|
|
760
788
|
* @param [requests] The requests to add.
|
|
761
|
-
* @param [options] Options for the
|
|
789
|
+
* @param [options] Options for adding the initial requests.
|
|
762
790
|
*/
|
|
763
791
|
async run(requests, options) {
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
if (this.hasFinishedBefore) {
|
|
769
|
-
// When executing the run method for the second time explicitly,
|
|
770
|
-
// we need to purge the RQ to allow processing the same requests again — this is important so users can
|
|
771
|
-
// pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
|
|
772
|
-
// ignored — as a failed request is still handled.
|
|
773
|
-
// By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
|
|
774
|
-
// When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
|
|
775
|
-
// When `purgeRequestQueue` is explicitly `false`, nothing is purged.
|
|
776
|
-
const shouldPurge = purgeRequestQueue !== false;
|
|
777
|
-
const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
|
|
778
|
-
if (shouldPurge) {
|
|
779
|
-
await managerToPurge?.purge?.();
|
|
780
|
-
// The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
|
|
781
|
-
// sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
|
|
782
|
-
// the wrapper itself has already covered them.
|
|
783
|
-
if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
|
|
784
|
-
await this.requestManager.purgeDomainQueues();
|
|
785
|
-
}
|
|
792
|
+
// A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
|
|
793
|
+
return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
|
|
794
|
+
if (this.running) {
|
|
795
|
+
throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
|
|
786
796
|
}
|
|
787
|
-
//
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
this.#loggedPerRun.clear();
|
|
797
|
-
await purgeDefaultStorages({
|
|
798
|
-
onlyPurgeOnce: true,
|
|
799
|
-
storageBackend: serviceLocator.getStorageBackend(),
|
|
800
|
-
configuration: serviceLocator.getConfiguration(),
|
|
801
|
-
});
|
|
802
|
-
if (requests) {
|
|
803
|
-
await this.addRequests(requests, addRequestsOptions);
|
|
804
|
-
}
|
|
805
|
-
try {
|
|
806
|
-
await this.init();
|
|
807
|
-
await this.statistics.startCapturing();
|
|
808
|
-
}
|
|
809
|
-
catch (error) {
|
|
810
|
-
// Clean up here before propagating, otherwise a failed startup would leave the process hanging.
|
|
811
|
-
await this.teardown().catch((teardownError) => {
|
|
812
|
-
this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
|
|
797
|
+
// An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
|
|
798
|
+
// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
|
|
799
|
+
// run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
|
|
800
|
+
this.#concurrencySystem = this.#resolveConcurrencySystem();
|
|
801
|
+
const run = new CrawlerRun({
|
|
802
|
+
log: this.log,
|
|
803
|
+
concurrencySystem: this.#concurrencySystem.value,
|
|
804
|
+
taskLoopOptions: this.#taskLoopOptions,
|
|
805
|
+
consumer: this.#identity,
|
|
813
806
|
});
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
try {
|
|
833
|
-
await this.#autoscaledPool.run();
|
|
834
|
-
}
|
|
835
|
-
finally {
|
|
836
|
-
await this.statistics.stopCapturing();
|
|
837
|
-
await this.teardown();
|
|
838
|
-
process.off('SIGINT', sigintHandler);
|
|
839
|
-
eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
|
|
840
|
-
eventManager.off(EventType.ABORTING, boundPauseOnMigration);
|
|
841
|
-
const finalStats = this.statistics.calculate();
|
|
842
|
-
stats = {
|
|
843
|
-
requestsFinished: this.statistics.state.requestsFinished,
|
|
844
|
-
requestsFailed: this.statistics.state.requestsFailed,
|
|
845
|
-
retryHistogram: this.statistics.requestRetryHistogram,
|
|
846
|
-
...finalStats,
|
|
847
|
-
};
|
|
848
|
-
this.log.info('Final request statistics:', stats);
|
|
849
|
-
if (this.statistics.errorTracker.total !== 0) {
|
|
850
|
-
const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
|
|
851
|
-
this.log.info(`Error analysis:`, {
|
|
852
|
-
totalErrors: this.statistics.errorTracker.total,
|
|
853
|
-
uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
|
|
854
|
-
mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
|
|
807
|
+
this.#run = run;
|
|
808
|
+
try {
|
|
809
|
+
if (this.#hasFinishedBefore) {
|
|
810
|
+
// A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
|
|
811
|
+
await this.#statisticsDep.ifOwned(async (stats) => {
|
|
812
|
+
stats.reset();
|
|
813
|
+
await stats.resetStore();
|
|
814
|
+
});
|
|
815
|
+
// A session pool the crawler built follows the same blank-slate rule as its statistics.
|
|
816
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => {
|
|
817
|
+
pool.reset();
|
|
818
|
+
await pool.resetStore();
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
await purgeDefaultStorages({
|
|
822
|
+
onlyPurgeOnce: true,
|
|
823
|
+
storageBackend: serviceLocator.getStorageBackend(),
|
|
824
|
+
configuration: serviceLocator.getConfiguration(),
|
|
855
825
|
});
|
|
826
|
+
if (requests) {
|
|
827
|
+
await this.addRequests(requests, options);
|
|
828
|
+
}
|
|
829
|
+
// An injected governor is the caller's to run; an owned one is started here and stopped again
|
|
830
|
+
// below if the startup fails. Either way the task loop rejects one that is not running.
|
|
831
|
+
await this.#concurrencySystem.ifOwned((system) => system.start());
|
|
832
|
+
await this.init();
|
|
833
|
+
await this.statistics.startCapturing();
|
|
834
|
+
}
|
|
835
|
+
catch (error) {
|
|
836
|
+
// Clean up here before propagating, otherwise a failed startup would leave the process hanging.
|
|
837
|
+
await this.teardown().catch((teardownError) => {
|
|
838
|
+
this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
|
|
839
|
+
});
|
|
840
|
+
// The run never began, so let the instance be run again instead of leaving it wedged as `running`.
|
|
841
|
+
await run.finish();
|
|
842
|
+
await this.#concurrencySystem.ifOwned((system) => system.stop());
|
|
843
|
+
throw error;
|
|
844
|
+
}
|
|
845
|
+
const periodicLogger = this.#getPeriodicLogger();
|
|
846
|
+
this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
|
|
847
|
+
const sigintHandler = async () => {
|
|
848
|
+
this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
|
|
849
|
+
await this.#pauseOnMigration();
|
|
850
|
+
await run.abort();
|
|
851
|
+
};
|
|
852
|
+
// Attach a listener to handle migration and aborting events gracefully.
|
|
853
|
+
const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
|
|
854
|
+
process.once('SIGINT', sigintHandler);
|
|
855
|
+
const eventManager = serviceLocator.getEventManager();
|
|
856
|
+
eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
|
|
857
|
+
eventManager.on(EventType.ABORTING, boundPauseOnMigration);
|
|
858
|
+
let stats = {};
|
|
859
|
+
try {
|
|
860
|
+
await run.dispatchRequests();
|
|
856
861
|
}
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
862
|
+
finally {
|
|
863
|
+
await this.statistics.stopCapturing();
|
|
864
|
+
await this.teardown();
|
|
865
|
+
process.off('SIGINT', sigintHandler);
|
|
866
|
+
eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
|
|
867
|
+
eventManager.off(EventType.ABORTING, boundPauseOnMigration);
|
|
868
|
+
const finalStats = this.statistics.calculate();
|
|
869
|
+
stats = {
|
|
870
|
+
requestsSucceeded: this.statistics.state.requestsSucceeded,
|
|
871
|
+
requestsFailed: this.statistics.state.requestsFailed,
|
|
872
|
+
retryHistogram: this.statistics.requestRetryHistogram,
|
|
873
|
+
...finalStats,
|
|
874
|
+
};
|
|
875
|
+
this.log.info('Final request statistics:', stats);
|
|
876
|
+
// A crawl that did nothing while the manager holds only handled requests is a mistake whoever
|
|
877
|
+
// handled them - this run, another crawler on the same queue, or a previous process. Starting
|
|
878
|
+
// against handled requests is not: that is what resuming a crawl looks like.
|
|
879
|
+
if (stats.requestsSucceeded + stats.requestsFailed === 0) {
|
|
880
|
+
// Never let the diagnostic itself break the run.
|
|
881
|
+
const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
|
|
882
|
+
if (alreadyHandled > 0) {
|
|
883
|
+
this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
|
|
884
|
+
`${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
|
|
885
|
+
'already handled, and a failed request counts as handled too. Nothing ' +
|
|
886
|
+
'empties a queue between runs, so to crawl them again, purge it ' +
|
|
887
|
+
'(`await queue.purge()`) or use a fresh one (e.g. ' +
|
|
888
|
+
'`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
|
|
889
|
+
'crawler instance.');
|
|
863
890
|
}
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
|
|
891
|
+
}
|
|
892
|
+
if (this.statistics.errorTracker.total !== 0) {
|
|
893
|
+
const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
|
|
894
|
+
this.log.info(`Error analysis:`, {
|
|
895
|
+
totalErrors: this.statistics.errorTracker.total,
|
|
896
|
+
uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
|
|
897
|
+
mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
const client = serviceLocator.getStorageBackend();
|
|
901
|
+
if (client.teardown) {
|
|
902
|
+
let finished = false;
|
|
903
|
+
setTimeout(() => {
|
|
904
|
+
if (!finished) {
|
|
905
|
+
this.log.info('Waiting for the storage to write its state to file system.');
|
|
906
|
+
}
|
|
907
|
+
}, 1000);
|
|
908
|
+
await client.teardown();
|
|
909
|
+
finished = true;
|
|
910
|
+
}
|
|
911
|
+
periodicLogger.stop();
|
|
912
|
+
this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
|
|
913
|
+
await run.finish();
|
|
914
|
+
await this.#concurrencySystem.ifOwned((system) => system.stop());
|
|
915
|
+
this.#hasFinishedBefore = true;
|
|
867
916
|
}
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
this.running = false;
|
|
871
|
-
this.hasFinishedBefore = true;
|
|
872
|
-
}
|
|
873
|
-
return stats;
|
|
917
|
+
return stats;
|
|
918
|
+
}));
|
|
874
919
|
}
|
|
875
920
|
/**
|
|
876
921
|
* Gracefully stops the current run of the crawler.
|
|
@@ -880,11 +925,12 @@ export class BasicCrawler {
|
|
|
880
925
|
* To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
|
|
881
926
|
*/
|
|
882
927
|
stop(reason = 'The crawler has been gracefully stopped.') {
|
|
883
|
-
|
|
928
|
+
const run = this.#liveRun;
|
|
929
|
+
if (run === undefined) {
|
|
930
|
+
this.log.warning('Cannot stop a crawler that is not running.');
|
|
884
931
|
return;
|
|
885
932
|
}
|
|
886
|
-
|
|
887
|
-
this.#unexpectedStop = true;
|
|
933
|
+
run.stop(reason);
|
|
888
934
|
}
|
|
889
935
|
/**
|
|
890
936
|
* Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
|
|
@@ -895,52 +941,41 @@ export class BasicCrawler {
|
|
|
895
941
|
* throughout, since a shared one may still be serving other crawlers.
|
|
896
942
|
*/
|
|
897
943
|
async pause(timeoutSecs) {
|
|
898
|
-
|
|
944
|
+
const run = this.#dispatchingRun;
|
|
945
|
+
if (run === undefined) {
|
|
899
946
|
this.log.warning('Cannot pause a crawler that is not running.');
|
|
900
947
|
return;
|
|
901
948
|
}
|
|
902
|
-
await
|
|
949
|
+
await run.pause(timeoutSecs);
|
|
903
950
|
}
|
|
904
951
|
/**
|
|
905
952
|
* Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
|
|
906
953
|
* again. A no-op on a crawler that is not paused.
|
|
907
954
|
*/
|
|
908
955
|
resume() {
|
|
909
|
-
|
|
956
|
+
const run = this.#dispatchingRun;
|
|
957
|
+
if (run === undefined) {
|
|
910
958
|
this.log.warning('Cannot resume a crawler that is not running.');
|
|
911
959
|
return;
|
|
912
960
|
}
|
|
913
|
-
|
|
961
|
+
run.resume();
|
|
914
962
|
}
|
|
915
963
|
/**
|
|
916
964
|
* Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
|
|
917
965
|
* if none has been configured or opened yet.
|
|
918
966
|
*/
|
|
919
967
|
async getRequestManager() {
|
|
920
|
-
if (!this
|
|
921
|
-
this
|
|
922
|
-
}
|
|
923
|
-
// Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
|
|
924
|
-
// this point - and because everything that enqueues goes through here first, so nothing slips past the
|
|
925
|
-
// wrapper into the queue it hides.
|
|
926
|
-
if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
|
|
927
|
-
this.requestManager = new ThrottlingRequestManager({
|
|
928
|
-
inner: this.requestManager,
|
|
929
|
-
domains: 'all',
|
|
930
|
-
minCrawlDelaySecs: this.#sameDomainDelaySecs,
|
|
931
|
-
// What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
|
|
932
|
-
throttleBy: 'registrableDomain',
|
|
933
|
-
persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
|
|
934
|
-
});
|
|
968
|
+
if (!this.#requestManager) {
|
|
969
|
+
this.#requestManager = await this.#openOwnedRequestQueue();
|
|
935
970
|
}
|
|
936
971
|
// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
|
|
937
|
-
// now that `
|
|
972
|
+
// now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
|
|
938
973
|
// but guard so we do not re-issue it on every call.
|
|
939
974
|
if (!this.#requestManagerTimeoutsApplied) {
|
|
940
975
|
this.#requestManagerTimeoutsApplied = true;
|
|
941
|
-
await this
|
|
976
|
+
await this.#applyRequestManagerTimeouts(this.#requestManager);
|
|
942
977
|
}
|
|
943
|
-
return this
|
|
978
|
+
return this.#requestManager;
|
|
944
979
|
}
|
|
945
980
|
/**
|
|
946
981
|
* @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
|
|
@@ -950,16 +985,13 @@ export class BasicCrawler {
|
|
|
950
985
|
return this.getRequestManager();
|
|
951
986
|
}
|
|
952
987
|
/**
|
|
953
|
-
* Opens the default {@link RequestQueue}
|
|
954
|
-
* crawler-owned queue (so it gets purged between repeated `run()` calls).
|
|
955
|
-
* @private
|
|
988
|
+
* Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
|
|
956
989
|
*/
|
|
957
|
-
async openOwnedRequestQueue() {
|
|
990
|
+
async #openOwnedRequestQueue() {
|
|
958
991
|
// The first crawler instance uses the default queue (null identifier);
|
|
959
992
|
// subsequent instances get their own queue via a unique alias so they don't collide.
|
|
960
|
-
const identifier = this
|
|
961
|
-
|
|
962
|
-
return this.#ownedRequestQueue.set(requestQueue);
|
|
993
|
+
const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
|
|
994
|
+
return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
|
|
963
995
|
}
|
|
964
996
|
/**
|
|
965
997
|
* Tells a request manager how long we expect to hold a fetched request, so that one backed by a
|
|
@@ -968,13 +1000,13 @@ export class BasicCrawler {
|
|
|
968
1000
|
* request from being handed out a second time while it is still being processed — and it works
|
|
969
1001
|
* regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
|
|
970
1002
|
*/
|
|
971
|
-
async applyRequestManagerTimeouts(requestManager) {
|
|
1003
|
+
async #applyRequestManagerTimeouts(requestManager) {
|
|
972
1004
|
// A router route may hold a request for longer than the crawler's own timeout, and we cannot know
|
|
973
1005
|
// which routes a run will hit, so reserve for the longest one any route asked for. The hint is
|
|
974
1006
|
// raise-only, so erring high here is safe.
|
|
975
1007
|
const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
|
|
976
|
-
const handlerTimeoutSecs = Math.max(this
|
|
977
|
-
await requestManager.
|
|
1008
|
+
const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
|
|
1009
|
+
await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
|
|
978
1010
|
}
|
|
979
1011
|
/**
|
|
980
1012
|
* Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
|
|
@@ -983,7 +1015,7 @@ export class BasicCrawler {
|
|
|
983
1015
|
* the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
|
|
984
1016
|
* `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
|
|
985
1017
|
*/
|
|
986
|
-
async validateRequestUserData(source) {
|
|
1018
|
+
async #validateRequestUserData(source) {
|
|
987
1019
|
if (typeof source === 'string') {
|
|
988
1020
|
return;
|
|
989
1021
|
}
|
|
@@ -1004,11 +1036,11 @@ export class BasicCrawler {
|
|
|
1004
1036
|
}
|
|
1005
1037
|
async useState(defaultValue = {}) {
|
|
1006
1038
|
const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
1007
|
-
if (this
|
|
1008
|
-
const stateKey = `${BasicCrawler
|
|
1039
|
+
if (this.#identity.hasExplicitId) {
|
|
1040
|
+
const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
|
|
1009
1041
|
return kvs.getAutoSavedValue(stateKey, defaultValue);
|
|
1010
1042
|
}
|
|
1011
|
-
BasicCrawler.#useStateAnonymousIndices.add(this
|
|
1043
|
+
BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
|
|
1012
1044
|
if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
|
|
1013
1045
|
serviceLocator
|
|
1014
1046
|
.getLogger()
|
|
@@ -1017,38 +1049,36 @@ export class BasicCrawler {
|
|
|
1017
1049
|
'To fix this, provide a unique `id` option to each crawler instance. \n' +
|
|
1018
1050
|
'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
|
|
1019
1051
|
}
|
|
1020
|
-
return kvs.getAutoSavedValue(BasicCrawler
|
|
1052
|
+
return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
|
|
1021
1053
|
}
|
|
1022
|
-
async getPendingRequestCountApproximation() {
|
|
1054
|
+
async #getPendingRequestCountApproximation() {
|
|
1023
1055
|
return (await this.requestManager?.getPendingCount()) ?? 0;
|
|
1024
1056
|
}
|
|
1025
|
-
async calculateEnqueuedRequestLimit(explicitLimit) {
|
|
1026
|
-
if (this
|
|
1057
|
+
async #calculateEnqueuedRequestLimit(explicitLimit) {
|
|
1058
|
+
if (this.#maxRequestsPerCrawl === undefined) {
|
|
1027
1059
|
return explicitLimit;
|
|
1028
1060
|
}
|
|
1029
|
-
const limit = Math.max(0, this
|
|
1061
|
+
const limit = Math.max(0, this.#maxRequestsPerCrawl -
|
|
1062
|
+
this.#handledRequestsCount -
|
|
1063
|
+
(await this.#getPendingRequestCountApproximation()));
|
|
1030
1064
|
return Math.min(limit, explicitLimit ?? Infinity);
|
|
1031
1065
|
}
|
|
1032
|
-
async handleSkippedRequest(options) {
|
|
1066
|
+
async #handleSkippedRequest(options) {
|
|
1033
1067
|
// A skipped request is a *successful* outcome, but the interrupt still unwinds through the
|
|
1034
1068
|
// transaction scope, which rolls back - so the skip bookkeeping must write directly.
|
|
1035
1069
|
await withDirectStorageAccess(async () => {
|
|
1070
|
+
// Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
|
|
1071
|
+
// against the logger for the crawler's lifetime rather than per run.
|
|
1036
1072
|
if (options.reason === 'limit') {
|
|
1037
|
-
this.
|
|
1038
|
-
`${this
|
|
1073
|
+
this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
|
|
1074
|
+
`${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
|
|
1039
1075
|
}
|
|
1040
1076
|
if (options.reason === 'depth') {
|
|
1041
|
-
this.
|
|
1077
|
+
this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
|
|
1042
1078
|
}
|
|
1043
|
-
await this
|
|
1079
|
+
await this.#onSkippedRequest?.(options);
|
|
1044
1080
|
});
|
|
1045
1081
|
}
|
|
1046
|
-
logOncePerRun(key, message, level = 'info') {
|
|
1047
|
-
if (!this.#loggedPerRun.has(key)) {
|
|
1048
|
-
this.log[level](message);
|
|
1049
|
-
this.#loggedPerRun.add(key);
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
1082
|
/**
|
|
1053
1083
|
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
|
|
1054
1084
|
* adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
|
|
@@ -1077,25 +1107,47 @@ export class BasicCrawler {
|
|
|
1077
1107
|
// its own per-item validation below, and validating an absent label/userData here would spuriously
|
|
1078
1108
|
// check them against a registered default-route schema.
|
|
1079
1109
|
if (options.label !== undefined || options.userData !== undefined) {
|
|
1080
|
-
await this
|
|
1110
|
+
await this.#validateRequestUserData({ label: options.label, userData: options.userData });
|
|
1081
1111
|
}
|
|
1082
|
-
const requestLimit = await this
|
|
1112
|
+
const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
|
|
1083
1113
|
const strategy = options.strategy ?? EnqueueStrategy.All;
|
|
1084
|
-
const urlExcludePatternObjects = options.exclude?.length
|
|
1085
|
-
|
|
1086
|
-
: [];
|
|
1087
|
-
const urlPatternObjects = options.include?.length
|
|
1088
|
-
? constructUrlPatternObjects(options.include)
|
|
1089
|
-
: [];
|
|
1114
|
+
const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
|
|
1115
|
+
const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
|
|
1090
1116
|
// The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
|
|
1091
1117
|
// (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
|
|
1092
|
-
const
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
const
|
|
1096
|
-
const maxCrawlDepth = this.maxCrawlDepth;
|
|
1097
|
-
const validateRequestUserData = this.validateRequestUserData.bind(this);
|
|
1118
|
+
const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
|
|
1119
|
+
const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
|
|
1120
|
+
const maxCrawlDepth = this.#maxCrawlDepth;
|
|
1121
|
+
const validateRequestUserData = this.#validateRequestUserData.bind(this);
|
|
1098
1122
|
const allSkipped = [];
|
|
1123
|
+
// A skipped source (which can carry arbitrary userData) is only retained if something reads it -
|
|
1124
|
+
// otherwise the URL alone is enough to build the callback argument and to log with.
|
|
1125
|
+
const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
|
|
1126
|
+
const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
|
|
1127
|
+
const reportSkippedRequests = async () => {
|
|
1128
|
+
const skippedRequests = allSkipped.splice(0);
|
|
1129
|
+
if (skippedRequests.length === 0) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
const skippedRobotsUrls = skippedRequests
|
|
1133
|
+
.filter((s) => s.reason === 'robotsTxt')
|
|
1134
|
+
.map(({ source }) => (typeof source === 'string' ? source : source.url));
|
|
1135
|
+
if (skippedRobotsUrls.length > 0) {
|
|
1136
|
+
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
|
|
1137
|
+
}
|
|
1138
|
+
// Only log the limit message when an explicit `limit` was passed (not the internal
|
|
1139
|
+
// `maxRequestsPerCrawl`-derived one), and only once per call.
|
|
1140
|
+
if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
|
|
1141
|
+
this.log.info(requestLimit === options.limit
|
|
1142
|
+
? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
|
|
1143
|
+
: `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
|
|
1144
|
+
}
|
|
1145
|
+
await Promise.all(skippedRequests.map(async ({ source, reason }) => {
|
|
1146
|
+
const args = createSkippedRequestArgs(source, reason);
|
|
1147
|
+
await this.#handleSkippedRequest(args);
|
|
1148
|
+
await options.onSkippedRequest?.(args);
|
|
1149
|
+
}));
|
|
1150
|
+
};
|
|
1099
1151
|
async function* filteredRequests() {
|
|
1100
1152
|
for await (const request of requests) {
|
|
1101
1153
|
const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
|
|
@@ -1103,24 +1155,31 @@ export class BasicCrawler {
|
|
|
1103
1155
|
continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
|
|
1104
1156
|
}
|
|
1105
1157
|
if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
|
|
1106
|
-
allSkipped.push({
|
|
1158
|
+
allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
|
|
1107
1159
|
continue;
|
|
1108
1160
|
}
|
|
1109
1161
|
if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
|
|
1110
|
-
allSkipped.push({
|
|
1162
|
+
allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
|
|
1111
1163
|
continue;
|
|
1112
1164
|
}
|
|
1113
|
-
const
|
|
1165
|
+
const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
|
|
1114
1166
|
// Filter by user patterns first (with exclude)...
|
|
1115
|
-
let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy,
|
|
1167
|
+
let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
|
|
1116
1168
|
// ...then filter by the enqueue strategy (making this an AND check)
|
|
1117
|
-
|
|
1169
|
+
if (strategyBaseUrl) {
|
|
1170
|
+
filtered = filtered.filter((opts) => {
|
|
1171
|
+
const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
|
|
1172
|
+
if (!allowed)
|
|
1173
|
+
onSkippedByFilter(opts);
|
|
1174
|
+
return allowed;
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1118
1177
|
if (filtered.length === 0) {
|
|
1119
1178
|
continue;
|
|
1120
1179
|
}
|
|
1121
1180
|
let [finalOptions] = filtered;
|
|
1122
1181
|
if (options.transformRequestFunction) {
|
|
1123
|
-
const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({
|
|
1182
|
+
const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
|
|
1124
1183
|
if (transformed.length === 0) {
|
|
1125
1184
|
continue;
|
|
1126
1185
|
}
|
|
@@ -1132,7 +1191,7 @@ export class BasicCrawler {
|
|
|
1132
1191
|
}
|
|
1133
1192
|
const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
|
|
1134
1193
|
forefront: options.forefront,
|
|
1135
|
-
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
|
|
1194
|
+
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
|
|
1136
1195
|
batchSize: options.batchSize,
|
|
1137
1196
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
|
|
1138
1197
|
maxNewRequests: requestLimit,
|
|
@@ -1140,31 +1199,24 @@ export class BasicCrawler {
|
|
|
1140
1199
|
// Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
|
|
1141
1200
|
// explicit `limit` option)
|
|
1142
1201
|
for (const request of result.requestsOverLimit ?? []) {
|
|
1143
|
-
allSkipped.push({
|
|
1144
|
-
}
|
|
1145
|
-
if (allSkipped.length > 0) {
|
|
1146
|
-
const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
|
|
1147
|
-
if (skippedRobotsUrls.length > 0) {
|
|
1148
|
-
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
|
|
1149
|
-
}
|
|
1150
|
-
// Only log the limit message when an explicit `limit` was passed (not the internal
|
|
1151
|
-
// `maxRequestsPerCrawl`-derived one), and only once per call.
|
|
1152
|
-
if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
|
|
1153
|
-
this.log.info(requestLimit === options.limit
|
|
1154
|
-
? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
|
|
1155
|
-
: `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
|
|
1156
|
-
}
|
|
1157
|
-
await Promise.all(allSkipped.map(async ({ url, reason }) => {
|
|
1158
|
-
await this.handleSkippedRequest({ url, reason });
|
|
1159
|
-
await options.onSkippedRequest?.({ url, reason });
|
|
1160
|
-
}));
|
|
1202
|
+
allSkipped.push({ source: request, reason: 'limit' });
|
|
1161
1203
|
}
|
|
1162
|
-
|
|
1204
|
+
await reportSkippedRequests();
|
|
1205
|
+
const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
|
|
1206
|
+
await reportSkippedRequests();
|
|
1207
|
+
return addedRequests;
|
|
1208
|
+
});
|
|
1209
|
+
// Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
|
|
1210
|
+
// when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
|
|
1211
|
+
// Background failures are already logged by `drainRequestBatches`.
|
|
1212
|
+
void waitForAllRequestsToBeAdded.catch(() => { });
|
|
1213
|
+
return { ...result, waitForAllRequestsToBeAdded };
|
|
1163
1214
|
}
|
|
1164
1215
|
/**
|
|
1165
1216
|
* Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
|
|
1166
1217
|
*/
|
|
1167
1218
|
async pushData(data, datasetIdentifier) {
|
|
1219
|
+
tryCancel();
|
|
1168
1220
|
const dataset = await this.getDataset(datasetIdentifier);
|
|
1169
1221
|
return dataset.pushData(data);
|
|
1170
1222
|
}
|
|
@@ -1233,31 +1285,25 @@ export class BasicCrawler {
|
|
|
1233
1285
|
* Initializes the crawler.
|
|
1234
1286
|
*/
|
|
1235
1287
|
async init() {
|
|
1288
|
+
const run = this.#liveRun;
|
|
1289
|
+
if (run === undefined) {
|
|
1290
|
+
throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
|
|
1291
|
+
}
|
|
1236
1292
|
const eventManager = serviceLocator.getEventManager();
|
|
1237
1293
|
if (!eventManager.isInitialized()) {
|
|
1238
1294
|
await eventManager.init();
|
|
1239
|
-
|
|
1295
|
+
run.ownsEventManager = true;
|
|
1240
1296
|
}
|
|
1241
1297
|
// Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
|
|
1242
1298
|
// floored per request so it will not actually cut them short, but the configured value is then effectively
|
|
1243
1299
|
// ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
|
|
1244
1300
|
// navigation timeout only after `super()`.
|
|
1245
|
-
const phasesMillis = this.getNavigationTimeoutMillis() + this
|
|
1301
|
+
const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
|
|
1246
1302
|
if (this.internalTimeoutMillis < phasesMillis) {
|
|
1247
1303
|
this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
|
|
1248
1304
|
`and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
|
|
1249
1305
|
`so it does not cut them short.`);
|
|
1250
1306
|
}
|
|
1251
|
-
// An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
|
|
1252
|
-
// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
|
|
1253
|
-
// scaling. An injected one is long-lived and its lifecycle belongs to the caller.
|
|
1254
|
-
this.#concurrencySystemDep = this.#resolveConcurrencySystem();
|
|
1255
|
-
await this.#concurrencySystemDep.ifOwned((system) => system.start());
|
|
1256
|
-
this.#autoscaledPool = new AutoscaledPool({
|
|
1257
|
-
...this.taskLoopOptions,
|
|
1258
|
-
concurrencySystem: this.#concurrencySystemDep.value,
|
|
1259
|
-
consumer: this.identity,
|
|
1260
|
-
});
|
|
1261
1307
|
await this.getRequestManager();
|
|
1262
1308
|
}
|
|
1263
1309
|
/**
|
|
@@ -1274,9 +1320,9 @@ export class BasicCrawler {
|
|
|
1274
1320
|
* request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
|
|
1275
1321
|
* whichever is larger: the configured internal timeout, or this request's combined phase budget.
|
|
1276
1322
|
*/
|
|
1277
|
-
async withRequestTimeout(crawlingContext, work) {
|
|
1323
|
+
async #withRequestTimeout(crawlingContext, work) {
|
|
1278
1324
|
const { request } = crawlingContext;
|
|
1279
|
-
const phasesMillis = this.getNavigationTimeoutMillis() + this
|
|
1325
|
+
const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
|
|
1280
1326
|
const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
|
|
1281
1327
|
await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
|
|
1282
1328
|
}
|
|
@@ -1287,14 +1333,14 @@ export class BasicCrawler {
|
|
|
1287
1333
|
* @param label The request's route label, or `undefined` for the default route / no specific request.
|
|
1288
1334
|
* @param fallbackMillis Timeout to use when no route overrides it.
|
|
1289
1335
|
*/
|
|
1290
|
-
resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this
|
|
1291
|
-
return this
|
|
1336
|
+
#resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
|
|
1337
|
+
return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
|
|
1292
1338
|
}
|
|
1293
1339
|
/**
|
|
1294
1340
|
* The timeout the router route with the given label asked for, or `undefined` when it did not override one
|
|
1295
1341
|
* (or the request handler is not a router at all).
|
|
1296
1342
|
*/
|
|
1297
|
-
getRouteTimeoutMillis(label) {
|
|
1343
|
+
#getRouteTimeoutMillis(label) {
|
|
1298
1344
|
const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
|
|
1299
1345
|
if (typeof getTimeoutSecs !== 'function') {
|
|
1300
1346
|
return undefined;
|
|
@@ -1303,14 +1349,15 @@ export class BasicCrawler {
|
|
|
1303
1349
|
return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
|
|
1304
1350
|
}
|
|
1305
1351
|
async runRequestHandler(crawlingContext) {
|
|
1306
|
-
const timeoutMillis = this
|
|
1352
|
+
const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
|
|
1307
1353
|
await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
|
|
1308
1354
|
}
|
|
1309
1355
|
/**
|
|
1310
1356
|
* Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
|
|
1311
|
-
* Deliberately does **not** commit on return -
|
|
1312
|
-
*
|
|
1357
|
+
* Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
|
|
1358
|
+
* normal return says nothing about success. `handleRequest` commits on success.
|
|
1313
1359
|
*/
|
|
1360
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
|
|
1314
1361
|
async runInStorageTransaction(callback) {
|
|
1315
1362
|
if (!this.#transactionalStorageEnabled) {
|
|
1316
1363
|
return callback();
|
|
@@ -1327,7 +1374,7 @@ export class BasicCrawler {
|
|
|
1327
1374
|
}
|
|
1328
1375
|
finally {
|
|
1329
1376
|
if (transaction.state === 'open') {
|
|
1330
|
-
// `handleRequest` commits or rolls back on every normal path, so an open transaction on
|
|
1377
|
+
// `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
|
|
1331
1378
|
// a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
|
|
1332
1379
|
// expected. Either way, discard the unvalidated writes; only the former is worth flagging.
|
|
1333
1380
|
if (!threw) {
|
|
@@ -1350,7 +1397,7 @@ export class BasicCrawler {
|
|
|
1350
1397
|
throw new SessionError(`Request blocked - received ${statusCode} status code.`);
|
|
1351
1398
|
}
|
|
1352
1399
|
}
|
|
1353
|
-
async isAllowedBasedOnRobotsTxtFile(url) {
|
|
1400
|
+
async #isAllowedBasedOnRobotsTxtFile(url) {
|
|
1354
1401
|
if (!this.#respectRobotsTxtFile) {
|
|
1355
1402
|
return true;
|
|
1356
1403
|
}
|
|
@@ -1359,45 +1406,52 @@ export class BasicCrawler {
|
|
|
1359
1406
|
if (robotsTxtFile) {
|
|
1360
1407
|
const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
|
|
1361
1408
|
if (crawlDelay !== undefined) {
|
|
1362
|
-
this
|
|
1409
|
+
this.#applyCrawlDelay(url, crawlDelay);
|
|
1363
1410
|
}
|
|
1364
1411
|
}
|
|
1365
1412
|
return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
|
|
1366
1413
|
}
|
|
1367
1414
|
/**
|
|
1368
|
-
* Records an HTTP 429 against the URL's domain so the request manager can
|
|
1415
|
+
* Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
|
|
1369
1416
|
*
|
|
1370
1417
|
* @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
|
|
1371
|
-
* @returns `true` if
|
|
1418
|
+
* @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
|
|
1372
1419
|
* {@link RequestThrottledError} rather than treating the response as a blocked session.
|
|
1373
1420
|
*/
|
|
1374
1421
|
recordDomainRateLimit(url, retryAfterHeader) {
|
|
1375
|
-
if (
|
|
1376
|
-
|
|
1422
|
+
if (this.requestManager?.recordPacingSignal({
|
|
1423
|
+
reason: 'rateLimited',
|
|
1424
|
+
url,
|
|
1425
|
+
waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
|
|
1426
|
+
})) {
|
|
1377
1427
|
return true;
|
|
1378
1428
|
}
|
|
1379
1429
|
const domain = hostnameOrUrl(url);
|
|
1380
|
-
this.
|
|
1381
|
-
'so the response is handled like any other, with no per-domain delay. ' +
|
|
1382
|
-
|
|
1383
|
-
'
|
|
1430
|
+
this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
|
|
1431
|
+
'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
|
|
1432
|
+
`\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
|
|
1433
|
+
'`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
|
|
1384
1434
|
return false;
|
|
1385
1435
|
}
|
|
1386
|
-
/**
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1436
|
+
/** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
|
|
1437
|
+
#applyCrawlDelay(url, delaySeconds) {
|
|
1438
|
+
// robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
|
|
1439
|
+
// host share a clock).
|
|
1440
|
+
if (this.requestManager?.recordPacingSignal({
|
|
1441
|
+
reason: 'minInterval',
|
|
1442
|
+
url,
|
|
1443
|
+
intervalMs: delaySeconds * 1000,
|
|
1444
|
+
scope: 'hostname',
|
|
1445
|
+
})) {
|
|
1394
1446
|
return;
|
|
1395
1447
|
}
|
|
1396
1448
|
const domain = hostnameOrUrl(url);
|
|
1397
|
-
this.
|
|
1398
|
-
'
|
|
1399
|
-
|
|
1449
|
+
this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
|
|
1450
|
+
'manager does not pace that domain, so its requests will not be paced. Set ' +
|
|
1451
|
+
`\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
|
|
1452
|
+
'`requestManager`.');
|
|
1400
1453
|
}
|
|
1454
|
+
/** @internal */
|
|
1401
1455
|
async getRobotsTxtFileForUrl(url) {
|
|
1402
1456
|
if (!this.#respectRobotsTxtFile) {
|
|
1403
1457
|
return undefined;
|
|
@@ -1408,7 +1462,15 @@ export class BasicCrawler {
|
|
|
1408
1462
|
if (cachedRobotsTxtFile) {
|
|
1409
1463
|
return cachedRobotsTxtFile;
|
|
1410
1464
|
}
|
|
1411
|
-
|
|
1465
|
+
// Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
|
|
1466
|
+
// fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
|
|
1467
|
+
// alone would then decide whether everything is allowed or disallowed.
|
|
1468
|
+
const robotsTxtFile = await RobotsTxtFile.find(url, {
|
|
1469
|
+
httpClient: this.httpClient,
|
|
1470
|
+
session: await this.sessionPool.getSession(),
|
|
1471
|
+
timeoutMillis: 30_000,
|
|
1472
|
+
logger: this.log,
|
|
1473
|
+
});
|
|
1412
1474
|
this.#robotsTxtFileCache.add(origin, robotsTxtFile);
|
|
1413
1475
|
return robotsTxtFile;
|
|
1414
1476
|
}
|
|
@@ -1417,10 +1479,10 @@ export class BasicCrawler {
|
|
|
1417
1479
|
return undefined;
|
|
1418
1480
|
}
|
|
1419
1481
|
}
|
|
1420
|
-
async pauseOnMigration() {
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
await
|
|
1482
|
+
async #pauseOnMigration() {
|
|
1483
|
+
const run = this.#dispatchingRun;
|
|
1484
|
+
if (run) {
|
|
1485
|
+
await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
|
|
1424
1486
|
if (err.message.includes('running tasks did not finish')) {
|
|
1425
1487
|
this.log.error('The crawler was paused due to migration to another host, ' +
|
|
1426
1488
|
"but some requests did not finish in time. Those requests' results may be duplicated.");
|
|
@@ -1430,106 +1492,113 @@ export class BasicCrawler {
|
|
|
1430
1492
|
}
|
|
1431
1493
|
});
|
|
1432
1494
|
}
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
if (this.requestManager?.persistState) {
|
|
1437
|
-
if (await this.requestManager.isFinished())
|
|
1438
|
-
return;
|
|
1439
|
-
await this.requestManager.persistState().catch((err) => {
|
|
1440
|
-
if (err.message.includes('Cannot persist state.')) {
|
|
1441
|
-
this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
|
|
1442
|
-
'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
|
|
1443
|
-
'constructor to ensure your crawling state is persisted through host migrations and restarts.');
|
|
1444
|
-
}
|
|
1445
|
-
else {
|
|
1446
|
-
this.log.exception(err, 'An unexpected error occurred when the crawler ' +
|
|
1447
|
-
"attempted to persist its request list's state.");
|
|
1448
|
-
}
|
|
1449
|
-
});
|
|
1450
|
-
}
|
|
1451
|
-
})();
|
|
1452
|
-
await Promise.all([requestManagerPersistPromise, this.statistics.persistState?.()]);
|
|
1495
|
+
// Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
|
|
1496
|
+
// the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
|
|
1497
|
+
await this.statistics.persistState?.();
|
|
1453
1498
|
}
|
|
1454
1499
|
/**
|
|
1455
1500
|
* Fetches the next request to process from the underlying request provider.
|
|
1456
1501
|
*/
|
|
1457
|
-
async fetchNextRequest() {
|
|
1502
|
+
async #fetchNextRequest() {
|
|
1458
1503
|
if (this.requestManager === undefined) {
|
|
1459
1504
|
throw new Error(`fetchNextRequest called on an uninitialized crawler`);
|
|
1460
1505
|
}
|
|
1461
1506
|
return this.requestManager.fetchNextRequest();
|
|
1462
1507
|
}
|
|
1463
|
-
/**
|
|
1464
|
-
|
|
1508
|
+
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
1509
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1510
|
+
async handleRequest(crawlingContext, requestSource) {
|
|
1465
1511
|
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
|
|
1466
1512
|
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
|
|
1467
1513
|
// execute (and re-report) on top of a request the crawler has already moved past.
|
|
1468
1514
|
if (crawlingContext[timeoutExpiredKey]?.()) {
|
|
1469
1515
|
return;
|
|
1470
1516
|
}
|
|
1471
|
-
const
|
|
1517
|
+
const { request } = crawlingContext;
|
|
1518
|
+
request.state = RequestState.REQUEST_HANDLER;
|
|
1519
|
+
await this.runRequestHandler(crawlingContext);
|
|
1520
|
+
// Commit *before* marking the request as handled, so a commit failure fails the request and
|
|
1521
|
+
// it is retried. This also closes the transaction, so everything below passes through.
|
|
1472
1522
|
// Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
|
|
1473
|
-
|
|
1474
|
-
|
|
1523
|
+
await currentStorageTransaction()?.commit();
|
|
1524
|
+
await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1525
|
+
this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
|
|
1526
|
+
// reclaim session if request finishes successfully
|
|
1527
|
+
request.state = RequestState.DONE;
|
|
1528
|
+
crawlingContext.session.markGood();
|
|
1529
|
+
}
|
|
1530
|
+
/**
|
|
1531
|
+
* Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
|
|
1532
|
+
* or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
|
|
1533
|
+
* error handler has seen the context.
|
|
1534
|
+
*/
|
|
1535
|
+
async #handleRequestFailure(rawError, crawlingContext, requestSource) {
|
|
1536
|
+
const { request } = crawlingContext;
|
|
1537
|
+
// Roll back *before* any error handler runs - error handlers write to real storage precisely
|
|
1538
|
+
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
|
|
1539
|
+
currentStorageTransaction()?.rollback();
|
|
1540
|
+
const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
|
|
1475
1541
|
try {
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
|
|
1498
|
-
}
|
|
1542
|
+
const unwrappedError = this.#unwrapError(rawError);
|
|
1543
|
+
// A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
|
|
1544
|
+
const err = unwrappedError instanceof ContextPipelineInterruptedError
|
|
1545
|
+
? await this.#trySkipRequest(unwrappedError, request, requestSource)
|
|
1546
|
+
: unwrappedError;
|
|
1547
|
+
if (err) {
|
|
1548
|
+
await this.#runErrorHandler(err, crawlingContext, requestSource);
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
finally {
|
|
1552
|
+
resumeRequestTimeout?.();
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
async #runErrorHandler(err, context, requestSource) {
|
|
1556
|
+
const { request } = context;
|
|
1557
|
+
const retryCountBefore = request.retryCount;
|
|
1558
|
+
try {
|
|
1559
|
+
request.state = RequestState.ERROR_HANDLER;
|
|
1560
|
+
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1561
|
+
// The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
|
|
1562
|
+
if (request.state !== RequestState.SKIPPED) {
|
|
1499
1563
|
request.state = RequestState.DONE;
|
|
1500
1564
|
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1565
|
+
}
|
|
1566
|
+
catch (secondaryError) {
|
|
1567
|
+
const unwrappedSecondaryError = this.#unwrapError(secondaryError);
|
|
1568
|
+
const timedOut = unwrappedSecondaryError instanceof TimeoutError;
|
|
1569
|
+
request.state = RequestState.ERROR;
|
|
1570
|
+
if (timedOut) {
|
|
1571
|
+
// Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
|
|
1572
|
+
// retry is counted, so that an error handler that always times out does not retry it forever.
|
|
1573
|
+
this.log.warning(unwrappedSecondaryError.message);
|
|
1574
|
+
if (request.retryCount === retryCountBefore) {
|
|
1575
|
+
request.retryCount++;
|
|
1511
1576
|
}
|
|
1512
|
-
request.state = RequestState.ERROR;
|
|
1513
|
-
throw unwrappedSecondaryError;
|
|
1514
1577
|
}
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1578
|
+
else if (!(unwrappedSecondaryError instanceof CriticalError)) {
|
|
1579
|
+
// avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
|
|
1580
|
+
const apifySpecific = process.env.APIFY_IS_AT_HOME
|
|
1581
|
+
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
|
|
1582
|
+
: '';
|
|
1583
|
+
this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
|
|
1584
|
+
`This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
|
|
1519
1585
|
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
await requestSource.reclaimRequest(request);
|
|
1528
|
-
}
|
|
1529
|
-
catch {
|
|
1530
|
-
// The request was never in progress, or could not be reclaimed. Either way it's fine.
|
|
1531
|
-
}
|
|
1586
|
+
// Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
|
|
1587
|
+
// CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
|
|
1588
|
+
try {
|
|
1589
|
+
await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
|
|
1590
|
+
}
|
|
1591
|
+
catch {
|
|
1592
|
+
// The request was never in progress, or could not be reclaimed. Either way it's fine.
|
|
1532
1593
|
}
|
|
1594
|
+
if (!timedOut) {
|
|
1595
|
+
throw unwrappedSecondaryError;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
// decrease the session score if the request fails (but the error handler did not throw);
|
|
1599
|
+
// skip when the error is a SessionError, which already retired the session
|
|
1600
|
+
if (!this.#errorAbsolvesSession(err)) {
|
|
1601
|
+
context.session?.markBad();
|
|
1533
1602
|
}
|
|
1534
1603
|
}
|
|
1535
1604
|
/**
|
|
@@ -1541,6 +1610,11 @@ export class BasicCrawler {
|
|
|
1541
1610
|
if (typeof request === 'string') {
|
|
1542
1611
|
yield { url: request, crawlDepth: newRequestDepth };
|
|
1543
1612
|
}
|
|
1613
|
+
else if (request instanceof Request) {
|
|
1614
|
+
const crawlee = (request.userData.__crawlee ??= {});
|
|
1615
|
+
crawlee.crawlDepth ??= newRequestDepth;
|
|
1616
|
+
yield request;
|
|
1617
|
+
}
|
|
1544
1618
|
else {
|
|
1545
1619
|
request.crawlDepth ??= newRequestDepth;
|
|
1546
1620
|
yield request;
|
|
@@ -1551,7 +1625,7 @@ export class BasicCrawler {
|
|
|
1551
1625
|
* Run async callback with given timeout and retry. Returns the result of the callback.
|
|
1552
1626
|
* @ignore
|
|
1553
1627
|
*/
|
|
1554
|
-
async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
|
|
1628
|
+
async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
|
|
1555
1629
|
try {
|
|
1556
1630
|
return await addTimeoutToPromise(handler, timeout, error);
|
|
1557
1631
|
}
|
|
@@ -1559,65 +1633,86 @@ export class BasicCrawler {
|
|
|
1559
1633
|
if (retried <= maxRetries) {
|
|
1560
1634
|
// we retry on any error, not just timeout
|
|
1561
1635
|
this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
|
|
1562
|
-
return this
|
|
1636
|
+
return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
|
|
1563
1637
|
}
|
|
1564
1638
|
throw e;
|
|
1565
1639
|
}
|
|
1566
1640
|
}
|
|
1567
1641
|
/**
|
|
1568
|
-
*
|
|
1642
|
+
* Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
|
|
1643
|
+
* wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
|
|
1569
1644
|
*/
|
|
1570
|
-
async isTaskReadyFunction() {
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1645
|
+
async #isTaskReadyFunction() {
|
|
1646
|
+
if (this.requestManager === undefined) {
|
|
1647
|
+
return false;
|
|
1648
|
+
}
|
|
1649
|
+
const state = await this.requestManager.checkReadiness();
|
|
1650
|
+
if (state.status === 'waiting' && state.readyAt !== undefined) {
|
|
1651
|
+
this.#run?.scheduleWake(state.readyAt);
|
|
1652
|
+
}
|
|
1653
|
+
return state.status === 'ready';
|
|
1578
1654
|
}
|
|
1579
1655
|
/**
|
|
1580
1656
|
* Unwraps errors thrown by the context pipeline to get the actual user error.
|
|
1581
|
-
* RequestHandlerError and
|
|
1657
|
+
* RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
|
|
1582
1658
|
*/
|
|
1583
|
-
unwrapError(error) {
|
|
1659
|
+
#unwrapError(error) {
|
|
1584
1660
|
if (error instanceof RequestHandlerError ||
|
|
1585
1661
|
error instanceof ContextPipelineInitializationError ||
|
|
1586
1662
|
error instanceof ContextPipelineCleanupError) {
|
|
1587
|
-
return this
|
|
1663
|
+
return this.#unwrapError(error.cause);
|
|
1588
1664
|
}
|
|
1589
1665
|
return error;
|
|
1590
1666
|
}
|
|
1591
1667
|
/**
|
|
1592
1668
|
* Handles errors thrown by user provided requestHandler()
|
|
1593
|
-
*
|
|
1594
|
-
* @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
|
|
1595
1669
|
*/
|
|
1596
|
-
|
|
1670
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1671
|
+
async requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
1672
|
+
const { request } = crawlingContext;
|
|
1597
1673
|
if (error instanceof RequestThrottledError) {
|
|
1598
|
-
// The domain told us to come back later, so the request was never really attempted. Put it back
|
|
1599
|
-
// without recording a failure - it costs neither a retry nor session reputation.
|
|
1674
|
+
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1675
|
+
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
1600
1676
|
this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
|
|
1601
1677
|
id: request.id,
|
|
1602
1678
|
url: request.url,
|
|
1603
1679
|
});
|
|
1604
|
-
await source.reclaimRequest(request, { forefront:
|
|
1680
|
+
await source.reclaimRequest(request, { forefront: true });
|
|
1605
1681
|
return;
|
|
1606
1682
|
}
|
|
1607
1683
|
request.pushErrorMessage(error);
|
|
1608
1684
|
if (error instanceof CriticalError) {
|
|
1609
1685
|
throw error;
|
|
1610
1686
|
}
|
|
1611
|
-
const shouldRetryRequest = this
|
|
1687
|
+
const shouldRetryRequest = this.#canRequestBeRetried(request, error);
|
|
1612
1688
|
if (shouldRetryRequest) {
|
|
1613
1689
|
await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
|
|
1614
|
-
|
|
1615
|
-
|
|
1690
|
+
try {
|
|
1691
|
+
await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
|
|
1692
|
+
error);
|
|
1693
|
+
}
|
|
1694
|
+
catch (handlerError) {
|
|
1695
|
+
if (handlerError instanceof ContextPipelineInterruptedError) {
|
|
1696
|
+
const skipError = await this.#trySkipRequest(handlerError, request, source);
|
|
1697
|
+
if (!skipError) {
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
// Retry for the original error. Routing `skipError` through `errorHandler` again
|
|
1701
|
+
// would loop forever with a handler that always skips.
|
|
1702
|
+
this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
|
|
1703
|
+
error: skipError.message,
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
else {
|
|
1707
|
+
throw handlerError;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1616
1710
|
if (error instanceof SessionError) {
|
|
1617
1711
|
crawlingContext.session?.retire();
|
|
1618
1712
|
}
|
|
1619
1713
|
if (!request.noRetry) {
|
|
1620
1714
|
request.retryCount++;
|
|
1715
|
+
this.statistics.registerRetry?.(request.retryCount);
|
|
1621
1716
|
const { url, retryCount, id } = request;
|
|
1622
1717
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
1623
1718
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
@@ -1627,7 +1722,7 @@ export class BasicCrawler {
|
|
|
1627
1722
|
url,
|
|
1628
1723
|
retryCount,
|
|
1629
1724
|
});
|
|
1630
|
-
await source.reclaimRequest(request
|
|
1725
|
+
await source.reclaimRequest(request);
|
|
1631
1726
|
return;
|
|
1632
1727
|
}
|
|
1633
1728
|
}
|
|
@@ -1648,18 +1743,38 @@ export class BasicCrawler {
|
|
|
1648
1743
|
// or failed more than retryCount times and will not be retried anymore.
|
|
1649
1744
|
// Mark the request as failed and do not retry.
|
|
1650
1745
|
await source.markRequestAsHandled(request);
|
|
1651
|
-
this.statistics.
|
|
1746
|
+
this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
|
|
1652
1747
|
await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
|
|
1653
1748
|
}
|
|
1749
|
+
/** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
|
|
1750
|
+
async #trySkipRequest(error, request, source) {
|
|
1751
|
+
try {
|
|
1752
|
+
await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1753
|
+
request.state = RequestState.SKIPPED;
|
|
1754
|
+
request.noRetry = true;
|
|
1755
|
+
await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1756
|
+
this.statistics.discardRequestRecord(request.id || request.uniqueKey);
|
|
1757
|
+
return undefined;
|
|
1758
|
+
}
|
|
1759
|
+
catch (settleError) {
|
|
1760
|
+
return this.#unwrapError(settleError);
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1654
1764
|
async handleFailedRequestHandler(crawlingContext, error) {
|
|
1655
1765
|
// Always log the last error regardless if the user provided a failedRequestHandler
|
|
1656
1766
|
const { id, url, method, uniqueKey } = crawlingContext.request;
|
|
1657
1767
|
const message = this.getMessageFromError(error, true);
|
|
1658
1768
|
this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
|
|
1659
|
-
|
|
1660
|
-
await this
|
|
1769
|
+
try {
|
|
1770
|
+
await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
|
|
1661
1771
|
error);
|
|
1662
1772
|
}
|
|
1773
|
+
catch (handlerError) {
|
|
1774
|
+
if (!(handlerError instanceof ContextPipelineInterruptedError))
|
|
1775
|
+
throw handlerError;
|
|
1776
|
+
this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
|
|
1777
|
+
}
|
|
1663
1778
|
}
|
|
1664
1779
|
/**
|
|
1665
1780
|
* Resolves the most verbose error message from a thrown error
|
|
@@ -1674,7 +1789,8 @@ export class BasicCrawler {
|
|
|
1674
1789
|
const baseDir = process.cwd();
|
|
1675
1790
|
const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
|
|
1676
1791
|
if (error instanceof TimeoutError) {
|
|
1677
|
-
|
|
1792
|
+
// stack in timeout errors does not really help
|
|
1793
|
+
return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
|
|
1678
1794
|
}
|
|
1679
1795
|
return process.env.CRAWLEE_VERBOSE_LOG || forceStack
|
|
1680
1796
|
? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
|
|
@@ -1684,10 +1800,12 @@ export class BasicCrawler {
|
|
|
1684
1800
|
* Whether the session should be spared for this error - either because it was already retired, or because the
|
|
1685
1801
|
* failure says nothing about the session (a rate limit is a property of the domain).
|
|
1686
1802
|
*/
|
|
1687
|
-
errorAbsolvesSession(error) {
|
|
1688
|
-
return error instanceof SessionError ||
|
|
1803
|
+
#errorAbsolvesSession(error) {
|
|
1804
|
+
return (error instanceof SessionError ||
|
|
1805
|
+
error instanceof RequestThrottledError ||
|
|
1806
|
+
error instanceof ContextPipelineInterruptedError);
|
|
1689
1807
|
}
|
|
1690
|
-
canRequestBeRetried(request, error) {
|
|
1808
|
+
#canRequestBeRetried(request, error) {
|
|
1691
1809
|
// Request should never be retried, or the error encountered makes it not able to be retried.
|
|
1692
1810
|
if (request.noRetry || error instanceof NonRetryableError) {
|
|
1693
1811
|
return false;
|
|
@@ -1697,29 +1815,46 @@ export class BasicCrawler {
|
|
|
1697
1815
|
return true;
|
|
1698
1816
|
}
|
|
1699
1817
|
// Ensure there are more retries available for the request
|
|
1700
|
-
const maxRequestRetries = request.maxRetries ?? this
|
|
1818
|
+
const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
|
|
1701
1819
|
return request.retryCount < maxRequestRetries;
|
|
1702
1820
|
}
|
|
1703
1821
|
/**
|
|
1704
|
-
*
|
|
1822
|
+
* Ends the run in progress immediately, without waiting for the requests already in flight.
|
|
1705
1823
|
*
|
|
1706
|
-
* This
|
|
1824
|
+
* This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
|
|
1825
|
+
* a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
|
|
1826
|
+
* released by {@link BasicCrawler.destroy|`destroy()`} instead.
|
|
1707
1827
|
*
|
|
1708
1828
|
* To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
|
|
1709
1829
|
*/
|
|
1710
1830
|
async teardown() {
|
|
1831
|
+
const eventManager = serviceLocator.getEventManager();
|
|
1832
|
+
const ownsEventManager = this.#run?.ownsEventManager === true;
|
|
1833
|
+
// First, so that the pool writes its final state once - its teardown stops it listening before the
|
|
1834
|
+
// persistence event below goes out.
|
|
1835
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
|
|
1711
1836
|
// When this crawler initialized the event manager, its close() call emits
|
|
1712
1837
|
// the final persistence event after the crawler-specific state has been
|
|
1713
1838
|
// saved. External event managers still need an explicit event here.
|
|
1714
|
-
if (!
|
|
1715
|
-
|
|
1839
|
+
if (!ownsEventManager) {
|
|
1840
|
+
eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
|
|
1716
1841
|
}
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
await serviceLocator.getEventManager().close();
|
|
1842
|
+
if (ownsEventManager) {
|
|
1843
|
+
await eventManager.close();
|
|
1720
1844
|
}
|
|
1721
|
-
await this.#
|
|
1722
|
-
|
|
1845
|
+
await this.#run?.end();
|
|
1846
|
+
}
|
|
1847
|
+
/**
|
|
1848
|
+
* Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
|
|
1849
|
+
* `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
|
|
1850
|
+
* could otherwise have run again.
|
|
1851
|
+
*/
|
|
1852
|
+
async destroy() {
|
|
1853
|
+
// An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
|
|
1854
|
+
await this.#run?.end();
|
|
1855
|
+
}
|
|
1856
|
+
async [Symbol.asyncDispose]() {
|
|
1857
|
+
await this.destroy();
|
|
1723
1858
|
}
|
|
1724
1859
|
getCookieHeaderFromRequest(request) {
|
|
1725
1860
|
if (request.headers?.Cookie && request.headers?.cookie) {
|
|
@@ -1728,7 +1863,7 @@ export class BasicCrawler {
|
|
|
1728
1863
|
}
|
|
1729
1864
|
return request.headers?.Cookie || request.headers?.cookie || '';
|
|
1730
1865
|
}
|
|
1731
|
-
requestMatchesEnqueueStrategy(request) {
|
|
1866
|
+
#requestMatchesEnqueueStrategy(request) {
|
|
1732
1867
|
// If `skipNavigation` was used, just return `true`
|
|
1733
1868
|
try {
|
|
1734
1869
|
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
|
|
@@ -1741,8 +1876,7 @@ export class BasicCrawler {
|
|
|
1741
1876
|
throw err;
|
|
1742
1877
|
}
|
|
1743
1878
|
const { url, loadedUrl } = request;
|
|
1744
|
-
|
|
1745
|
-
const strategy = request['enqueueStrategy'];
|
|
1879
|
+
const strategy = request.enqueueStrategy;
|
|
1746
1880
|
// No strategy set, so we assume it matches, or it was added outside of enqueueLinks
|
|
1747
1881
|
if (!strategy) {
|
|
1748
1882
|
return true;
|
|
@@ -1751,30 +1885,7 @@ export class BasicCrawler {
|
|
|
1751
1885
|
if (!loadedUrl) {
|
|
1752
1886
|
return true;
|
|
1753
1887
|
}
|
|
1754
|
-
|
|
1755
|
-
const loadedBaseUrl = new URL(loadedUrl);
|
|
1756
|
-
switch (strategy) {
|
|
1757
|
-
case EnqueueStrategy.SameHostname: {
|
|
1758
|
-
return baseUrl.hostname === loadedBaseUrl.hostname;
|
|
1759
|
-
}
|
|
1760
|
-
case EnqueueStrategy.SameDomain: {
|
|
1761
|
-
const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
|
|
1762
|
-
if (baseUrlHostname) {
|
|
1763
|
-
const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
|
|
1764
|
-
return baseUrlHostname === loadedBaseUrlHostname;
|
|
1765
|
-
}
|
|
1766
|
-
// Can happen for IPs, we just check like same origin
|
|
1767
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1768
|
-
}
|
|
1769
|
-
case EnqueueStrategy.SameOrigin: {
|
|
1770
|
-
// Same as hostname, but also checks protocol
|
|
1771
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1772
|
-
}
|
|
1773
|
-
case EnqueueStrategy.All:
|
|
1774
|
-
default: {
|
|
1775
|
-
return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1888
|
+
return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
|
|
1778
1889
|
}
|
|
1779
1890
|
}
|
|
1780
1891
|
/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
|