@crawlee/basic 4.0.0-beta.28 → 4.0.0-beta.281
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +370 -374
- package/internals/basic-crawler.js +1416 -857
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +165 -0
- package/internals/sitemap-request-loader.js +492 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -1,17 +1,48 @@
|
|
|
1
|
-
import { writeFile } from 'node:fs/promises';
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import ow from 'ow';
|
|
9
|
-
import { getDomain } from 'tldts';
|
|
3
|
+
import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
|
|
4
|
+
import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
|
|
5
|
+
import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
|
|
6
|
+
import { RobotsTxtFile } from '@crawlee/utils';
|
|
7
|
+
import { z } from 'zod';
|
|
10
8
|
import { LruCache } from '@apify/datastructures';
|
|
11
|
-
import
|
|
12
|
-
import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
|
|
9
|
+
import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
13
10
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
11
|
+
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
12
|
+
import { mergeCookies } from './cookie-utils.js';
|
|
13
|
+
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
14
|
+
import { CrawlingRequest, RequestState } from './crawling-request.js';
|
|
15
|
+
import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
|
|
16
|
+
import { Statistics } from './crawlers/statistics.js';
|
|
17
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
|
|
18
|
+
import { parseRetryAfterHeader } from './http.js';
|
|
19
|
+
import { CrawlerRun } from './crawler-run.js';
|
|
20
|
+
import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
|
|
14
21
|
import { createSendRequest } from './send-request.js';
|
|
22
|
+
import { Router, validateUserData } from './router.js';
|
|
23
|
+
import { BLOCKED_STATUS_CODES } from './session-pool/consts.js';
|
|
24
|
+
import { Session } from './session-pool/session.js';
|
|
25
|
+
import { SessionPool } from './session-pool/session-pool.js';
|
|
26
|
+
import { ThrottlingRequestManager } from './throttling-request-manager.js';
|
|
27
|
+
class LazyDefaultHttpClient extends BaseHttpClient {
|
|
28
|
+
#delegatePromise;
|
|
29
|
+
constructor(options) {
|
|
30
|
+
super(options);
|
|
31
|
+
this.#delegatePromise = import('@crawlee/impit-client')
|
|
32
|
+
.then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
|
|
33
|
+
.catch(() => {
|
|
34
|
+
(options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
|
|
35
|
+
'Falling back to native fetch — proxy support and browser fingerprinting are unavailable.');
|
|
36
|
+
return new FetchHttpClient(options);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
fetch() {
|
|
40
|
+
throw new Error('LazyDefaultHttpClient delegates `sendRequest` entirely; `fetch` is never called.');
|
|
41
|
+
}
|
|
42
|
+
async sendRequest(...args) {
|
|
43
|
+
return (await this.#delegatePromise).sendRequest(...args);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
15
46
|
/**
|
|
16
47
|
* Since there's no set number of seconds before the container is terminated after
|
|
17
48
|
* a migration event, we need some reasonable number to use for RequestList persistence.
|
|
@@ -22,111 +53,119 @@ import { createSendRequest } from './send-request.js';
|
|
|
22
53
|
* @ignore
|
|
23
54
|
*/
|
|
24
55
|
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
55
|
-
* underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
|
|
56
|
-
*
|
|
57
|
-
* **Example usage:**
|
|
58
|
-
*
|
|
59
|
-
* ```javascript
|
|
60
|
-
* import { BasicCrawler, Dataset } from 'crawlee';
|
|
61
|
-
*
|
|
62
|
-
* // Create a crawler instance
|
|
63
|
-
* const crawler = new BasicCrawler({
|
|
64
|
-
* async requestHandler({ request, sendRequest }) {
|
|
65
|
-
* // 'request' contains an instance of the Request class
|
|
66
|
-
* // Here we simply fetch the HTML of the page and store it to a dataset
|
|
67
|
-
* const { body } = await sendRequest({
|
|
68
|
-
* url: request.url,
|
|
69
|
-
* method: request.method,
|
|
70
|
-
* body: request.payload,
|
|
71
|
-
* headers: request.headers,
|
|
72
|
-
* });
|
|
73
|
-
*
|
|
74
|
-
* await Dataset.pushData({
|
|
75
|
-
* url: request.url,
|
|
76
|
-
* html: body,
|
|
77
|
-
* })
|
|
78
|
-
* },
|
|
79
|
-
* });
|
|
80
|
-
*
|
|
81
|
-
* // Enqueue the initial requests and run the crawler
|
|
82
|
-
* await crawler.run([
|
|
83
|
-
* 'http://www.example.com/page-1',
|
|
84
|
-
* 'http://www.example.com/page-2',
|
|
85
|
-
* ]);
|
|
86
|
-
* ```
|
|
87
|
-
* @category Crawlers
|
|
88
|
-
*/
|
|
56
|
+
// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
|
|
57
|
+
export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
|
|
58
|
+
const urlPatternSchema = z.union([
|
|
59
|
+
z.string(),
|
|
60
|
+
z.instanceof(RegExp),
|
|
61
|
+
schemas.objectWithKeys(['glob']),
|
|
62
|
+
schemas.objectWithKeys(['regexp']),
|
|
63
|
+
]);
|
|
64
|
+
// `looseObject` (rather than `strictObject`) lets subclasses forward their own extraction-only options
|
|
65
|
+
// (e.g. `selector`) straight through without having to strip them out first.
|
|
66
|
+
const addRequestsOptionsSchema = z.looseObject({
|
|
67
|
+
forefront: z.boolean().optional(),
|
|
68
|
+
cache: z.boolean().optional(),
|
|
69
|
+
waitForAllRequestsToBeAdded: z.boolean().optional(),
|
|
70
|
+
batchSize: schemas.anyNumber.optional(),
|
|
71
|
+
waitBetweenBatchesMillis: schemas.anyNumber.optional(),
|
|
72
|
+
maxNewRequests: schemas.anyNumber.optional(),
|
|
73
|
+
limit: schemas.anyNumber.optional(),
|
|
74
|
+
baseUrl: z.string().optional(),
|
|
75
|
+
userData: schemas.anyObject.optional(),
|
|
76
|
+
label: z.string().optional(),
|
|
77
|
+
sessionId: z.string().optional(),
|
|
78
|
+
skipNavigation: z.boolean().optional(),
|
|
79
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
|
|
80
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
81
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
82
|
+
strategy: z.enum(EnqueueStrategy).optional(),
|
|
83
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
84
|
+
});
|
|
89
85
|
export class BasicCrawler {
|
|
90
|
-
static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
|
|
86
|
+
static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
|
|
87
|
+
/**
|
|
88
|
+
* Tracks the number of crawler instances created. The first crawler uses the default
|
|
89
|
+
* request queue; subsequent ones get their own queue via a unique alias so they don't
|
|
90
|
+
* collide.
|
|
91
|
+
*/
|
|
92
|
+
static #instanceCount = 0;
|
|
91
93
|
/**
|
|
92
94
|
* Tracks crawler instances that accessed shared state without having an explicit id.
|
|
93
95
|
* Used to detect and warn about multiple crawlers sharing the same state.
|
|
94
96
|
*/
|
|
95
|
-
static
|
|
97
|
+
static #useStateAnonymousIndices = new Set();
|
|
98
|
+
/** Backs the {@link BasicCrawler.statistics|`statistics`} getter. */
|
|
99
|
+
#statisticsDep;
|
|
96
100
|
/**
|
|
97
|
-
*
|
|
101
|
+
* The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
|
|
102
|
+
* crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
|
|
98
103
|
*/
|
|
99
|
-
|
|
104
|
+
get statistics() {
|
|
105
|
+
return this.#statisticsDep.value;
|
|
106
|
+
}
|
|
107
|
+
#requestManager;
|
|
100
108
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
109
|
+
* The main request-handling component of the crawler. It manages the requests that the crawler processes,
|
|
110
|
+
* combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
|
|
111
|
+
* via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
|
|
103
112
|
*/
|
|
104
|
-
|
|
113
|
+
get requestManager() {
|
|
114
|
+
return this.#requestManager;
|
|
115
|
+
}
|
|
116
|
+
/** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
|
|
117
|
+
#sessionPoolDep;
|
|
105
118
|
/**
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* Only available if used by the crawler.
|
|
119
|
+
* A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
|
|
120
|
+
* {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
|
|
109
121
|
*/
|
|
110
|
-
|
|
122
|
+
get sessionPool() {
|
|
123
|
+
return this.#sessionPoolDep.value;
|
|
124
|
+
}
|
|
111
125
|
/**
|
|
112
|
-
*
|
|
126
|
+
* Whether the request-processing-time hint has already been forwarded to the request manager. The hint
|
|
127
|
+
* derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
|
|
128
|
+
* so it only needs to be applied once, at the first async access of the manager.
|
|
113
129
|
*/
|
|
114
|
-
|
|
130
|
+
#requestManagerTimeoutsApplied = false;
|
|
115
131
|
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
132
|
+
* Resolves the governor for one run: either the injected
|
|
133
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
|
|
134
|
+
* the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
|
|
118
135
|
*/
|
|
119
|
-
|
|
136
|
+
#resolveConcurrencySystem;
|
|
137
|
+
/** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
|
|
138
|
+
#run;
|
|
139
|
+
#concurrencySystem;
|
|
140
|
+
/** The run in progress, if any — a finished run is history and cannot be driven. */
|
|
141
|
+
get #liveRun() {
|
|
142
|
+
return this.#run?.isLive === true ? this.#run : undefined;
|
|
143
|
+
}
|
|
144
|
+
/** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
|
|
145
|
+
get #dispatchingRun() {
|
|
146
|
+
return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
|
|
147
|
+
}
|
|
148
|
+
/** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
|
|
149
|
+
get running() {
|
|
150
|
+
return this.#liveRun !== undefined;
|
|
151
|
+
}
|
|
120
152
|
/**
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
153
|
+
* The concurrency governor the run in progress is booking its requests against — either the
|
|
154
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
|
|
155
|
+
* crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
|
|
156
|
+
*
|
|
157
|
+
* > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
|
|
158
|
+
* during a run rather than caching it across runs.
|
|
159
|
+
*
|
|
160
|
+
* {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
|
|
161
|
+
* owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
|
|
162
|
+
* `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
|
|
126
163
|
*/
|
|
127
|
-
|
|
164
|
+
get concurrencySystem() {
|
|
165
|
+
return this.#concurrencySystem?.value;
|
|
166
|
+
}
|
|
128
167
|
/**
|
|
129
|
-
* A reference to the underlying {@link
|
|
168
|
+
* A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
|
|
130
169
|
* Only available if used by the crawler.
|
|
131
170
|
*/
|
|
132
171
|
proxyConfiguration;
|
|
@@ -135,265 +174,535 @@ export class BasicCrawler {
|
|
|
135
174
|
* See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
|
|
136
175
|
*/
|
|
137
176
|
router = Router.create();
|
|
138
|
-
|
|
139
|
-
|
|
177
|
+
#basicContextPipeline;
|
|
178
|
+
/**
|
|
179
|
+
* The basic part of the context pipeline. Unlike the subclass pipeline, this
|
|
180
|
+
* part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
|
|
181
|
+
* pipelines expect the basic crawler fields to already be present in the context at runtime.
|
|
182
|
+
*
|
|
183
|
+
* Context built with this pipeline can be passed into multiple crawler pipelines at once.
|
|
184
|
+
*/
|
|
185
|
+
get #basicPipeline() {
|
|
186
|
+
this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
|
|
187
|
+
return this.#basicContextPipeline;
|
|
188
|
+
}
|
|
189
|
+
#contextPipeline;
|
|
190
|
+
/** @internal */
|
|
140
191
|
get contextPipeline() {
|
|
141
|
-
if (this
|
|
142
|
-
this
|
|
192
|
+
if (this.#contextPipeline === undefined) {
|
|
193
|
+
this.#contextPipeline = this.#buildFinalContextPipeline();
|
|
143
194
|
}
|
|
144
|
-
return this
|
|
195
|
+
return this.#contextPipeline;
|
|
196
|
+
}
|
|
197
|
+
#hasFinishedBefore = false;
|
|
198
|
+
/** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
|
|
199
|
+
get hasFinishedBefore() {
|
|
200
|
+
return this.#hasFinishedBefore;
|
|
201
|
+
}
|
|
202
|
+
#log;
|
|
203
|
+
get log() {
|
|
204
|
+
return this.#log;
|
|
145
205
|
}
|
|
146
|
-
running = false;
|
|
147
|
-
hasFinishedBefore = false;
|
|
148
|
-
log;
|
|
149
206
|
requestHandler;
|
|
150
|
-
errorHandler;
|
|
151
|
-
failedRequestHandler;
|
|
152
|
-
requestHandlerTimeoutMillis;
|
|
207
|
+
#errorHandler;
|
|
208
|
+
#failedRequestHandler;
|
|
209
|
+
#requestHandlerTimeoutMillis;
|
|
153
210
|
internalTimeoutMillis;
|
|
154
|
-
maxRequestRetries;
|
|
155
|
-
maxCrawlDepth;
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
211
|
+
#maxRequestRetries;
|
|
212
|
+
#maxCrawlDepth;
|
|
213
|
+
#maxRequestsPerCrawl;
|
|
214
|
+
get #handledRequestsCount() {
|
|
215
|
+
return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
|
|
216
|
+
}
|
|
217
|
+
#statusMessageLoggingInterval;
|
|
218
|
+
#statusMessageCallback;
|
|
219
|
+
blockedStatusCodes = new Set();
|
|
220
|
+
additionalHttpErrorStatusCodes;
|
|
221
|
+
#ignoreHttpErrorStatusCodes;
|
|
222
|
+
/**
|
|
223
|
+
* The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
|
|
224
|
+
* user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
|
|
225
|
+
* {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
|
|
226
|
+
* settable here.
|
|
227
|
+
*/
|
|
228
|
+
#taskLoopOptions;
|
|
166
229
|
httpClient;
|
|
167
230
|
retryOnBlocked;
|
|
168
|
-
respectRobotsTxtFile;
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
robotsTxtFileCache;
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
231
|
+
#respectRobotsTxtFile;
|
|
232
|
+
/** Whether `runInStorageTransaction()` opens a transaction at all. */
|
|
233
|
+
#transactionalStorageEnabled;
|
|
234
|
+
/** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
|
|
235
|
+
#storageWritePolicy;
|
|
236
|
+
#onSkippedRequest;
|
|
237
|
+
#robotsTxtFileCache;
|
|
238
|
+
#identity;
|
|
239
|
+
#contextPipelineOptions;
|
|
240
|
+
/**
|
|
241
|
+
* @internal
|
|
242
|
+
*/
|
|
178
243
|
static optionsShape = {
|
|
179
|
-
contextPipelineBuilder:
|
|
180
|
-
extendContext:
|
|
181
|
-
requestList:
|
|
182
|
-
requestQueue:
|
|
244
|
+
contextPipelineBuilder: schemas.anyObject.optional(),
|
|
245
|
+
extendContext: schemas.anyFunction.optional(),
|
|
246
|
+
requestList: validators.requestList.optional(),
|
|
247
|
+
requestQueue: validators.requestQueue.optional(),
|
|
248
|
+
requestManager: validators.requestManager.optional(),
|
|
183
249
|
// Subclasses override this function instead of passing it
|
|
184
250
|
// in constructor, so this validation needs to apply only
|
|
185
251
|
// if the user creates an instance of BasicCrawler directly.
|
|
186
|
-
requestHandler:
|
|
187
|
-
requestHandlerTimeoutSecs:
|
|
188
|
-
errorHandler:
|
|
189
|
-
failedRequestHandler:
|
|
190
|
-
maxRequestRetries:
|
|
191
|
-
sameDomainDelaySecs:
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
proxyConfiguration:
|
|
199
|
-
statusMessageLoggingInterval:
|
|
200
|
-
statusMessageCallback:
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
252
|
+
requestHandler: schemas.anyFunction.optional(),
|
|
253
|
+
requestHandlerTimeoutSecs: schemas.anyNumber.optional(),
|
|
254
|
+
errorHandler: schemas.anyFunction.optional(),
|
|
255
|
+
failedRequestHandler: schemas.anyFunction.optional(),
|
|
256
|
+
maxRequestRetries: schemas.anyNumber.default(3),
|
|
257
|
+
sameDomainDelaySecs: schemas.anyNumber.default(0),
|
|
258
|
+
maxRequestsPerCrawl: schemas.anyNumber.optional(),
|
|
259
|
+
maxCrawlDepth: schemas.anyNumber.optional(),
|
|
260
|
+
// No zod default — subclasses provide their own fallback (e.g. HTTP-optimized pool options).
|
|
261
|
+
taskLoopOptions: schemas.anyObject.optional(),
|
|
262
|
+
concurrencySystem: schemas.anyObject.optional(),
|
|
263
|
+
sessionPool: validators.sessionPool.optional(),
|
|
264
|
+
proxyConfiguration: validators.proxyConfiguration.optional(),
|
|
265
|
+
statusMessageLoggingInterval: schemas.anyNumber.default(10),
|
|
266
|
+
statusMessageCallback: schemas.anyFunction.optional(),
|
|
267
|
+
additionalHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
|
|
268
|
+
ignoreHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
|
|
269
|
+
blockedStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').optional(),
|
|
270
|
+
retryOnBlocked: z.boolean().default(false),
|
|
271
|
+
respectRobotsTxtFile: z.union([z.boolean(), schemas.anyObject]).default(false),
|
|
272
|
+
transactionalStorage: z
|
|
273
|
+
.union([z.boolean(), z.strictObject({ requestQueue: z.enum(['deferred', 'writeThrough']).optional() })])
|
|
274
|
+
.optional(),
|
|
275
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
276
|
+
httpClient: schemas.httpClient.optional(),
|
|
277
|
+
configuration: z.instanceof(Configuration).optional(),
|
|
278
|
+
storageBackend: validators.storageBackend.optional(),
|
|
279
|
+
eventManager: z.instanceof(EventManager).optional(),
|
|
280
|
+
logger: validators.logger.optional(),
|
|
208
281
|
// AutoscaledPool shorthands
|
|
209
|
-
minConcurrency:
|
|
210
|
-
maxConcurrency:
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
282
|
+
minConcurrency: schemas.anyNumber.optional(),
|
|
283
|
+
maxConcurrency: schemas.anyNumber.optional(),
|
|
284
|
+
initialConcurrency: schemas.anyNumber.optional(),
|
|
285
|
+
maxRequestsPerMinute: schemas.anyNumber
|
|
286
|
+
.refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
|
|
287
|
+
.refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
|
|
288
|
+
.optional(),
|
|
289
|
+
keepAlive: z.boolean().optional(),
|
|
290
|
+
statistics: schemas.anyObject.optional(),
|
|
291
|
+
id: z.string().optional(),
|
|
218
292
|
};
|
|
293
|
+
static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
|
|
219
294
|
/**
|
|
220
295
|
* All `BasicCrawler` parameters are passed via an options object.
|
|
221
296
|
*/
|
|
222
297
|
constructor(options = {}) {
|
|
223
|
-
|
|
224
|
-
const {
|
|
298
|
+
const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
|
|
299
|
+
const {
|
|
300
|
+
// oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
|
|
301
|
+
requestList,
|
|
302
|
+
// oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
|
|
303
|
+
requestQueue, requestManager, maxRequestRetries, sameDomainDelaySecs, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes, ignoreHttpErrorStatusCodes,
|
|
225
304
|
// Service locator options
|
|
226
|
-
configuration,
|
|
305
|
+
configuration, storageBackend, eventManager, logger,
|
|
227
306
|
// AutoscaledPool shorthands
|
|
228
|
-
minConcurrency, maxConcurrency, maxRequestsPerMinute,
|
|
229
|
-
//
|
|
230
|
-
|
|
307
|
+
minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
|
|
308
|
+
// All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
|
|
309
|
+
// one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
|
|
310
|
+
// hammering a site.
|
|
311
|
+
if (concurrencySystem !== undefined &&
|
|
312
|
+
(minConcurrency !== undefined ||
|
|
313
|
+
maxConcurrency !== undefined ||
|
|
314
|
+
initialConcurrency !== undefined ||
|
|
315
|
+
maxRequestsPerMinute !== undefined)) {
|
|
316
|
+
throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
|
|
317
|
+
'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
|
|
318
|
+
'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
|
|
319
|
+
}
|
|
231
320
|
// Create per-crawler service locator if custom services were provided.
|
|
232
321
|
// This wraps every method on the crawler instance so that calls to the global `serviceLocator`
|
|
233
322
|
// (via AsyncLocalStorage) resolve to this scoped instance instead.
|
|
234
323
|
// We also enter the scope for the rest of the constructor body, so that any code below
|
|
235
324
|
// that accesses `serviceLocator` will see the correct (scoped) instance.
|
|
236
325
|
let serviceLocatorScope = { enterScope: () => { }, exitScope: () => { } };
|
|
237
|
-
if (
|
|
326
|
+
if (storageBackend ||
|
|
238
327
|
eventManager ||
|
|
328
|
+
logger ||
|
|
239
329
|
(configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {
|
|
240
|
-
|
|
330
|
+
// Inherit the ambient locator's already-set services for anything not explicitly
|
|
331
|
+
// provided - e.g. only passing a `logger` must not detach the crawler from a globally
|
|
332
|
+
// configured storage backend.
|
|
333
|
+
const ambientServices = serviceLocator.getServicesIfSet();
|
|
334
|
+
const scopedServiceLocator = new ServiceLocator(configuration ?? ambientServices.configuration, eventManager ?? ambientServices.eventManager, storageBackend ?? ambientServices.storageBackend, logger ?? ambientServices.logger);
|
|
241
335
|
serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
|
|
242
336
|
}
|
|
243
337
|
try {
|
|
244
338
|
serviceLocatorScope.enterScope();
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
this.crawlerId = id ?? cryptoRandomObjectId();
|
|
249
|
-
// Store the builder so that it can be run when the contextPipeline is needed.
|
|
250
|
-
// Invoking it immediately would cause problems with parent constructor call order.
|
|
251
|
-
this.contextPipelineBuilder = () => {
|
|
252
|
-
let contextPipeline = (options.contextPipelineBuilder?.() ??
|
|
253
|
-
ContextPipeline.create()); // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext
|
|
254
|
-
if (options.extendContext !== undefined) {
|
|
255
|
-
contextPipeline = contextPipeline.compose({
|
|
256
|
-
action: async (context) => await options.extendContext(context),
|
|
257
|
-
});
|
|
258
|
-
}
|
|
259
|
-
contextPipeline = contextPipeline.compose({
|
|
260
|
-
action: async (context) => {
|
|
261
|
-
const { request } = context;
|
|
262
|
-
if (!this.requestMatchesEnqueueStrategy(request)) {
|
|
263
|
-
// eslint-disable-next-line dot-notation
|
|
264
|
-
const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
|
|
265
|
-
this.log.debug(message);
|
|
266
|
-
request.noRetry = true;
|
|
267
|
-
request.state = RequestState.SKIPPED;
|
|
268
|
-
await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
|
|
269
|
-
throw new ContextPipelineInterruptedError(message);
|
|
270
|
-
}
|
|
271
|
-
return context;
|
|
272
|
-
},
|
|
273
|
-
});
|
|
274
|
-
return contextPipeline;
|
|
339
|
+
this.#contextPipelineOptions = {
|
|
340
|
+
contextPipelineBuilder: parsedOptions.contextPipelineBuilder,
|
|
341
|
+
extendContext: parsedOptions.extendContext,
|
|
275
342
|
};
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
343
|
+
this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
|
|
344
|
+
// Initialize the Configuration instance to avoid lazy loading in the components
|
|
345
|
+
serviceLocator.getConfiguration();
|
|
346
|
+
const instanceIndex = BasicCrawler.#instanceCount++;
|
|
347
|
+
this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
|
|
348
|
+
if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
|
|
349
|
+
throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
|
|
282
350
|
}
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
351
|
+
const suppliedManager = requestManager ?? requestQueue;
|
|
352
|
+
// Offered before building a pacer of our own: anything that paces takes the floor - through any
|
|
353
|
+
// number of wrappers, since they all forward - so no domain ends up with two clocks.
|
|
354
|
+
const floorTaken = sameDomainDelaySecs > 0 &&
|
|
355
|
+
(suppliedManager?.recordPacingSignal({
|
|
356
|
+
reason: 'minIntervalEverywhere',
|
|
357
|
+
intervalMs: sameDomainDelaySecs * 1000,
|
|
358
|
+
// What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
|
|
359
|
+
scope: 'registrableDomain',
|
|
360
|
+
}) ??
|
|
361
|
+
false);
|
|
362
|
+
const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
|
|
363
|
+
// Built here rather than at first use so it can sit *inside* the tandem below, which is where a
|
|
364
|
+
// loader's transferred requests pass through it.
|
|
365
|
+
const writableManager = pacerNeeded
|
|
366
|
+
? new ThrottlingRequestManager({
|
|
367
|
+
domains: 'all',
|
|
368
|
+
minCrawlDelaySecs: sameDomainDelaySecs,
|
|
369
|
+
throttleBy: 'registrableDomain',
|
|
370
|
+
persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
|
|
371
|
+
// A factory, because the default queue is only opened on first use.
|
|
372
|
+
inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
|
|
373
|
+
})
|
|
374
|
+
: suppliedManager;
|
|
375
|
+
if (requestList !== undefined) {
|
|
376
|
+
// The list is read first, while new requests still have somewhere writable to go.
|
|
377
|
+
this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
|
|
286
378
|
}
|
|
287
|
-
|
|
379
|
+
else if (writableManager !== undefined) {
|
|
380
|
+
// A RequestQueue is itself a request manager.
|
|
381
|
+
this.#requestManager = writableManager;
|
|
382
|
+
}
|
|
383
|
+
this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
|
|
288
384
|
this.proxyConfiguration = proxyConfiguration;
|
|
289
|
-
this
|
|
290
|
-
this
|
|
291
|
-
this
|
|
292
|
-
this.
|
|
293
|
-
this
|
|
294
|
-
this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
|
|
295
|
-
this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
|
|
385
|
+
this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
|
|
386
|
+
this.#statusMessageCallback = statusMessageCallback;
|
|
387
|
+
this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
|
|
388
|
+
this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
|
|
389
|
+
this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
|
|
296
390
|
this.requestHandler = requestHandler ?? this.router;
|
|
297
|
-
this
|
|
298
|
-
this
|
|
391
|
+
this.#failedRequestHandler = failedRequestHandler;
|
|
392
|
+
this.#errorHandler = errorHandler;
|
|
299
393
|
if (requestHandlerTimeoutSecs) {
|
|
300
|
-
this
|
|
394
|
+
this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
|
|
301
395
|
}
|
|
302
396
|
else {
|
|
303
|
-
this
|
|
397
|
+
this.#requestHandlerTimeoutMillis = 60_000;
|
|
304
398
|
}
|
|
305
399
|
this.retryOnBlocked = retryOnBlocked;
|
|
306
|
-
this
|
|
307
|
-
|
|
308
|
-
const
|
|
400
|
+
this.#respectRobotsTxtFile = respectRobotsTxtFile;
|
|
401
|
+
// The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
|
|
402
|
+
const transactionalStorageOption = transactionalStorage;
|
|
403
|
+
this.#transactionalStorageEnabled = transactionalStorageOption !== false;
|
|
404
|
+
this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
|
|
405
|
+
this.#onSkippedRequest = onSkippedRequest;
|
|
309
406
|
// allow at least 5min for internal timeouts
|
|
310
407
|
this.internalTimeoutMillis =
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
log
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
this.sessionPoolOptions = {
|
|
330
|
-
...sessionPoolOptions,
|
|
331
|
-
log,
|
|
332
|
-
};
|
|
333
|
-
if (this.retryOnBlocked) {
|
|
334
|
-
this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
|
|
335
|
-
if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
|
|
336
|
-
log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`);
|
|
337
|
-
}
|
|
408
|
+
serviceLocator.getConfiguration().internalTimeoutMillis ??
|
|
409
|
+
Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
|
|
410
|
+
this.#maxRequestRetries = maxRequestRetries;
|
|
411
|
+
this.#maxCrawlDepth = maxCrawlDepth;
|
|
412
|
+
this.#statisticsDep = OwnedOrInjected.resolve(statistics,
|
|
413
|
+
// A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
|
|
414
|
+
// only be satisfied by an injected instance carrying the matching `state`, so this branch does
|
|
415
|
+
// not run in that case - hence the cast.
|
|
416
|
+
() => new Statistics({
|
|
417
|
+
logMessage: `${this.constructor.name} request statistics:`,
|
|
418
|
+
log: this.log,
|
|
419
|
+
id: this.#identity.id,
|
|
420
|
+
}));
|
|
421
|
+
if (sessionPool && proxyConfiguration) {
|
|
422
|
+
this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
|
|
423
|
+
'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
|
|
424
|
+
'`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
|
|
425
|
+
'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
|
|
338
426
|
}
|
|
339
|
-
this
|
|
427
|
+
this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
|
|
428
|
+
createSessionFunction: async (opts) => new Session({
|
|
429
|
+
...opts?.sessionOptions,
|
|
430
|
+
proxyInfo: this.proxyConfiguration
|
|
431
|
+
? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
|
|
432
|
+
: opts?.sessionOptions?.proxyInfo,
|
|
433
|
+
}),
|
|
434
|
+
}));
|
|
435
|
+
this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
|
|
340
436
|
const maxSignedInteger = 2 ** 31 - 1;
|
|
341
|
-
if (this
|
|
342
|
-
log.warning(`requestHandlerTimeoutMillis ${this
|
|
437
|
+
if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
|
|
438
|
+
this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
|
|
343
439
|
` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
|
|
344
|
-
this
|
|
440
|
+
this.#requestHandlerTimeoutMillis = maxSignedInteger;
|
|
345
441
|
}
|
|
346
442
|
this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
|
|
347
|
-
this
|
|
348
|
-
const isMaxPagesExceeded = () => this
|
|
443
|
+
this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
|
|
444
|
+
const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
|
|
349
445
|
// eslint-disable-next-line prefer-const
|
|
350
|
-
let { isFinishedFunction, isTaskReadyFunction } =
|
|
446
|
+
let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
|
|
351
447
|
// override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
|
|
352
448
|
if (keepAlive) {
|
|
353
449
|
isFinishedFunction = async () => false;
|
|
354
450
|
}
|
|
355
|
-
const
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
451
|
+
const crawlerOwnedTaskLoopConfiguration = {
|
|
452
|
+
runTaskFunction: async () => {
|
|
453
|
+
const source = this.requestManager;
|
|
454
|
+
if (!source)
|
|
455
|
+
throw new Error('Request provider is not initialized!');
|
|
456
|
+
const request = await this.#resolveRequest();
|
|
457
|
+
if (!request) {
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
// Started here, rather than in `handleRequest`, so that a failure during context pipeline
|
|
461
|
+
// initialization (e.g. a browser page timing out before the request handler ever runs) is
|
|
462
|
+
// still accounted for by `recordRequestFailure` below - which is a no-op without a
|
|
463
|
+
// matching `recordRequestStart`.
|
|
464
|
+
this.statistics.recordRequestStart(request.id || request.uniqueKey);
|
|
465
|
+
const crawlingContext = { request };
|
|
466
|
+
try {
|
|
467
|
+
// The transaction spans the whole pipeline call, covering the navigation hooks
|
|
468
|
+
// and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
|
|
469
|
+
await this.runInStorageTransaction(async () =>
|
|
470
|
+
// Navigation, the navigation hooks and the request handler are timed individually, but the
|
|
471
|
+
// phases between them are not, so a request could still get stuck indefinitely. This is the
|
|
472
|
+
// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
|
|
473
|
+
await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source), (error) => this.#handleRequestFailure(error, crawlingContext, source))));
|
|
474
|
+
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
|
|
477
|
+
// take the normal error flow. Anything else is fatal.
|
|
478
|
+
if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
|
|
479
|
+
const unwrappedError = this.#unwrapError(error);
|
|
480
|
+
await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, this.requestManager);
|
|
481
|
+
if (!this.#errorAbsolvesSession(unwrappedError)) {
|
|
482
|
+
crawlingContext.session?.markBad();
|
|
483
|
+
}
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
throw this.#unwrapError(error);
|
|
487
|
+
}
|
|
488
|
+
},
|
|
360
489
|
isTaskReadyFunction: async () => {
|
|
361
490
|
if (isMaxPagesExceeded()) {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
`${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
|
|
365
|
-
this.shouldLogMaxProcessedRequestsExceeded = false;
|
|
366
|
-
}
|
|
491
|
+
this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
|
|
492
|
+
`${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
|
|
367
493
|
return false;
|
|
368
494
|
}
|
|
369
|
-
|
|
495
|
+
if (this.#run?.stopRequested) {
|
|
496
|
+
this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
|
|
497
|
+
'Ongoing requests will be allowed to complete.');
|
|
498
|
+
return false;
|
|
499
|
+
}
|
|
500
|
+
return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
|
|
370
501
|
},
|
|
371
502
|
isFinishedFunction: async () => {
|
|
372
503
|
if (isMaxPagesExceeded()) {
|
|
373
|
-
log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this
|
|
504
|
+
this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
|
|
374
505
|
'and all requests that were in progress at that time have now finished. ' +
|
|
375
|
-
`In total, the crawler processed ${this
|
|
506
|
+
`In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
|
|
376
507
|
return true;
|
|
377
508
|
}
|
|
509
|
+
if (this.#run?.stopRequested) {
|
|
510
|
+
this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
|
|
511
|
+
return true;
|
|
512
|
+
}
|
|
513
|
+
// `maybeFinish()` calls this only once nothing is in flight (`autoscaled-pool.ts`) - the point
|
|
514
|
+
// where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
|
|
515
|
+
// and the only place where throwing does not abandon requests mid-processing.
|
|
516
|
+
const state = await this.requestManager?.checkReadiness();
|
|
517
|
+
// Under `keepAlive`, outliving a domain that will not let us through is the whole point.
|
|
518
|
+
if (state?.status === 'stalled' && !keepAlive) {
|
|
519
|
+
throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
|
|
520
|
+
}
|
|
378
521
|
const isFinished = isFinishedFunction
|
|
379
522
|
? await isFinishedFunction()
|
|
380
|
-
:
|
|
523
|
+
: state === undefined || state.status === 'finished';
|
|
381
524
|
if (isFinished) {
|
|
382
525
|
const reason = isFinishedFunction
|
|
383
526
|
? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
|
|
384
527
|
: 'All requests from the queue have been processed, the crawler will shut down.';
|
|
385
|
-
log.info(reason);
|
|
528
|
+
this.log.info(reason);
|
|
386
529
|
}
|
|
387
530
|
return isFinished;
|
|
388
531
|
},
|
|
389
|
-
log,
|
|
532
|
+
log: this.log,
|
|
390
533
|
};
|
|
391
|
-
this
|
|
534
|
+
this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
|
|
535
|
+
this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
|
|
536
|
+
minConcurrency,
|
|
537
|
+
maxConcurrency,
|
|
538
|
+
maxTasksPerMinute: maxRequestsPerMinute,
|
|
539
|
+
// Spread conditionally - an explicit `undefined` would clobber a subclass default, see
|
|
540
|
+
// `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
|
|
541
|
+
...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
|
|
542
|
+
log: this.log,
|
|
543
|
+
}));
|
|
392
544
|
}
|
|
393
545
|
finally {
|
|
394
546
|
serviceLocatorScope.exitScope();
|
|
395
547
|
}
|
|
396
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
|
|
551
|
+
* `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
|
|
552
|
+
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
|
|
553
|
+
*
|
|
554
|
+
* Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
|
|
555
|
+
* concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
|
|
556
|
+
*/
|
|
557
|
+
createDefaultConcurrencySystem(options) {
|
|
558
|
+
return new ConcurrencySystem(options);
|
|
559
|
+
}
|
|
560
|
+
/**
|
|
561
|
+
* Determines if the given HTTP status code is an error status code given
|
|
562
|
+
* the default behaviour and user-set preferences.
|
|
563
|
+
* @param status
|
|
564
|
+
* @returns `true` if the status code is considered an error, `false` otherwise
|
|
565
|
+
*/
|
|
566
|
+
isErrorStatusCode(status) {
|
|
567
|
+
const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
|
|
568
|
+
const includeError = this.additionalHttpErrorStatusCodes.has(status);
|
|
569
|
+
return (status >= 500 && !excludeError) || includeError;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
|
|
573
|
+
* This handles base context creation, session resolution, and context helpers.
|
|
574
|
+
*/
|
|
575
|
+
#buildBasicContextPipeline() {
|
|
576
|
+
return ContextPipeline.create()
|
|
577
|
+
.compose(this.#checkRobotsTxt.bind(this))
|
|
578
|
+
.compose(this.#createBaseContext.bind(this))
|
|
579
|
+
.compose(this.#resolveSession.bind(this))
|
|
580
|
+
.compose(this.#createContextHelpers.bind(this));
|
|
581
|
+
}
|
|
582
|
+
async #checkRobotsTxt({ request }) {
|
|
583
|
+
if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
|
|
584
|
+
this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
|
|
585
|
+
throw new ContextPipelineInterruptedError('robotsTxt');
|
|
586
|
+
}
|
|
587
|
+
return {};
|
|
588
|
+
}
|
|
589
|
+
#createBaseContext(context, onCleanup) {
|
|
590
|
+
return {
|
|
591
|
+
id: cryptoRandomObjectId(10),
|
|
592
|
+
log: this.log,
|
|
593
|
+
pushData: this.pushData.bind(this),
|
|
594
|
+
useState: this.useState.bind(this),
|
|
595
|
+
getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
|
|
596
|
+
registerDeferredCleanup: (cleanup) => {
|
|
597
|
+
onCleanup(async () => {
|
|
598
|
+
try {
|
|
599
|
+
await cleanup();
|
|
600
|
+
}
|
|
601
|
+
catch (error) {
|
|
602
|
+
this.log.debug('Error in deferred cleanup', { error });
|
|
603
|
+
}
|
|
604
|
+
});
|
|
605
|
+
},
|
|
606
|
+
afterStorageCommit: (callback) => {
|
|
607
|
+
const transaction = currentStorageTransaction();
|
|
608
|
+
if (transaction === undefined) {
|
|
609
|
+
throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
|
|
610
|
+
"With `transactionalStorage: false` the request's writes are applied as they are " +
|
|
611
|
+
'made, so they throw at the call site instead of at commit time.');
|
|
612
|
+
}
|
|
613
|
+
transaction.afterCommit(callback);
|
|
614
|
+
},
|
|
615
|
+
extendTimeout: (secs) => {
|
|
616
|
+
const extraMillis = secs * 1000;
|
|
617
|
+
// the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...
|
|
618
|
+
extendTimeout(extraMillis);
|
|
619
|
+
// ...the internal timeout around the whole request, which is not an `addTimeoutToPromise` frame...
|
|
620
|
+
context[extendTimeoutKey]?.(extraMillis);
|
|
621
|
+
// ...and, when called from within the navigation phase, its shared window, so extending a hook
|
|
622
|
+
// extends the whole navigation budget rather than just that hook's step.
|
|
623
|
+
if (context[navigationDeadlineKey] !== undefined) {
|
|
624
|
+
context[navigationDeadlineKey] += extraMillis;
|
|
625
|
+
}
|
|
626
|
+
// Extension failure must not fail the request that asked for more time.
|
|
627
|
+
this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
|
|
628
|
+
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
|
|
629
|
+
});
|
|
630
|
+
},
|
|
631
|
+
skipRequest: (message) => {
|
|
632
|
+
this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
|
|
633
|
+
throw new ContextPipelineInterruptedError('manual', message);
|
|
634
|
+
},
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
|
|
639
|
+
* instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
|
|
640
|
+
* `markRequestAsHandled`, so nothing observes the difference.
|
|
641
|
+
*/
|
|
642
|
+
async #resolveRequest() {
|
|
643
|
+
const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
644
|
+
if (!fetched) {
|
|
645
|
+
return null;
|
|
646
|
+
}
|
|
647
|
+
const request = CrawlingRequest.fromSchema(fetched);
|
|
648
|
+
// Reset loadedUrl so an old one is not carried over to retries.
|
|
649
|
+
request.loadedUrl = undefined;
|
|
650
|
+
return request;
|
|
651
|
+
}
|
|
652
|
+
async #resolveSession({ request }) {
|
|
653
|
+
const session = await this.#timeoutAndRetry(async () => {
|
|
654
|
+
const existingSession = await this.sessionPool.getSession(request.sessionId);
|
|
655
|
+
if (!existingSession) {
|
|
656
|
+
throw new MissingSessionError(request.sessionId);
|
|
657
|
+
}
|
|
658
|
+
return existingSession;
|
|
659
|
+
}, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
660
|
+
return { session, proxyInfo: session?.proxyInfo };
|
|
661
|
+
}
|
|
662
|
+
async #createContextHelpers({ request, session }) {
|
|
663
|
+
const addRequests = async (requests, options = {}) => {
|
|
664
|
+
const newCrawlDepth = request.crawlDepth + 1;
|
|
665
|
+
const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
|
|
666
|
+
return await this.addRequests(requestsGenerator, options);
|
|
667
|
+
};
|
|
668
|
+
const sendRequest = createSendRequest(this.httpClient, request, session);
|
|
669
|
+
return { addRequests, sendRequest };
|
|
670
|
+
}
|
|
671
|
+
#buildFinalContextPipeline() {
|
|
672
|
+
const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
|
|
673
|
+
ContextPipeline.create());
|
|
674
|
+
// `extendContext` runs *before* the subclass navigation pipeline (which includes the
|
|
675
|
+
// pre/post-navigation hooks). This makes the extension visible to those hooks and to the
|
|
676
|
+
// request handler alike. The trade-off is that `extendContext` cannot access
|
|
677
|
+
// navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those
|
|
678
|
+
// don't exist yet at this point in the pipeline.
|
|
679
|
+
// The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at
|
|
680
|
+
// runtime (the pipeline copies each middleware's returned members onto the shared context), but
|
|
681
|
+
// TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
|
|
682
|
+
// casts below are sound because `buildFinalContextPipeline` is declared to return the fully
|
|
683
|
+
// resolved `ExtendedContext` (= `Context & ContextExtension`).
|
|
684
|
+
const { extendContext } = this.#contextPipelineOptions;
|
|
685
|
+
let contextPipeline;
|
|
686
|
+
if (extendContext !== undefined) {
|
|
687
|
+
contextPipeline = ContextPipeline.create()
|
|
688
|
+
.compose(extendContext)
|
|
689
|
+
.chain(subclassPipeline);
|
|
690
|
+
}
|
|
691
|
+
else {
|
|
692
|
+
contextPipeline = subclassPipeline;
|
|
693
|
+
}
|
|
694
|
+
contextPipeline = contextPipeline.compose(async (context) => {
|
|
695
|
+
const { request } = context;
|
|
696
|
+
if (request && !this.#requestMatchesEnqueueStrategy(request)) {
|
|
697
|
+
// eslint-disable-next-line dot-notation
|
|
698
|
+
const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
|
|
699
|
+
this.log.debug(message);
|
|
700
|
+
throw new ContextPipelineInterruptedError('redirect');
|
|
701
|
+
}
|
|
702
|
+
return context;
|
|
703
|
+
});
|
|
704
|
+
return contextPipeline;
|
|
705
|
+
}
|
|
397
706
|
/**
|
|
398
707
|
* Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
|
|
399
708
|
* Used for retrying requests that failed due to proxy errors.
|
|
@@ -401,158 +710,212 @@ export class BasicCrawler {
|
|
|
401
710
|
* @param error The error to check.
|
|
402
711
|
*/
|
|
403
712
|
isProxyError(error) {
|
|
404
|
-
return ROTATE_PROXY_ERRORS.some((x) => this.
|
|
713
|
+
return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
|
|
405
714
|
}
|
|
406
715
|
/**
|
|
716
|
+
* Sets the status message for the current crawler run.
|
|
717
|
+
*
|
|
407
718
|
* This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
|
|
719
|
+
*
|
|
720
|
+
* The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
|
|
721
|
+
* event. Integrations such as the Apify SDK subscribe to that event and forward the message to
|
|
722
|
+
* their status-reporting backend (e.g. the Apify platform).
|
|
408
723
|
*/
|
|
409
|
-
|
|
724
|
+
setStatusMessage(message, options = {}) {
|
|
410
725
|
const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
//
|
|
417
|
-
|
|
726
|
+
// Each allowed level has its own method on the logger, so this goes through them rather than through
|
|
727
|
+
// `logWithLevel`, which is abstract and therefore cannot be instrumented.
|
|
728
|
+
this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
|
|
729
|
+
// Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
|
|
730
|
+
// subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
|
|
731
|
+
// Setting the status message is not a storage concern, so we intentionally don't route it
|
|
732
|
+
// through the storage client anymore.
|
|
733
|
+
serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
|
|
734
|
+
crawlerId: this.#identity.id,
|
|
735
|
+
message,
|
|
736
|
+
isStatusMessageTerminal: options.isStatusMessageTerminal,
|
|
737
|
+
level: options.level,
|
|
738
|
+
});
|
|
418
739
|
}
|
|
419
|
-
getPeriodicLogger() {
|
|
420
|
-
let previousState = { ...this.
|
|
740
|
+
#getPeriodicLogger() {
|
|
741
|
+
let previousState = { ...this.statistics.state };
|
|
421
742
|
const getOperationMode = () => {
|
|
422
|
-
const { requestsFailed } = this.
|
|
743
|
+
const { requestsFailed } = this.statistics.state;
|
|
423
744
|
const { requestsFailed: previousRequestsFailed } = previousState;
|
|
424
|
-
previousState = { ...this.
|
|
425
|
-
|
|
426
|
-
|
|
745
|
+
previousState = { ...this.statistics.state };
|
|
746
|
+
const failedDelta = requestsFailed - previousRequestsFailed;
|
|
747
|
+
if (failedDelta > 0) {
|
|
748
|
+
return { mode: 'ERROR', failedDelta };
|
|
427
749
|
}
|
|
428
|
-
return 'REGULAR';
|
|
750
|
+
return { mode: 'REGULAR', failedDelta: 0 };
|
|
429
751
|
};
|
|
430
752
|
const log = async () => {
|
|
431
|
-
const operationMode = getOperationMode();
|
|
753
|
+
const { mode: operationMode, failedDelta } = getOperationMode();
|
|
432
754
|
let message;
|
|
433
755
|
if (operationMode === 'ERROR') {
|
|
434
|
-
message = `Experiencing problems, ${
|
|
756
|
+
message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
|
|
435
757
|
}
|
|
436
758
|
else {
|
|
437
|
-
const total = this.requestManager?.getTotalCount();
|
|
438
|
-
message = `Crawled ${this.
|
|
759
|
+
const total = await this.requestManager?.getTotalCount();
|
|
760
|
+
message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
|
|
439
761
|
}
|
|
440
|
-
if (this
|
|
441
|
-
await this
|
|
762
|
+
if (this.#statusMessageCallback) {
|
|
763
|
+
await this.#statusMessageCallback({
|
|
442
764
|
crawler: this,
|
|
443
|
-
state: this.
|
|
765
|
+
state: this.statistics.state,
|
|
444
766
|
previousState,
|
|
445
767
|
message,
|
|
446
768
|
});
|
|
447
769
|
return;
|
|
448
770
|
}
|
|
449
|
-
|
|
771
|
+
this.setStatusMessage(message);
|
|
450
772
|
};
|
|
451
|
-
const interval = setInterval(log, this
|
|
773
|
+
const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
|
|
452
774
|
return { log, stop: () => clearInterval(interval) };
|
|
453
775
|
}
|
|
454
776
|
/**
|
|
455
|
-
* Runs the crawler. Returns a promise that resolves once
|
|
456
|
-
*
|
|
777
|
+
* Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
|
|
778
|
+
* finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
|
|
779
|
+
* default "the request manager is empty") reports that the crawl is over.
|
|
457
780
|
*
|
|
458
781
|
* We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
|
|
459
782
|
* running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
|
|
460
783
|
*
|
|
784
|
+
* Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
|
|
785
|
+
* run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
|
|
786
|
+
* if that is what you want.
|
|
787
|
+
*
|
|
461
788
|
* @param [requests] The requests to add.
|
|
462
|
-
* @param [options] Options for the
|
|
789
|
+
* @param [options] Options for adding the initial requests.
|
|
463
790
|
*/
|
|
464
791
|
async run(requests, options) {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
if (this.hasFinishedBefore) {
|
|
470
|
-
// When executing the run method for the second time explicitly,
|
|
471
|
-
// we need to purge the default RQ to allow processing the same requests again - this is important so users can
|
|
472
|
-
// pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
|
|
473
|
-
// ignored - as a failed requests is still handled.
|
|
474
|
-
if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
|
|
475
|
-
await this.requestQueue.drop();
|
|
476
|
-
this.requestQueue = await this._getRequestQueue();
|
|
477
|
-
this.requestManager = undefined;
|
|
478
|
-
await this.initializeRequestManager();
|
|
479
|
-
this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
|
|
792
|
+
// A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
|
|
793
|
+
return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
|
|
794
|
+
if (this.running) {
|
|
795
|
+
throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
|
|
480
796
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
stats = {
|
|
524
|
-
requestsFinished: this.stats.state.requestsFinished,
|
|
525
|
-
requestsFailed: this.stats.state.requestsFailed,
|
|
526
|
-
retryHistogram: this.stats.requestRetryHistogram,
|
|
527
|
-
...finalStats,
|
|
528
|
-
};
|
|
529
|
-
this.log.info('Final request statistics:', stats);
|
|
530
|
-
if (this.stats.errorTracker.total !== 0) {
|
|
531
|
-
const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
|
|
532
|
-
this.log.info(`Error analysis:`, {
|
|
533
|
-
totalErrors: this.stats.errorTracker.total,
|
|
534
|
-
uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),
|
|
535
|
-
mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
|
|
797
|
+
// An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
|
|
798
|
+
// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
|
|
799
|
+
// run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
|
|
800
|
+
this.#concurrencySystem = this.#resolveConcurrencySystem();
|
|
801
|
+
const run = new CrawlerRun({
|
|
802
|
+
log: this.log,
|
|
803
|
+
concurrencySystem: this.#concurrencySystem.value,
|
|
804
|
+
taskLoopOptions: this.#taskLoopOptions,
|
|
805
|
+
consumer: this.#identity,
|
|
806
|
+
});
|
|
807
|
+
this.#run = run;
|
|
808
|
+
try {
|
|
809
|
+
if (this.#hasFinishedBefore) {
|
|
810
|
+
// A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
|
|
811
|
+
await this.#statisticsDep.ifOwned(async (stats) => {
|
|
812
|
+
stats.reset();
|
|
813
|
+
await stats.resetStore();
|
|
814
|
+
});
|
|
815
|
+
// A session pool the crawler built follows the same blank-slate rule as its statistics.
|
|
816
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => {
|
|
817
|
+
pool.reset();
|
|
818
|
+
await pool.resetStore();
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
await purgeDefaultStorages({
|
|
822
|
+
onlyPurgeOnce: true,
|
|
823
|
+
storageBackend: serviceLocator.getStorageBackend(),
|
|
824
|
+
configuration: serviceLocator.getConfiguration(),
|
|
825
|
+
});
|
|
826
|
+
if (requests) {
|
|
827
|
+
await this.addRequests(requests, options);
|
|
828
|
+
}
|
|
829
|
+
// An injected governor is the caller's to run; an owned one is started here and stopped again
|
|
830
|
+
// below if the startup fails. Either way the task loop rejects one that is not running.
|
|
831
|
+
await this.#concurrencySystem.ifOwned((system) => system.start());
|
|
832
|
+
await this.init();
|
|
833
|
+
await this.statistics.startCapturing();
|
|
834
|
+
}
|
|
835
|
+
catch (error) {
|
|
836
|
+
// Clean up here before propagating, otherwise a failed startup would leave the process hanging.
|
|
837
|
+
await this.teardown().catch((teardownError) => {
|
|
838
|
+
this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
|
|
536
839
|
});
|
|
840
|
+
// The run never began, so let the instance be run again instead of leaving it wedged as `running`.
|
|
841
|
+
await run.finish();
|
|
842
|
+
await this.#concurrencySystem.ifOwned((system) => system.stop());
|
|
843
|
+
throw error;
|
|
844
|
+
}
|
|
845
|
+
const periodicLogger = this.#getPeriodicLogger();
|
|
846
|
+
this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
|
|
847
|
+
const sigintHandler = async () => {
|
|
848
|
+
this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
|
|
849
|
+
await this.#pauseOnMigration();
|
|
850
|
+
await run.abort();
|
|
851
|
+
};
|
|
852
|
+
// Attach a listener to handle migration and aborting events gracefully.
|
|
853
|
+
const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
|
|
854
|
+
process.once('SIGINT', sigintHandler);
|
|
855
|
+
const eventManager = serviceLocator.getEventManager();
|
|
856
|
+
eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
|
|
857
|
+
eventManager.on(EventType.ABORTING, boundPauseOnMigration);
|
|
858
|
+
let stats = {};
|
|
859
|
+
try {
|
|
860
|
+
await run.dispatchRequests();
|
|
537
861
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
862
|
+
finally {
|
|
863
|
+
await this.statistics.stopCapturing();
|
|
864
|
+
await this.teardown();
|
|
865
|
+
process.off('SIGINT', sigintHandler);
|
|
866
|
+
eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
|
|
867
|
+
eventManager.off(EventType.ABORTING, boundPauseOnMigration);
|
|
868
|
+
const finalStats = this.statistics.calculate();
|
|
869
|
+
stats = {
|
|
870
|
+
requestsSucceeded: this.statistics.state.requestsSucceeded,
|
|
871
|
+
requestsFailed: this.statistics.state.requestsFailed,
|
|
872
|
+
retryHistogram: this.statistics.requestRetryHistogram,
|
|
873
|
+
...finalStats,
|
|
874
|
+
};
|
|
875
|
+
this.log.info('Final request statistics:', stats);
|
|
876
|
+
// A crawl that did nothing while the manager holds only handled requests is a mistake whoever
|
|
877
|
+
// handled them - this run, another crawler on the same queue, or a previous process. Starting
|
|
878
|
+
// against handled requests is not: that is what resuming a crawl looks like.
|
|
879
|
+
if (stats.requestsSucceeded + stats.requestsFailed === 0) {
|
|
880
|
+
// Never let the diagnostic itself break the run.
|
|
881
|
+
const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
|
|
882
|
+
if (alreadyHandled > 0) {
|
|
883
|
+
this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
|
|
884
|
+
`${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
|
|
885
|
+
'already handled, and a failed request counts as handled too. Nothing ' +
|
|
886
|
+
'empties a queue between runs, so to crawl them again, purge it ' +
|
|
887
|
+
'(`await queue.purge()`) or use a fresh one (e.g. ' +
|
|
888
|
+
'`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
|
|
889
|
+
'crawler instance.');
|
|
544
890
|
}
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
|
|
891
|
+
}
|
|
892
|
+
if (this.statistics.errorTracker.total !== 0) {
|
|
893
|
+
const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
|
|
894
|
+
this.log.info(`Error analysis:`, {
|
|
895
|
+
totalErrors: this.statistics.errorTracker.total,
|
|
896
|
+
uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
|
|
897
|
+
mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
const client = serviceLocator.getStorageBackend();
|
|
901
|
+
if (client.teardown) {
|
|
902
|
+
let finished = false;
|
|
903
|
+
setTimeout(() => {
|
|
904
|
+
if (!finished) {
|
|
905
|
+
this.log.info('Waiting for the storage to write its state to file system.');
|
|
906
|
+
}
|
|
907
|
+
}, 1000);
|
|
908
|
+
await client.teardown();
|
|
909
|
+
finished = true;
|
|
910
|
+
}
|
|
911
|
+
periodicLogger.stop();
|
|
912
|
+
this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
|
|
913
|
+
await run.finish();
|
|
914
|
+
await this.#concurrencySystem.ifOwned((system) => system.stop());
|
|
915
|
+
this.#hasFinishedBefore = true;
|
|
548
916
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
void this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
|
|
552
|
-
this.running = false;
|
|
553
|
-
this.hasFinishedBefore = true;
|
|
554
|
-
}
|
|
555
|
-
return stats;
|
|
917
|
+
return stats;
|
|
918
|
+
}));
|
|
556
919
|
}
|
|
557
920
|
/**
|
|
558
921
|
* Gracefully stops the current run of the crawler.
|
|
@@ -561,71 +924,160 @@ export class BasicCrawler {
|
|
|
561
924
|
*
|
|
562
925
|
* To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
|
|
563
926
|
*/
|
|
564
|
-
stop(
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
927
|
+
stop(reason = 'The crawler has been gracefully stopped.') {
|
|
928
|
+
const run = this.#liveRun;
|
|
929
|
+
if (run === undefined) {
|
|
930
|
+
this.log.warning('Cannot stop a crawler that is not running.');
|
|
931
|
+
return;
|
|
932
|
+
}
|
|
933
|
+
run.stop(reason);
|
|
934
|
+
}
|
|
935
|
+
/**
|
|
936
|
+
* Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
|
|
937
|
+
* after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
|
|
938
|
+
* run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
|
|
939
|
+
*
|
|
940
|
+
* > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
|
|
941
|
+
* throughout, since a shared one may still be serving other crawlers.
|
|
942
|
+
*/
|
|
943
|
+
async pause(timeoutSecs) {
|
|
944
|
+
const run = this.#dispatchingRun;
|
|
945
|
+
if (run === undefined) {
|
|
946
|
+
this.log.warning('Cannot pause a crawler that is not running.');
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
await run.pause(timeoutSecs);
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
|
|
953
|
+
* again. A no-op on a crawler that is not paused.
|
|
954
|
+
*/
|
|
955
|
+
resume() {
|
|
956
|
+
const run = this.#dispatchingRun;
|
|
957
|
+
if (run === undefined) {
|
|
958
|
+
this.log.warning('Cannot resume a crawler that is not running.');
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
run.resume();
|
|
962
|
+
}
|
|
963
|
+
/**
|
|
964
|
+
* Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
|
|
965
|
+
* if none has been configured or opened yet.
|
|
966
|
+
*/
|
|
967
|
+
async getRequestManager() {
|
|
968
|
+
if (!this.#requestManager) {
|
|
969
|
+
this.#requestManager = await this.#openOwnedRequestQueue();
|
|
970
|
+
}
|
|
971
|
+
// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
|
|
972
|
+
// now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
|
|
973
|
+
// but guard so we do not re-issue it on every call.
|
|
974
|
+
if (!this.#requestManagerTimeoutsApplied) {
|
|
975
|
+
this.#requestManagerTimeoutsApplied = true;
|
|
976
|
+
await this.#applyRequestManagerTimeouts(this.#requestManager);
|
|
977
|
+
}
|
|
978
|
+
return this.#requestManager;
|
|
574
979
|
}
|
|
980
|
+
/**
|
|
981
|
+
* @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
|
|
982
|
+
* crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
|
|
983
|
+
*/
|
|
575
984
|
async getRequestQueue() {
|
|
576
|
-
|
|
577
|
-
|
|
985
|
+
return this.getRequestManager();
|
|
986
|
+
}
|
|
987
|
+
/**
|
|
988
|
+
* Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
|
|
989
|
+
*/
|
|
990
|
+
async #openOwnedRequestQueue() {
|
|
991
|
+
// The first crawler instance uses the default queue (null identifier);
|
|
992
|
+
// subsequent instances get their own queue via a unique alias so they don't collide.
|
|
993
|
+
const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
|
|
994
|
+
return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Tells a request manager how long we expect to hold a fetched request, so that one backed by a
|
|
998
|
+
* locking storage backend keeps it reserved for slightly longer than the request handler timeout
|
|
999
|
+
* (with some padding for overhead), but never for less than a minute. This prevents a long-running
|
|
1000
|
+
* request from being handed out a second time while it is still being processed — and it works
|
|
1001
|
+
* regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
|
|
1002
|
+
*/
|
|
1003
|
+
async #applyRequestManagerTimeouts(requestManager) {
|
|
1004
|
+
// A router route may hold a request for longer than the crawler's own timeout, and we cannot know
|
|
1005
|
+
// which routes a run will hit, so reserve for the longest one any route asked for. The hint is
|
|
1006
|
+
// raise-only, so erring high here is safe.
|
|
1007
|
+
const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
|
|
1008
|
+
const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
|
|
1009
|
+
await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
|
|
1010
|
+
}
|
|
1011
|
+
/**
|
|
1012
|
+
* Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
|
|
1013
|
+
* for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
|
|
1014
|
+
* mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
|
|
1015
|
+
* the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
|
|
1016
|
+
* `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
|
|
1017
|
+
*/
|
|
1018
|
+
async #validateRequestUserData(source) {
|
|
1019
|
+
if (typeof source === 'string') {
|
|
1020
|
+
return;
|
|
578
1021
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
1022
|
+
const getSchema = this.requestHandler.getSchema;
|
|
1023
|
+
if (typeof getSchema !== 'function') {
|
|
1024
|
+
return;
|
|
582
1025
|
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
1026
|
+
// Resolve the label via its public accessors only — the top-level `label` of a `RequestOptions` or the
|
|
1027
|
+
// `Request.label` getter — rather than reaching into `userData`, where the request happens to store it.
|
|
1028
|
+
const target = source;
|
|
1029
|
+
const schema = getSchema(target.label);
|
|
1030
|
+
if (!schema) {
|
|
1031
|
+
return;
|
|
588
1032
|
}
|
|
589
|
-
|
|
1033
|
+
// Store the parsed value rather than the raw input, so the queue holds the same coerced `userData` the
|
|
1034
|
+
// handler will see. Assigning through a `Request` instance's setter keeps its internal `__crawlee` meta.
|
|
1035
|
+
target.userData = await validateUserData(target.label, schema, target.userData ?? {});
|
|
590
1036
|
}
|
|
591
1037
|
async useState(defaultValue = {}) {
|
|
592
|
-
const kvs = await KeyValueStore.open(null, {
|
|
593
|
-
if (this.hasExplicitId) {
|
|
594
|
-
const stateKey = `${BasicCrawler
|
|
1038
|
+
const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
1039
|
+
if (this.#identity.hasExplicitId) {
|
|
1040
|
+
const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
|
|
595
1041
|
return kvs.getAutoSavedValue(stateKey, defaultValue);
|
|
596
1042
|
}
|
|
597
|
-
BasicCrawler.
|
|
598
|
-
if (BasicCrawler.
|
|
599
|
-
|
|
1043
|
+
BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
|
|
1044
|
+
if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
|
|
1045
|
+
serviceLocator
|
|
1046
|
+
.getLogger()
|
|
1047
|
+
.warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
|
|
600
1048
|
'This means they will share the same state object, which is likely unintended. \n' +
|
|
601
1049
|
'To fix this, provide a unique `id` option to each crawler instance. \n' +
|
|
602
1050
|
'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
|
|
603
1051
|
}
|
|
604
|
-
return kvs.getAutoSavedValue(BasicCrawler
|
|
1052
|
+
return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
|
|
605
1053
|
}
|
|
606
|
-
|
|
607
|
-
return this.requestManager?.getPendingCount() ?? 0;
|
|
1054
|
+
async #getPendingRequestCountApproximation() {
|
|
1055
|
+
return (await this.requestManager?.getPendingCount()) ?? 0;
|
|
608
1056
|
}
|
|
609
|
-
calculateEnqueuedRequestLimit(explicitLimit) {
|
|
610
|
-
if (this
|
|
1057
|
+
async #calculateEnqueuedRequestLimit(explicitLimit) {
|
|
1058
|
+
if (this.#maxRequestsPerCrawl === undefined) {
|
|
611
1059
|
return explicitLimit;
|
|
612
1060
|
}
|
|
613
|
-
const limit = Math.max(0, this
|
|
1061
|
+
const limit = Math.max(0, this.#maxRequestsPerCrawl -
|
|
1062
|
+
this.#handledRequestsCount -
|
|
1063
|
+
(await this.#getPendingRequestCountApproximation()));
|
|
614
1064
|
return Math.min(limit, explicitLimit ?? Infinity);
|
|
615
1065
|
}
|
|
616
|
-
async handleSkippedRequest(options) {
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
this.log.info('The number of requests enqueued by the crawler reached the enqueueLinks limit.');
|
|
1066
|
+
async #handleSkippedRequest(options) {
|
|
1067
|
+
// A skipped request is a *successful* outcome, but the interrupt still unwinds through the
|
|
1068
|
+
// transaction scope, which rolls back - so the skip bookkeeping must write directly.
|
|
1069
|
+
await withDirectStorageAccess(async () => {
|
|
1070
|
+
// Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
|
|
1071
|
+
// against the logger for the crawler's lifetime rather than per run.
|
|
1072
|
+
if (options.reason === 'limit') {
|
|
1073
|
+
this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
|
|
1074
|
+
`${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
|
|
626
1075
|
}
|
|
627
|
-
|
|
628
|
-
|
|
1076
|
+
if (options.reason === 'depth') {
|
|
1077
|
+
this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
|
|
1078
|
+
}
|
|
1079
|
+
await this.#onSkippedRequest?.(options);
|
|
1080
|
+
});
|
|
629
1081
|
}
|
|
630
1082
|
/**
|
|
631
1083
|
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
|
|
@@ -633,76 +1085,148 @@ export class BasicCrawler {
|
|
|
633
1085
|
* the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
|
|
634
1086
|
* the `waitForAllRequestsToBeAdded` promise you get in the response object.
|
|
635
1087
|
*
|
|
1088
|
+
* Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
|
|
1089
|
+
* enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
|
|
1090
|
+
* relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
|
|
1091
|
+
* to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
|
|
1092
|
+
*
|
|
636
1093
|
* This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
|
|
637
1094
|
*
|
|
638
1095
|
* @param requests The requests to add
|
|
639
1096
|
* @param options Options for the request queue
|
|
640
1097
|
*/
|
|
641
1098
|
async addRequests(requests, options = {}) {
|
|
642
|
-
await this.
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
1099
|
+
await this.getRequestManager();
|
|
1100
|
+
if (!isIterable(requests) && !isAsyncIterable(requests)) {
|
|
1101
|
+
throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
|
|
1102
|
+
}
|
|
1103
|
+
parseArgument(options, addRequestsOptionsSchema, 'EnqueueUrlsOptions');
|
|
1104
|
+
// `label`/`userData` apply to every request this call produces, so a single upfront validation
|
|
1105
|
+
// against the label's schema covers them all and fails the whole call fast, rather than failing
|
|
1106
|
+
// lazily once the generator below is drained. Skipped when neither is set - each item still gets
|
|
1107
|
+
// its own per-item validation below, and validating an absent label/userData here would spuriously
|
|
1108
|
+
// check them against a registered default-route schema.
|
|
1109
|
+
if (options.label !== undefined || options.userData !== undefined) {
|
|
1110
|
+
await this.#validateRequestUserData({ label: options.label, userData: options.userData });
|
|
1111
|
+
}
|
|
1112
|
+
const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
|
|
1113
|
+
const strategy = options.strategy ?? EnqueueStrategy.All;
|
|
1114
|
+
const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
|
|
1115
|
+
const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
|
|
1116
|
+
// The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
|
|
1117
|
+
// (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
|
|
1118
|
+
const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
|
|
1119
|
+
const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
|
|
1120
|
+
const maxCrawlDepth = this.#maxCrawlDepth;
|
|
1121
|
+
const validateRequestUserData = this.#validateRequestUserData.bind(this);
|
|
1122
|
+
const allSkipped = [];
|
|
1123
|
+
// A skipped source (which can carry arbitrary userData) is only retained if something reads it -
|
|
1124
|
+
// otherwise the URL alone is enough to build the callback argument and to log with.
|
|
1125
|
+
const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
|
|
1126
|
+
const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
|
|
1127
|
+
const reportSkippedRequests = async () => {
|
|
1128
|
+
const skippedRequests = allSkipped.splice(0);
|
|
1129
|
+
if (skippedRequests.length === 0) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
const skippedRobotsUrls = skippedRequests
|
|
1133
|
+
.filter((s) => s.reason === 'robotsTxt')
|
|
1134
|
+
.map(({ source }) => (typeof source === 'string' ? source : source.url));
|
|
1135
|
+
if (skippedRobotsUrls.length > 0) {
|
|
1136
|
+
this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
|
|
1137
|
+
}
|
|
1138
|
+
// Only log the limit message when an explicit `limit` was passed (not the internal
|
|
1139
|
+
// `maxRequestsPerCrawl`-derived one), and only once per call.
|
|
1140
|
+
if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
|
|
1141
|
+
this.log.info(requestLimit === options.limit
|
|
1142
|
+
? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
|
|
1143
|
+
: `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
|
|
1144
|
+
}
|
|
1145
|
+
await Promise.all(skippedRequests.map(async ({ source, reason }) => {
|
|
1146
|
+
const args = createSkippedRequestArgs(source, reason);
|
|
1147
|
+
await this.#handleSkippedRequest(args);
|
|
1148
|
+
await options.onSkippedRequest?.(args);
|
|
1149
|
+
}));
|
|
1150
|
+
};
|
|
652
1151
|
async function* filteredRequests() {
|
|
653
|
-
let yieldedRequestCount = 0;
|
|
654
1152
|
for await (const request of requests) {
|
|
655
|
-
const
|
|
656
|
-
if (
|
|
657
|
-
|
|
1153
|
+
const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
|
|
1154
|
+
if (!requestOptions) {
|
|
1155
|
+
continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
|
|
1156
|
+
}
|
|
1157
|
+
if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
|
|
1158
|
+
allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
|
|
658
1159
|
continue;
|
|
659
1160
|
}
|
|
660
|
-
if (
|
|
661
|
-
|
|
1161
|
+
if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
|
|
1162
|
+
allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
|
|
662
1163
|
continue;
|
|
663
1164
|
}
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
1165
|
+
const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
|
|
1166
|
+
// Filter by user patterns first (with exclude)...
|
|
1167
|
+
let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
|
|
1168
|
+
// ...then filter by the enqueue strategy (making this an AND check)
|
|
1169
|
+
if (strategyBaseUrl) {
|
|
1170
|
+
filtered = filtered.filter((opts) => {
|
|
1171
|
+
const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
|
|
1172
|
+
if (!allowed)
|
|
1173
|
+
onSkippedByFilter(opts);
|
|
1174
|
+
return allowed;
|
|
1175
|
+
});
|
|
667
1176
|
}
|
|
668
|
-
|
|
669
|
-
|
|
1177
|
+
if (filtered.length === 0) {
|
|
1178
|
+
continue;
|
|
1179
|
+
}
|
|
1180
|
+
let [finalOptions] = filtered;
|
|
1181
|
+
if (options.transformRequestFunction) {
|
|
1182
|
+
const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
|
|
1183
|
+
if (transformed.length === 0) {
|
|
1184
|
+
continue;
|
|
1185
|
+
}
|
|
1186
|
+
[finalOptions] = transformed;
|
|
670
1187
|
}
|
|
1188
|
+
await validateRequestUserData(finalOptions);
|
|
1189
|
+
yield new Request(finalOptions);
|
|
671
1190
|
}
|
|
672
1191
|
}
|
|
673
|
-
const result = await this.requestManager.addRequestsBatched(filteredRequests(),
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
1192
|
+
const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
|
|
1193
|
+
forefront: options.forefront,
|
|
1194
|
+
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
|
|
1195
|
+
batchSize: options.batchSize,
|
|
1196
|
+
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
|
|
1197
|
+
maxNewRequests: requestLimit,
|
|
1198
|
+
});
|
|
1199
|
+
// Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
|
|
1200
|
+
// explicit `limit` option)
|
|
1201
|
+
for (const request of result.requestsOverLimit ?? []) {
|
|
1202
|
+
allSkipped.push({ source: request, reason: 'limit' });
|
|
1203
|
+
}
|
|
1204
|
+
await reportSkippedRequests();
|
|
1205
|
+
const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
|
|
1206
|
+
await reportSkippedRequests();
|
|
1207
|
+
return addedRequests;
|
|
1208
|
+
});
|
|
1209
|
+
// Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
|
|
1210
|
+
// when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
|
|
1211
|
+
// Background failures are already logged by `drainRequestBatches`.
|
|
1212
|
+
void waitForAllRequestsToBeAdded.catch(() => { });
|
|
1213
|
+
return { ...result, waitForAllRequestsToBeAdded };
|
|
693
1214
|
}
|
|
694
1215
|
/**
|
|
695
1216
|
* Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
|
|
696
1217
|
*/
|
|
697
|
-
async pushData(data,
|
|
698
|
-
|
|
1218
|
+
async pushData(data, datasetIdentifier) {
|
|
1219
|
+
tryCancel();
|
|
1220
|
+
const dataset = await this.getDataset(datasetIdentifier);
|
|
699
1221
|
return dataset.pushData(data);
|
|
700
1222
|
}
|
|
701
1223
|
/**
|
|
702
1224
|
* Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
|
|
703
1225
|
*/
|
|
704
|
-
async getDataset(
|
|
705
|
-
return Dataset.open(
|
|
1226
|
+
async getDataset(identifier) {
|
|
1227
|
+
return Dataset.open(identifier, {
|
|
1228
|
+
configuration: serviceLocator.getConfiguration(),
|
|
1229
|
+
});
|
|
706
1230
|
}
|
|
707
1231
|
/**
|
|
708
1232
|
* Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
|
|
@@ -717,8 +1241,9 @@ export class BasicCrawler {
|
|
|
717
1241
|
*/
|
|
718
1242
|
async exportData(path, format, options) {
|
|
719
1243
|
const supportedFormats = ['json', 'csv'];
|
|
720
|
-
|
|
721
|
-
|
|
1244
|
+
const formatMatch = /\.(json|csv)$/i.exec(path);
|
|
1245
|
+
if (!format && formatMatch) {
|
|
1246
|
+
format = formatMatch[1].toLowerCase();
|
|
722
1247
|
}
|
|
723
1248
|
if (!format) {
|
|
724
1249
|
throw new Error(`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`);
|
|
@@ -737,6 +1262,7 @@ export class BasicCrawler {
|
|
|
737
1262
|
const keys = options?.collectAllKeys
|
|
738
1263
|
? Array.from(new Set(items.flatMap(Object.keys)))
|
|
739
1264
|
: Object.keys(items[0]);
|
|
1265
|
+
const { stringify } = await import('csv-stringify/sync');
|
|
740
1266
|
value = stringify([
|
|
741
1267
|
keys,
|
|
742
1268
|
...items.map((item) => {
|
|
@@ -744,13 +1270,13 @@ export class BasicCrawler {
|
|
|
744
1270
|
}),
|
|
745
1271
|
]);
|
|
746
1272
|
}
|
|
747
|
-
await
|
|
1273
|
+
await mkdir(dirname(path), { recursive: true });
|
|
748
1274
|
await writeFile(path, value);
|
|
749
1275
|
this.log.info(`Export to ${path} finished!`);
|
|
750
1276
|
}
|
|
751
1277
|
if (format === 'json') {
|
|
752
|
-
await
|
|
753
|
-
await
|
|
1278
|
+
await mkdir(dirname(path), { recursive: true });
|
|
1279
|
+
await writeFile(path, `${JSON.stringify(items, null, 4)}\n`);
|
|
754
1280
|
this.log.info(`Export to ${path} finished!`);
|
|
755
1281
|
}
|
|
756
1282
|
return items;
|
|
@@ -758,58 +1284,194 @@ export class BasicCrawler {
|
|
|
758
1284
|
/**
|
|
759
1285
|
* Initializes the crawler.
|
|
760
1286
|
*/
|
|
761
|
-
async
|
|
1287
|
+
async init() {
|
|
1288
|
+
const run = this.#liveRun;
|
|
1289
|
+
if (run === undefined) {
|
|
1290
|
+
throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
|
|
1291
|
+
}
|
|
762
1292
|
const eventManager = serviceLocator.getEventManager();
|
|
763
1293
|
if (!eventManager.isInitialized()) {
|
|
764
1294
|
await eventManager.init();
|
|
765
|
-
|
|
766
|
-
}
|
|
767
|
-
//
|
|
768
|
-
//
|
|
769
|
-
// (
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
1295
|
+
run.ownsEventManager = true;
|
|
1296
|
+
}
|
|
1297
|
+
// Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
|
|
1298
|
+
// floored per request so it will not actually cut them short, but the configured value is then effectively
|
|
1299
|
+
// ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
|
|
1300
|
+
// navigation timeout only after `super()`.
|
|
1301
|
+
const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
|
|
1302
|
+
if (this.internalTimeoutMillis < phasesMillis) {
|
|
1303
|
+
this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
|
|
1304
|
+
`and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
|
|
1305
|
+
`so it does not cut them short.`);
|
|
1306
|
+
}
|
|
1307
|
+
await this.getRequestManager();
|
|
1308
|
+
}
|
|
1309
|
+
/**
|
|
1310
|
+
* The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
|
|
1311
|
+
* to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
|
|
1312
|
+
* browser crawlers override it with their `navigationTimeoutSecs`.
|
|
1313
|
+
*/
|
|
1314
|
+
getNavigationTimeoutMillis() {
|
|
1315
|
+
return 0;
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
|
|
1319
|
+
* that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
|
|
1320
|
+
* request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
|
|
1321
|
+
* whichever is larger: the configured internal timeout, or this request's combined phase budget.
|
|
1322
|
+
*/
|
|
1323
|
+
async #withRequestTimeout(crawlingContext, work) {
|
|
1324
|
+
const { request } = crawlingContext;
|
|
1325
|
+
const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
|
|
1326
|
+
const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
|
|
1327
|
+
await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
|
|
1328
|
+
}
|
|
1329
|
+
/**
|
|
1330
|
+
* The request handler timeout for a request with the given route label. A router route may override the
|
|
1331
|
+
* crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
|
|
1332
|
+
*
|
|
1333
|
+
* @param label The request's route label, or `undefined` for the default route / no specific request.
|
|
1334
|
+
* @param fallbackMillis Timeout to use when no route overrides it.
|
|
1335
|
+
*/
|
|
1336
|
+
#resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
|
|
1337
|
+
return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
|
|
1338
|
+
}
|
|
1339
|
+
/**
|
|
1340
|
+
* The timeout the router route with the given label asked for, or `undefined` when it did not override one
|
|
1341
|
+
* (or the request handler is not a router at all).
|
|
1342
|
+
*/
|
|
1343
|
+
#getRouteTimeoutMillis(label) {
|
|
1344
|
+
const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
|
|
1345
|
+
if (typeof getTimeoutSecs !== 'function') {
|
|
1346
|
+
return undefined;
|
|
775
1347
|
}
|
|
776
|
-
|
|
777
|
-
|
|
1348
|
+
const timeoutSecs = getTimeoutSecs(label);
|
|
1349
|
+
return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
|
|
778
1350
|
}
|
|
779
1351
|
async runRequestHandler(crawlingContext) {
|
|
780
|
-
|
|
781
|
-
|
|
1352
|
+
const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
|
|
1353
|
+
await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
|
|
1354
|
+
}
|
|
1355
|
+
/**
|
|
1356
|
+
* Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
|
|
1357
|
+
* Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
|
|
1358
|
+
* normal return says nothing about success. `handleRequest` commits on success.
|
|
1359
|
+
*/
|
|
1360
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
|
|
1361
|
+
async runInStorageTransaction(callback) {
|
|
1362
|
+
if (!this.#transactionalStorageEnabled) {
|
|
1363
|
+
return callback();
|
|
1364
|
+
}
|
|
1365
|
+
const transaction = createStorageTransaction({
|
|
1366
|
+
policy: this.#storageWritePolicy,
|
|
1367
|
+
commitTimeoutMillis: this.internalTimeoutMillis,
|
|
782
1368
|
});
|
|
1369
|
+
let threw = true;
|
|
1370
|
+
try {
|
|
1371
|
+
const result = await transaction.run(callback);
|
|
1372
|
+
threw = false;
|
|
1373
|
+
return result;
|
|
1374
|
+
}
|
|
1375
|
+
finally {
|
|
1376
|
+
if (transaction.state === 'open') {
|
|
1377
|
+
// `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
|
|
1378
|
+
// a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
|
|
1379
|
+
// expected. Either way, discard the unvalidated writes; only the former is worth flagging.
|
|
1380
|
+
if (!threw) {
|
|
1381
|
+
this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
|
|
1382
|
+
'returned normally. Its writes are being discarded. Please report this.');
|
|
1383
|
+
}
|
|
1384
|
+
transaction.rollback();
|
|
1385
|
+
}
|
|
1386
|
+
// Unconditional: `failed` is a terminal state that the branch above never reaches.
|
|
1387
|
+
transaction.dispose();
|
|
1388
|
+
}
|
|
783
1389
|
}
|
|
784
1390
|
/**
|
|
785
1391
|
* Handles blocked request
|
|
786
1392
|
*/
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
1393
|
+
throwOnBlockedRequest(statusCode) {
|
|
1394
|
+
if (this.retryOnBlocked)
|
|
1395
|
+
return;
|
|
1396
|
+
if (this.blockedStatusCodes.has(statusCode)) {
|
|
1397
|
+
throw new SessionError(`Request blocked - received ${statusCode} status code.`);
|
|
791
1398
|
}
|
|
792
1399
|
}
|
|
793
|
-
async isAllowedBasedOnRobotsTxtFile(url) {
|
|
794
|
-
if (!this
|
|
1400
|
+
async #isAllowedBasedOnRobotsTxtFile(url) {
|
|
1401
|
+
if (!this.#respectRobotsTxtFile) {
|
|
795
1402
|
return true;
|
|
796
1403
|
}
|
|
797
1404
|
const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
|
|
798
|
-
const userAgent = typeof this
|
|
1405
|
+
const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
|
|
1406
|
+
if (robotsTxtFile) {
|
|
1407
|
+
const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
|
|
1408
|
+
if (crawlDelay !== undefined) {
|
|
1409
|
+
this.#applyCrawlDelay(url, crawlDelay);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
799
1412
|
return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
|
|
800
1413
|
}
|
|
1414
|
+
/**
|
|
1415
|
+
* Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
|
|
1416
|
+
*
|
|
1417
|
+
* @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
|
|
1418
|
+
* @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
|
|
1419
|
+
* {@link RequestThrottledError} rather than treating the response as a blocked session.
|
|
1420
|
+
*/
|
|
1421
|
+
recordDomainRateLimit(url, retryAfterHeader) {
|
|
1422
|
+
if (this.requestManager?.recordPacingSignal({
|
|
1423
|
+
reason: 'rateLimited',
|
|
1424
|
+
url,
|
|
1425
|
+
waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
|
|
1426
|
+
})) {
|
|
1427
|
+
return true;
|
|
1428
|
+
}
|
|
1429
|
+
const domain = hostnameOrUrl(url);
|
|
1430
|
+
this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
|
|
1431
|
+
'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
|
|
1432
|
+
`\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
|
|
1433
|
+
'`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
|
|
1434
|
+
return false;
|
|
1435
|
+
}
|
|
1436
|
+
/** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
|
|
1437
|
+
#applyCrawlDelay(url, delaySeconds) {
|
|
1438
|
+
// robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
|
|
1439
|
+
// host share a clock).
|
|
1440
|
+
if (this.requestManager?.recordPacingSignal({
|
|
1441
|
+
reason: 'minInterval',
|
|
1442
|
+
url,
|
|
1443
|
+
intervalMs: delaySeconds * 1000,
|
|
1444
|
+
scope: 'hostname',
|
|
1445
|
+
})) {
|
|
1446
|
+
return;
|
|
1447
|
+
}
|
|
1448
|
+
const domain = hostnameOrUrl(url);
|
|
1449
|
+
this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
|
|
1450
|
+
'manager does not pace that domain, so its requests will not be paced. Set ' +
|
|
1451
|
+
`\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
|
|
1452
|
+
'`requestManager`.');
|
|
1453
|
+
}
|
|
1454
|
+
/** @internal */
|
|
801
1455
|
async getRobotsTxtFileForUrl(url) {
|
|
802
|
-
if (!this
|
|
1456
|
+
if (!this.#respectRobotsTxtFile) {
|
|
803
1457
|
return undefined;
|
|
804
1458
|
}
|
|
805
1459
|
try {
|
|
806
1460
|
const origin = new URL(url).origin;
|
|
807
|
-
const cachedRobotsTxtFile = this
|
|
1461
|
+
const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
|
|
808
1462
|
if (cachedRobotsTxtFile) {
|
|
809
1463
|
return cachedRobotsTxtFile;
|
|
810
1464
|
}
|
|
811
|
-
|
|
812
|
-
|
|
1465
|
+
// Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
|
|
1466
|
+
// fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
|
|
1467
|
+
// alone would then decide whether everything is allowed or disallowed.
|
|
1468
|
+
const robotsTxtFile = await RobotsTxtFile.find(url, {
|
|
1469
|
+
httpClient: this.httpClient,
|
|
1470
|
+
session: await this.sessionPool.getSession(),
|
|
1471
|
+
timeoutMillis: 30_000,
|
|
1472
|
+
logger: this.log,
|
|
1473
|
+
});
|
|
1474
|
+
this.#robotsTxtFileCache.add(origin, robotsTxtFile);
|
|
813
1475
|
return robotsTxtFile;
|
|
814
1476
|
}
|
|
815
1477
|
catch (e) {
|
|
@@ -817,10 +1479,10 @@ export class BasicCrawler {
|
|
|
817
1479
|
return undefined;
|
|
818
1480
|
}
|
|
819
1481
|
}
|
|
820
|
-
async
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
await
|
|
1482
|
+
async #pauseOnMigration() {
|
|
1483
|
+
const run = this.#dispatchingRun;
|
|
1484
|
+
if (run) {
|
|
1485
|
+
await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
|
|
824
1486
|
if (err.message.includes('running tasks did not finish')) {
|
|
825
1487
|
this.log.error('The crawler was paused due to migration to another host, ' +
|
|
826
1488
|
"but some requests did not finish in time. Those requests' results may be duplicated.");
|
|
@@ -830,233 +1492,115 @@ export class BasicCrawler {
|
|
|
830
1492
|
}
|
|
831
1493
|
});
|
|
832
1494
|
}
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
return;
|
|
837
|
-
await this.requestList.persistState().catch((err) => {
|
|
838
|
-
if (err.message.includes('Cannot persist state.')) {
|
|
839
|
-
this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
|
|
840
|
-
'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
|
|
841
|
-
'constructor to ensure your crawling state is persisted through host migrations and restarts.');
|
|
842
|
-
}
|
|
843
|
-
else {
|
|
844
|
-
this.log.exception(err, 'An unexpected error occurred when the crawler ' +
|
|
845
|
-
"attempted to persist its request list's state.");
|
|
846
|
-
}
|
|
847
|
-
});
|
|
848
|
-
}
|
|
849
|
-
})();
|
|
850
|
-
await Promise.all([requestListPersistPromise, this.stats.persistState()]);
|
|
851
|
-
}
|
|
852
|
-
/**
|
|
853
|
-
* Initializes the RequestManager based on the configured requestList and requestQueue.
|
|
854
|
-
*/
|
|
855
|
-
async initializeRequestManager() {
|
|
856
|
-
if (this.requestManager !== undefined) {
|
|
857
|
-
return;
|
|
858
|
-
}
|
|
859
|
-
if (this.requestList && this.requestQueue) {
|
|
860
|
-
// Create a RequestManagerTandem if both RequestList and RequestQueue are provided
|
|
861
|
-
this.requestManager = new RequestManagerTandem(this.requestList, this.requestQueue);
|
|
862
|
-
}
|
|
863
|
-
else if (this.requestQueue) {
|
|
864
|
-
// Use RequestQueue directly if only it is provided
|
|
865
|
-
this.requestManager = this.requestQueue;
|
|
866
|
-
}
|
|
867
|
-
else if (this.requestList) {
|
|
868
|
-
// Use RequestList directly if only it is provided
|
|
869
|
-
// Make it compatible with the IRequestManager interface
|
|
870
|
-
this.requestManager = new RequestListAdapter(this.requestList);
|
|
871
|
-
}
|
|
872
|
-
// If neither RequestList nor RequestQueue is provided, leave the requestManager uninitialized until `getRequestQueue` is called
|
|
1495
|
+
// Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
|
|
1496
|
+
// the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
|
|
1497
|
+
await this.statistics.persistState?.();
|
|
873
1498
|
}
|
|
874
1499
|
/**
|
|
875
1500
|
* Fetches the next request to process from the underlying request provider.
|
|
876
1501
|
*/
|
|
877
|
-
async
|
|
1502
|
+
async #fetchNextRequest() {
|
|
878
1503
|
if (this.requestManager === undefined) {
|
|
879
|
-
throw new Error(`
|
|
1504
|
+
throw new Error(`fetchNextRequest called on an uninitialized crawler`);
|
|
880
1505
|
}
|
|
881
1506
|
return this.requestManager.fetchNextRequest();
|
|
882
1507
|
}
|
|
883
|
-
/**
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
return false;
|
|
892
|
-
}
|
|
893
|
-
const now = Date.now();
|
|
894
|
-
const lastAccessTime = this.domainAccessedTime.get(domain);
|
|
895
|
-
if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
|
|
896
|
-
this.domainAccessedTime.set(domain, now);
|
|
897
|
-
return false;
|
|
898
|
-
}
|
|
899
|
-
if (source instanceof RequestQueueV1) {
|
|
900
|
-
// eslint-disable-next-line dot-notation
|
|
901
|
-
source['inProgress']?.delete(request.id);
|
|
1508
|
+
/** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
|
|
1509
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1510
|
+
async handleRequest(crawlingContext, requestSource) {
|
|
1511
|
+
// An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
|
|
1512
|
+
// which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
|
|
1513
|
+
// execute (and re-report) on top of a request the crawler has already moved past.
|
|
1514
|
+
if (crawlingContext[timeoutExpiredKey]?.()) {
|
|
1515
|
+
return;
|
|
902
1516
|
}
|
|
903
|
-
const
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
1517
|
+
const { request } = crawlingContext;
|
|
1518
|
+
request.state = RequestState.REQUEST_HANDLER;
|
|
1519
|
+
await this.runRequestHandler(crawlingContext);
|
|
1520
|
+
// Commit *before* marking the request as handled, so a commit failure fails the request and
|
|
1521
|
+
// it is retried. This also closes the transaction, so everything below passes through.
|
|
1522
|
+
// Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
|
|
1523
|
+
await currentStorageTransaction()?.commit();
|
|
1524
|
+
await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1525
|
+
this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
|
|
1526
|
+
// reclaim session if request finishes successfully
|
|
1527
|
+
request.state = RequestState.DONE;
|
|
1528
|
+
crawlingContext.session.markGood();
|
|
914
1529
|
}
|
|
915
1530
|
/**
|
|
916
|
-
*
|
|
917
|
-
*
|
|
1531
|
+
* Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
|
|
1532
|
+
* or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
|
|
1533
|
+
* error handler has seen the context.
|
|
918
1534
|
*/
|
|
919
|
-
async
|
|
920
|
-
const
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
}
|
|
934
|
-
: undefined;
|
|
935
|
-
tryCancel();
|
|
936
|
-
if (!request || this.delayRequest(request, source)) {
|
|
937
|
-
return;
|
|
1535
|
+
async #handleRequestFailure(rawError, crawlingContext, requestSource) {
|
|
1536
|
+
const { request } = crawlingContext;
|
|
1537
|
+
// Roll back *before* any error handler runs - error handlers write to real storage precisely
|
|
1538
|
+
// because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
|
|
1539
|
+
currentStorageTransaction()?.rollback();
|
|
1540
|
+
const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
|
|
1541
|
+
try {
|
|
1542
|
+
const unwrappedError = this.#unwrapError(rawError);
|
|
1543
|
+
// A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
|
|
1544
|
+
const err = unwrappedError instanceof ContextPipelineInterruptedError
|
|
1545
|
+
? await this.#trySkipRequest(unwrappedError, request, requestSource)
|
|
1546
|
+
: unwrappedError;
|
|
1547
|
+
if (err) {
|
|
1548
|
+
await this.#runErrorHandler(err, crawlingContext, requestSource);
|
|
1549
|
+
}
|
|
938
1550
|
}
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
request.state = RequestState.SKIPPED;
|
|
942
|
-
request.noRetry = true;
|
|
943
|
-
await source.markRequestHandled(request);
|
|
944
|
-
await this.handleSkippedRequest({
|
|
945
|
-
url: request.url,
|
|
946
|
-
reason: 'robotsTxt',
|
|
947
|
-
});
|
|
948
|
-
return;
|
|
1551
|
+
finally {
|
|
1552
|
+
resumeRequestTimeout?.();
|
|
949
1553
|
}
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
const
|
|
953
|
-
|
|
954
|
-
const deferredCleanup = [];
|
|
955
|
-
const crawlingContext = {
|
|
956
|
-
id: cryptoRandomObjectId(10),
|
|
957
|
-
log: this.log,
|
|
958
|
-
request,
|
|
959
|
-
session,
|
|
960
|
-
proxyInfo: session?.proxyInfo,
|
|
961
|
-
enqueueLinks: async (options) => {
|
|
962
|
-
const requestQueue = await this.getRequestQueue();
|
|
963
|
-
return await this.enqueueLinksWithCrawlDepth(options, request, requestQueue);
|
|
964
|
-
},
|
|
965
|
-
addRequests: async (requests, options = {}) => {
|
|
966
|
-
const newCrawlDepth = request.crawlDepth + 1;
|
|
967
|
-
const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
|
|
968
|
-
await this.addRequests(requestsGenerator, options);
|
|
969
|
-
},
|
|
970
|
-
pushData: this.pushData.bind(this),
|
|
971
|
-
useState: this.useState.bind(this),
|
|
972
|
-
sendRequest: createSendRequest(this.httpClient, request, session),
|
|
973
|
-
getKeyValueStore: async (idOrName) => KeyValueStore.open(idOrName, { config: serviceLocator.getConfiguration() }),
|
|
974
|
-
registerDeferredCleanup: (cleanup) => {
|
|
975
|
-
deferredCleanup.push(cleanup);
|
|
976
|
-
},
|
|
977
|
-
};
|
|
978
|
-
let isRequestLocked = true;
|
|
1554
|
+
}
|
|
1555
|
+
async #runErrorHandler(err, context, requestSource) {
|
|
1556
|
+
const { request } = context;
|
|
1557
|
+
const retryCountBefore = request.retryCount;
|
|
979
1558
|
try {
|
|
980
|
-
request.state = RequestState.
|
|
981
|
-
await this.
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
this.stats.finishJob(statisticsId, request.retryCount);
|
|
985
|
-
this.handledRequestsCount++;
|
|
986
|
-
// reclaim session if request finishes successfully
|
|
987
|
-
request.state = RequestState.DONE;
|
|
988
|
-
crawlingContext.session?.markGood();
|
|
989
|
-
}
|
|
990
|
-
catch (rawError) {
|
|
991
|
-
const err = this.unwrapError(rawError);
|
|
992
|
-
try {
|
|
993
|
-
request.state = RequestState.ERROR_HANDLER;
|
|
994
|
-
await addTimeoutToPromise(async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
995
|
-
if (!(err instanceof CriticalError)) {
|
|
996
|
-
isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestHandled or reclaimRequest
|
|
997
|
-
}
|
|
1559
|
+
request.state = RequestState.ERROR_HANDLER;
|
|
1560
|
+
await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1561
|
+
// The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
|
|
1562
|
+
if (request.state !== RequestState.SKIPPED) {
|
|
998
1563
|
request.state = RequestState.DONE;
|
|
999
1564
|
}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1565
|
+
}
|
|
1566
|
+
catch (secondaryError) {
|
|
1567
|
+
const unwrappedSecondaryError = this.#unwrapError(secondaryError);
|
|
1568
|
+
const timedOut = unwrappedSecondaryError instanceof TimeoutError;
|
|
1569
|
+
request.state = RequestState.ERROR;
|
|
1570
|
+
if (timedOut) {
|
|
1571
|
+
// Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
|
|
1572
|
+
// retry is counted, so that an error handler that always times out does not retry it forever.
|
|
1573
|
+
this.log.warning(unwrappedSecondaryError.message);
|
|
1574
|
+
if (request.retryCount === retryCountBefore) {
|
|
1575
|
+
request.retryCount++;
|
|
1010
1576
|
}
|
|
1011
|
-
|
|
1577
|
+
}
|
|
1578
|
+
else if (!(unwrappedSecondaryError instanceof CriticalError)) {
|
|
1579
|
+
// avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
|
|
1580
|
+
const apifySpecific = process.env.APIFY_IS_AT_HOME
|
|
1581
|
+
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
|
|
1582
|
+
: '';
|
|
1583
|
+
this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
|
|
1584
|
+
`This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
|
|
1585
|
+
}
|
|
1586
|
+
// Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
|
|
1587
|
+
// CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
|
|
1588
|
+
try {
|
|
1589
|
+
await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
|
|
1590
|
+
}
|
|
1591
|
+
catch {
|
|
1592
|
+
// The request was never in progress, or could not be reclaimed. Either way it's fine.
|
|
1593
|
+
}
|
|
1594
|
+
if (!timedOut) {
|
|
1012
1595
|
throw unwrappedSecondaryError;
|
|
1013
1596
|
}
|
|
1014
|
-
// decrease the session score if the request fails (but the error handler did not throw)
|
|
1015
|
-
crawlingContext.session?.markBad();
|
|
1016
1597
|
}
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
try {
|
|
1022
|
-
await source.client.deleteRequestLock(request.id);
|
|
1023
|
-
}
|
|
1024
|
-
catch {
|
|
1025
|
-
// We don't have the lock, or the request was never locked. Either way it's fine
|
|
1026
|
-
}
|
|
1027
|
-
}
|
|
1598
|
+
// decrease the session score if the request fails (but the error handler did not throw);
|
|
1599
|
+
// skip when the error is a SessionError, which already retired the session
|
|
1600
|
+
if (!this.#errorAbsolvesSession(err)) {
|
|
1601
|
+
context.session?.markBad();
|
|
1028
1602
|
}
|
|
1029
1603
|
}
|
|
1030
|
-
/**
|
|
1031
|
-
* Wrapper around the crawling context's `enqueueLinks` method:
|
|
1032
|
-
* - Injects `crawlDepth` to each request being added based on the crawling context request.
|
|
1033
|
-
* - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
|
|
1034
|
-
* - These options can be overridden by the user.
|
|
1035
|
-
* @internal
|
|
1036
|
-
*/
|
|
1037
|
-
async enqueueLinksWithCrawlDepth(options, request, requestQueue) {
|
|
1038
|
-
const transformRequestFunctionWrapper = (requestOptions) => {
|
|
1039
|
-
requestOptions.crawlDepth = request.crawlDepth + 1;
|
|
1040
|
-
if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
|
|
1041
|
-
// Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
|
|
1042
|
-
// reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
|
|
1043
|
-
// rather than the generic `'transform'` reason.
|
|
1044
|
-
requestOptions.skippedReason = 'depth';
|
|
1045
|
-
return false;
|
|
1046
|
-
}
|
|
1047
|
-
// After injecting the crawlDepth, we call the user-provided transform function, if there is one.
|
|
1048
|
-
return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
|
|
1049
|
-
};
|
|
1050
|
-
return await enqueueLinks({
|
|
1051
|
-
requestQueue,
|
|
1052
|
-
robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
|
|
1053
|
-
onSkippedRequest: this.handleSkippedRequest,
|
|
1054
|
-
limit: this.calculateEnqueuedRequestLimit(options.limit),
|
|
1055
|
-
// Allow user options to override defaults set above ⤴
|
|
1056
|
-
...options,
|
|
1057
|
-
transformRequestFunction: transformRequestFunctionWrapper,
|
|
1058
|
-
});
|
|
1059
|
-
}
|
|
1060
1604
|
/**
|
|
1061
1605
|
* Generator function that yields requests injected with the given crawl depth.
|
|
1062
1606
|
* @internal
|
|
@@ -1066,6 +1610,11 @@ export class BasicCrawler {
|
|
|
1066
1610
|
if (typeof request === 'string') {
|
|
1067
1611
|
yield { url: request, crawlDepth: newRequestDepth };
|
|
1068
1612
|
}
|
|
1613
|
+
else if (request instanceof Request) {
|
|
1614
|
+
const crawlee = (request.userData.__crawlee ??= {});
|
|
1615
|
+
crawlee.crawlDepth ??= newRequestDepth;
|
|
1616
|
+
yield request;
|
|
1617
|
+
}
|
|
1069
1618
|
else {
|
|
1070
1619
|
request.crawlDepth ??= newRequestDepth;
|
|
1071
1620
|
yield request;
|
|
@@ -1076,7 +1625,7 @@ export class BasicCrawler {
|
|
|
1076
1625
|
* Run async callback with given timeout and retry. Returns the result of the callback.
|
|
1077
1626
|
* @ignore
|
|
1078
1627
|
*/
|
|
1079
|
-
async
|
|
1628
|
+
async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
|
|
1080
1629
|
try {
|
|
1081
1630
|
return await addTimeoutToPromise(handler, timeout, error);
|
|
1082
1631
|
}
|
|
@@ -1084,116 +1633,155 @@ export class BasicCrawler {
|
|
|
1084
1633
|
if (retried <= maxRetries) {
|
|
1085
1634
|
// we retry on any error, not just timeout
|
|
1086
1635
|
this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
|
|
1087
|
-
return this
|
|
1636
|
+
return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
|
|
1088
1637
|
}
|
|
1089
1638
|
throw e;
|
|
1090
1639
|
}
|
|
1091
1640
|
}
|
|
1092
1641
|
/**
|
|
1093
|
-
*
|
|
1094
|
-
|
|
1095
|
-
async _isTaskReadyFunction() {
|
|
1096
|
-
return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
|
|
1097
|
-
}
|
|
1098
|
-
/**
|
|
1099
|
-
* Returns true if both RequestList and RequestQueue have all requests finished.
|
|
1642
|
+
* Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
|
|
1643
|
+
* wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
|
|
1100
1644
|
*/
|
|
1101
|
-
async
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
const
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1645
|
+
async #isTaskReadyFunction() {
|
|
1646
|
+
if (this.requestManager === undefined) {
|
|
1647
|
+
return false;
|
|
1648
|
+
}
|
|
1649
|
+
const state = await this.requestManager.checkReadiness();
|
|
1650
|
+
if (state.status === 'waiting' && state.readyAt !== undefined) {
|
|
1651
|
+
this.#run?.scheduleWake(state.readyAt);
|
|
1652
|
+
}
|
|
1653
|
+
return state.status === 'ready';
|
|
1109
1654
|
}
|
|
1110
1655
|
/**
|
|
1111
1656
|
* Unwraps errors thrown by the context pipeline to get the actual user error.
|
|
1112
|
-
* RequestHandlerError and
|
|
1657
|
+
* RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
|
|
1113
1658
|
*/
|
|
1114
|
-
unwrapError(error) {
|
|
1659
|
+
#unwrapError(error) {
|
|
1115
1660
|
if (error instanceof RequestHandlerError ||
|
|
1116
1661
|
error instanceof ContextPipelineInitializationError ||
|
|
1117
1662
|
error instanceof ContextPipelineCleanupError) {
|
|
1118
|
-
return this
|
|
1663
|
+
return this.#unwrapError(error.cause);
|
|
1119
1664
|
}
|
|
1120
1665
|
return error;
|
|
1121
1666
|
}
|
|
1122
1667
|
/**
|
|
1123
1668
|
* Handles errors thrown by user provided requestHandler()
|
|
1124
1669
|
*/
|
|
1125
|
-
|
|
1670
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1671
|
+
async requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
1126
1672
|
const { request } = crawlingContext;
|
|
1673
|
+
if (error instanceof RequestThrottledError) {
|
|
1674
|
+
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1675
|
+
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
1676
|
+
this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
|
|
1677
|
+
id: request.id,
|
|
1678
|
+
url: request.url,
|
|
1679
|
+
});
|
|
1680
|
+
await source.reclaimRequest(request, { forefront: true });
|
|
1681
|
+
return;
|
|
1682
|
+
}
|
|
1127
1683
|
request.pushErrorMessage(error);
|
|
1128
1684
|
if (error instanceof CriticalError) {
|
|
1129
1685
|
throw error;
|
|
1130
1686
|
}
|
|
1131
|
-
const shouldRetryRequest = this
|
|
1687
|
+
const shouldRetryRequest = this.#canRequestBeRetried(request, error);
|
|
1132
1688
|
if (shouldRetryRequest) {
|
|
1133
|
-
await this.
|
|
1134
|
-
|
|
1135
|
-
|
|
1689
|
+
await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
|
|
1690
|
+
try {
|
|
1691
|
+
await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
|
|
1692
|
+
error);
|
|
1693
|
+
}
|
|
1694
|
+
catch (handlerError) {
|
|
1695
|
+
if (handlerError instanceof ContextPipelineInterruptedError) {
|
|
1696
|
+
const skipError = await this.#trySkipRequest(handlerError, request, source);
|
|
1697
|
+
if (!skipError) {
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
// Retry for the original error. Routing `skipError` through `errorHandler` again
|
|
1701
|
+
// would loop forever with a handler that always skips.
|
|
1702
|
+
this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
|
|
1703
|
+
error: skipError.message,
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
else {
|
|
1707
|
+
throw handlerError;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1136
1710
|
if (error instanceof SessionError) {
|
|
1137
|
-
|
|
1711
|
+
crawlingContext.session?.retire();
|
|
1138
1712
|
}
|
|
1139
1713
|
if (!request.noRetry) {
|
|
1140
1714
|
request.retryCount++;
|
|
1715
|
+
this.statistics.registerRetry?.(request.retryCount);
|
|
1141
1716
|
const { url, retryCount, id } = request;
|
|
1142
1717
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
1143
1718
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
1144
|
-
const message = this.
|
|
1719
|
+
const message = this.getMessageFromError(error);
|
|
1145
1720
|
this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
|
|
1146
1721
|
id,
|
|
1147
1722
|
url,
|
|
1148
1723
|
retryCount,
|
|
1149
1724
|
});
|
|
1150
|
-
await source.reclaimRequest(request
|
|
1725
|
+
await source.reclaimRequest(request);
|
|
1151
1726
|
return;
|
|
1152
1727
|
}
|
|
1153
1728
|
}
|
|
1729
|
+
if (error instanceof SessionError) {
|
|
1730
|
+
crawlingContext.session?.retire();
|
|
1731
|
+
}
|
|
1154
1732
|
// If the request is non-retryable, the error and snapshot aren't saved in the errorTrackerRetry object.
|
|
1155
1733
|
// Therefore, we pass the crawlingContext to the errorTracker.add method, enabling snapshot capture.
|
|
1156
1734
|
// This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
|
|
1157
1735
|
const { noRetry, maxRetries } = request;
|
|
1158
1736
|
if (noRetry || !maxRetries) {
|
|
1159
|
-
await this.
|
|
1737
|
+
await this.statistics.errorTracker.addAsync(error, crawlingContext);
|
|
1160
1738
|
}
|
|
1161
1739
|
else {
|
|
1162
|
-
this.
|
|
1740
|
+
this.statistics.errorTracker.add(error);
|
|
1163
1741
|
}
|
|
1164
1742
|
// If we get here, the request is either not retryable
|
|
1165
1743
|
// or failed more than retryCount times and will not be retried anymore.
|
|
1166
1744
|
// Mark the request as failed and do not retry.
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
this.
|
|
1170
|
-
await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
|
|
1745
|
+
await source.markRequestAsHandled(request);
|
|
1746
|
+
this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
|
|
1747
|
+
await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
|
|
1171
1748
|
}
|
|
1172
|
-
|
|
1749
|
+
/** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
|
|
1750
|
+
async #trySkipRequest(error, request, source) {
|
|
1173
1751
|
try {
|
|
1174
|
-
|
|
1752
|
+
await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1753
|
+
request.state = RequestState.SKIPPED;
|
|
1754
|
+
request.noRetry = true;
|
|
1755
|
+
await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
1756
|
+
this.statistics.discardRequestRecord(request.id || request.uniqueKey);
|
|
1757
|
+
return undefined;
|
|
1175
1758
|
}
|
|
1176
|
-
catch (
|
|
1177
|
-
|
|
1178
|
-
throw e;
|
|
1759
|
+
catch (settleError) {
|
|
1760
|
+
return this.#unwrapError(settleError);
|
|
1179
1761
|
}
|
|
1180
1762
|
}
|
|
1181
|
-
|
|
1763
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1764
|
+
async handleFailedRequestHandler(crawlingContext, error) {
|
|
1182
1765
|
// Always log the last error regardless if the user provided a failedRequestHandler
|
|
1183
1766
|
const { id, url, method, uniqueKey } = crawlingContext.request;
|
|
1184
|
-
const message = this.
|
|
1767
|
+
const message = this.getMessageFromError(error, true);
|
|
1185
1768
|
this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
|
|
1186
|
-
|
|
1187
|
-
await this
|
|
1769
|
+
try {
|
|
1770
|
+
await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
|
|
1188
1771
|
error);
|
|
1189
1772
|
}
|
|
1773
|
+
catch (handlerError) {
|
|
1774
|
+
if (!(handlerError instanceof ContextPipelineInterruptedError))
|
|
1775
|
+
throw handlerError;
|
|
1776
|
+
this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
|
|
1777
|
+
}
|
|
1190
1778
|
}
|
|
1191
1779
|
/**
|
|
1192
1780
|
* Resolves the most verbose error message from a thrown error
|
|
1193
1781
|
* @param error The error received
|
|
1194
1782
|
* @returns The message to be logged
|
|
1195
1783
|
*/
|
|
1196
|
-
|
|
1784
|
+
getMessageFromError(error, forceStack = false) {
|
|
1197
1785
|
if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
|
|
1198
1786
|
forceStack = true;
|
|
1199
1787
|
}
|
|
@@ -1201,17 +1789,25 @@ export class BasicCrawler {
|
|
|
1201
1789
|
const baseDir = process.cwd();
|
|
1202
1790
|
const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
|
|
1203
1791
|
if (error instanceof TimeoutError) {
|
|
1204
|
-
|
|
1792
|
+
// stack in timeout errors does not really help
|
|
1793
|
+
return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
|
|
1205
1794
|
}
|
|
1206
1795
|
return process.env.CRAWLEE_VERBOSE_LOG || forceStack
|
|
1207
1796
|
? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
|
|
1208
1797
|
: [error.message || error, userLine].join('\n');
|
|
1209
1798
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1799
|
+
/**
|
|
1800
|
+
* Whether the session should be spared for this error - either because it was already retired, or because the
|
|
1801
|
+
* failure says nothing about the session (a rate limit is a property of the domain).
|
|
1802
|
+
*/
|
|
1803
|
+
#errorAbsolvesSession(error) {
|
|
1804
|
+
return (error instanceof SessionError ||
|
|
1805
|
+
error instanceof RequestThrottledError ||
|
|
1806
|
+
error instanceof ContextPipelineInterruptedError);
|
|
1807
|
+
}
|
|
1808
|
+
#canRequestBeRetried(request, error) {
|
|
1809
|
+
// Request should never be retried, or the error encountered makes it not able to be retried.
|
|
1810
|
+
if (request.noRetry || error instanceof NonRetryableError) {
|
|
1215
1811
|
return false;
|
|
1216
1812
|
}
|
|
1217
1813
|
// User requested retry (we ignore retry count here as its explicitly told by the user to retry)
|
|
@@ -1219,61 +1815,68 @@ export class BasicCrawler {
|
|
|
1219
1815
|
return true;
|
|
1220
1816
|
}
|
|
1221
1817
|
// Ensure there are more retries available for the request
|
|
1222
|
-
const maxRequestRetries = request.maxRetries ?? this
|
|
1818
|
+
const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
|
|
1223
1819
|
return request.retryCount < maxRequestRetries;
|
|
1224
1820
|
}
|
|
1225
1821
|
/**
|
|
1226
|
-
*
|
|
1227
|
-
*/
|
|
1228
|
-
async _loadHandledRequestCount() {
|
|
1229
|
-
if (this.requestManager) {
|
|
1230
|
-
this.handledRequestsCount = await this.requestManager.handledCount();
|
|
1231
|
-
}
|
|
1232
|
-
}
|
|
1233
|
-
async _executeHooks(hooks, ...args) {
|
|
1234
|
-
if (Array.isArray(hooks) && hooks.length) {
|
|
1235
|
-
for (const hook of hooks) {
|
|
1236
|
-
await hook(...args);
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1239
|
-
}
|
|
1240
|
-
/**
|
|
1241
|
-
* Stops the crawler immediately.
|
|
1822
|
+
* Ends the run in progress immediately, without waiting for the requests already in flight.
|
|
1242
1823
|
*
|
|
1243
|
-
* This
|
|
1824
|
+
* This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
|
|
1825
|
+
* a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
|
|
1826
|
+
* released by {@link BasicCrawler.destroy|`destroy()`} instead.
|
|
1244
1827
|
*
|
|
1245
1828
|
* To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
|
|
1246
1829
|
*/
|
|
1247
1830
|
async teardown() {
|
|
1248
|
-
serviceLocator.getEventManager()
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1831
|
+
const eventManager = serviceLocator.getEventManager();
|
|
1832
|
+
const ownsEventManager = this.#run?.ownsEventManager === true;
|
|
1833
|
+
// First, so that the pool writes its final state once - its teardown stops it listening before the
|
|
1834
|
+
// persistence event below goes out.
|
|
1835
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
|
|
1836
|
+
// When this crawler initialized the event manager, its close() call emits
|
|
1837
|
+
// the final persistence event after the crawler-specific state has been
|
|
1838
|
+
// saved. External event managers still need an explicit event here.
|
|
1839
|
+
if (!ownsEventManager) {
|
|
1840
|
+
eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
|
|
1841
|
+
}
|
|
1842
|
+
if (ownsEventManager) {
|
|
1843
|
+
await eventManager.close();
|
|
1844
|
+
}
|
|
1845
|
+
await this.#run?.end();
|
|
1254
1846
|
}
|
|
1255
|
-
|
|
1847
|
+
/**
|
|
1848
|
+
* Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
|
|
1849
|
+
* `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
|
|
1850
|
+
* could otherwise have run again.
|
|
1851
|
+
*/
|
|
1852
|
+
async destroy() {
|
|
1853
|
+
// An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
|
|
1854
|
+
await this.#run?.end();
|
|
1855
|
+
}
|
|
1856
|
+
async [Symbol.asyncDispose]() {
|
|
1857
|
+
await this.destroy();
|
|
1858
|
+
}
|
|
1859
|
+
getCookieHeaderFromRequest(request) {
|
|
1256
1860
|
if (request.headers?.Cookie && request.headers?.cookie) {
|
|
1257
1861
|
this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
|
|
1258
1862
|
return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
|
|
1259
1863
|
}
|
|
1260
1864
|
return request.headers?.Cookie || request.headers?.cookie || '';
|
|
1261
1865
|
}
|
|
1262
|
-
|
|
1263
|
-
//
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1866
|
+
#requestMatchesEnqueueStrategy(request) {
|
|
1867
|
+
// If `skipNavigation` was used, just return `true`
|
|
1868
|
+
try {
|
|
1869
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
|
|
1870
|
+
request.loadedUrl;
|
|
1871
|
+
}
|
|
1872
|
+
catch (err) {
|
|
1873
|
+
if (err instanceof NavigationSkippedError) {
|
|
1874
|
+
return true;
|
|
1268
1875
|
}
|
|
1269
|
-
|
|
1876
|
+
throw err;
|
|
1270
1877
|
}
|
|
1271
|
-
return RequestQueue.open(null, { config: serviceLocator.getConfiguration() });
|
|
1272
|
-
}
|
|
1273
|
-
requestMatchesEnqueueStrategy(request) {
|
|
1274
1878
|
const { url, loadedUrl } = request;
|
|
1275
|
-
|
|
1276
|
-
const strategy = request['enqueueStrategy'];
|
|
1879
|
+
const strategy = request.enqueueStrategy;
|
|
1277
1880
|
// No strategy set, so we assume it matches, or it was added outside of enqueueLinks
|
|
1278
1881
|
if (!strategy) {
|
|
1279
1882
|
return true;
|
|
@@ -1282,57 +1885,13 @@ export class BasicCrawler {
|
|
|
1282
1885
|
if (!loadedUrl) {
|
|
1283
1886
|
return true;
|
|
1284
1887
|
}
|
|
1285
|
-
|
|
1286
|
-
const loadedBaseUrl = new URL(loadedUrl);
|
|
1287
|
-
switch (strategy) {
|
|
1288
|
-
case EnqueueStrategy.SameHostname: {
|
|
1289
|
-
return baseUrl.hostname === loadedBaseUrl.hostname;
|
|
1290
|
-
}
|
|
1291
|
-
case EnqueueStrategy.SameDomain: {
|
|
1292
|
-
const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
|
|
1293
|
-
if (baseUrlHostname) {
|
|
1294
|
-
const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
|
|
1295
|
-
return baseUrlHostname === loadedBaseUrlHostname;
|
|
1296
|
-
}
|
|
1297
|
-
// Can happen for IPs, we just check like same origin
|
|
1298
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1299
|
-
}
|
|
1300
|
-
case EnqueueStrategy.SameOrigin: {
|
|
1301
|
-
// Same as hostname, but also checks protocol
|
|
1302
|
-
return baseUrl.origin === loadedBaseUrl.origin;
|
|
1303
|
-
}
|
|
1304
|
-
case EnqueueStrategy.All:
|
|
1305
|
-
default: {
|
|
1306
|
-
return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1888
|
+
return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
|
|
1309
1889
|
}
|
|
1310
1890
|
}
|
|
1311
|
-
/**
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
*
|
|
1316
|
-
* > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
|
|
1317
|
-
*
|
|
1318
|
-
* ```ts
|
|
1319
|
-
* import { BasicCrawler, createBasicRouter } from 'crawlee';
|
|
1320
|
-
*
|
|
1321
|
-
* const router = createBasicRouter();
|
|
1322
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
1323
|
-
* ctx.log.info('...');
|
|
1324
|
-
* });
|
|
1325
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
1326
|
-
* ctx.log.info('...');
|
|
1327
|
-
* });
|
|
1328
|
-
*
|
|
1329
|
-
* const crawler = new BasicCrawler({
|
|
1330
|
-
* requestHandler: router,
|
|
1331
|
-
* });
|
|
1332
|
-
* await crawler.run();
|
|
1333
|
-
* ```
|
|
1334
|
-
*/
|
|
1891
|
+
/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
|
|
1892
|
+
function hostnameOrUrl(url) {
|
|
1893
|
+
return URL.canParse(url) ? new URL(url).hostname : url;
|
|
1894
|
+
}
|
|
1335
1895
|
export function createBasicRouter(routes) {
|
|
1336
1896
|
return Router.create(routes);
|
|
1337
1897
|
}
|
|
1338
|
-
//# sourceMappingURL=basic-crawler.js.map
|