@crawlee/browser 4.0.0-beta.27 → 4.0.0-beta.270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/internals/browser-crawler.d.ts +244 -184
- package/internals/browser-crawler.js +339 -274
- package/internals/browser-launcher.d.ts +71 -27
- package/internals/browser-launcher.js +78 -26
- package/package.json +11 -12
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/browser-crawler.d.ts.map +0 -1
- package/internals/browser-crawler.js.map +0 -1
- package/internals/browser-launcher.d.ts.map +0 -1
- package/internals/browser-launcher.js.map +0 -1
|
@@ -1,8 +1,21 @@
|
|
|
1
|
-
import { BasicCrawler,
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import { tryCancel } from '@apify/timeout';
|
|
1
|
+
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
|
|
2
|
+
import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
|
|
3
|
+
import { sleep } from '@crawlee/utils';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
|
|
6
|
+
const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
|
|
7
|
+
const readContextField = (ctx, key) => ctx[key];
|
|
8
|
+
/**
|
|
9
|
+
* Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
|
|
10
|
+
* or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
|
|
11
|
+
* naming the raw millisecond value rather than the configured window.
|
|
12
|
+
*/
|
|
13
|
+
function isNavigationTimeoutError(error) {
|
|
14
|
+
return (error instanceof TimeoutError ||
|
|
15
|
+
error?.name === 'TimeoutError' ||
|
|
16
|
+
error?.constructor?.name === 'TimeoutError' ||
|
|
17
|
+
/timeout.*exceeded/i.test(error?.message ?? ''));
|
|
18
|
+
}
|
|
6
19
|
/**
|
|
7
20
|
* Provides a simple framework for parallel crawling of web pages
|
|
8
21
|
* using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
|
|
@@ -15,303 +28,421 @@ import { tryCancel } from '@apify/timeout';
|
|
|
15
28
|
* If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
|
|
16
29
|
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
17
30
|
*
|
|
18
|
-
* The source URLs are represented by the {@link Request} objects that are fed from the
|
|
19
|
-
*
|
|
20
|
-
* constructor
|
|
31
|
+
* The source URLs are represented by the {@link Request} objects that are fed from the
|
|
32
|
+
* {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
|
|
33
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
|
|
21
34
|
* the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
22
35
|
* or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
|
|
23
36
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
37
|
+
* To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
|
|
38
|
+
* combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
|
|
39
|
+
* and pass the result as `requestManager`.
|
|
40
|
+
*
|
|
41
|
+
* > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
|
|
42
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
27
43
|
*
|
|
28
44
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
29
45
|
*
|
|
30
46
|
* `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
|
|
31
47
|
* and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
|
|
32
48
|
*
|
|
33
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
39
|
-
* underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
|
|
49
|
+
* New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
|
|
50
|
+
* {@link ConcurrencySystem}.
|
|
51
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
52
|
+
* `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
53
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
40
54
|
*
|
|
41
55
|
* > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
|
|
42
56
|
*
|
|
43
57
|
* @category Crawlers
|
|
44
58
|
*/
|
|
45
59
|
export class BrowserCrawler extends BasicCrawler {
|
|
46
|
-
|
|
60
|
+
/** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
|
|
61
|
+
#browserPoolDep;
|
|
47
62
|
/**
|
|
48
|
-
* A reference to the underlying
|
|
63
|
+
* A reference to the underlying browser pool that manages the crawler's browsers. Typed as
|
|
64
|
+
* {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
|
|
49
65
|
*/
|
|
50
|
-
browserPool
|
|
51
|
-
|
|
66
|
+
get browserPool() {
|
|
67
|
+
return this.#browserPoolDep.value;
|
|
68
|
+
}
|
|
52
69
|
ignoreShadowRoots;
|
|
53
70
|
ignoreIframes;
|
|
54
|
-
navigationTimeoutMillis;
|
|
55
|
-
preNavigationHooks;
|
|
56
|
-
postNavigationHooks;
|
|
57
|
-
|
|
71
|
+
#navigationTimeoutMillis;
|
|
72
|
+
#preNavigationHooks;
|
|
73
|
+
#postNavigationHooks;
|
|
74
|
+
#saveResponseCookies;
|
|
75
|
+
/**
|
|
76
|
+
* @internal
|
|
77
|
+
*/
|
|
58
78
|
static optionsShape = {
|
|
59
79
|
...BasicCrawler.optionsShape,
|
|
60
|
-
navigationTimeoutSecs:
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
proxyConfiguration:
|
|
80
|
+
navigationTimeoutSecs: schemas.anyNumber
|
|
81
|
+
.refine((value) => value > 0, 'Expected a number greater than 0')
|
|
82
|
+
.default(60),
|
|
83
|
+
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
84
|
+
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
85
|
+
browserPool: validators.browserPool.optional(),
|
|
86
|
+
browserPoolBuilder: schemas.anyFunction.optional(),
|
|
87
|
+
remoteBrowser: schemas.anyObject.optional(),
|
|
88
|
+
saveResponseCookies: z.boolean().default(true),
|
|
89
|
+
proxyConfiguration: validators.proxyConfiguration.optional(),
|
|
90
|
+
ignoreIframes: z.boolean().default(false),
|
|
91
|
+
ignoreShadowRoots: z.boolean().default(false),
|
|
70
92
|
};
|
|
93
|
+
/** @internal */
|
|
94
|
+
static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
|
|
71
95
|
/**
|
|
72
96
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
97
|
+
* @internal
|
|
73
98
|
*/
|
|
74
|
-
constructor(options
|
|
75
|
-
|
|
76
|
-
|
|
99
|
+
constructor(options) {
|
|
100
|
+
const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
|
|
101
|
+
if (browserPool) {
|
|
102
|
+
assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
|
|
103
|
+
}
|
|
104
|
+
const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
|
|
77
105
|
super({
|
|
78
106
|
...basicCrawlerOptions,
|
|
79
|
-
contextPipelineBuilder: () =>
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
107
|
+
contextPipelineBuilder: () => {
|
|
108
|
+
// A single navigation window covers the pre-navigation hooks, the navigation, and the
|
|
109
|
+
// post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
|
|
110
|
+
// hook eats into the same window the navigation uses. The navigation itself is bounded by
|
|
111
|
+
// capping its `gotoOptions.timeout` to the remaining budget.
|
|
112
|
+
const windowGuard = (step) => skipGuard(async (ctx) => {
|
|
113
|
+
const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
|
|
114
|
+
if (remaining <= 0) {
|
|
115
|
+
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
116
|
+
}
|
|
117
|
+
return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
118
|
+
});
|
|
119
|
+
let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
|
|
120
|
+
for (const hook of this.#preNavigationHooks) {
|
|
121
|
+
pipeline = pipeline.compose(windowGuard(hook));
|
|
122
|
+
}
|
|
123
|
+
pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
|
|
124
|
+
for (const hook of this.#postNavigationHooks) {
|
|
125
|
+
pipeline = pipeline.compose(windowGuard(hook));
|
|
126
|
+
}
|
|
127
|
+
return pipeline
|
|
128
|
+
.compose(skipGuard(this.#finalizeNavigation.bind(this)))
|
|
129
|
+
.compose(this.#handleBlockedRequestByContent.bind(this))
|
|
130
|
+
.compose(this.#restoreRequestState.bind(this));
|
|
131
|
+
},
|
|
132
|
+
extendContext,
|
|
133
|
+
});
|
|
134
|
+
this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
|
|
135
|
+
// The public option hooks are extension-aware; internal storage uses the base context type
|
|
136
|
+
// (the pipeline composes hooks against the concrete context, which does not statically carry
|
|
137
|
+
// `ContextExtension`). The extension members are present at runtime regardless.
|
|
138
|
+
this.#preNavigationHooks = preNavigationHooks;
|
|
139
|
+
this.#postNavigationHooks = postNavigationHooks;
|
|
95
140
|
this.ignoreIframes = ignoreIframes;
|
|
96
141
|
this.ignoreShadowRoots = ignoreShadowRoots;
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
this.persistCookiesPerSession = false;
|
|
106
|
-
}
|
|
107
|
-
if (launchContext?.userAgent) {
|
|
108
|
-
if (browserPoolOptions.useFingerprints)
|
|
109
|
-
this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
|
|
110
|
-
browserPoolOptions.useFingerprints = false;
|
|
111
|
-
}
|
|
112
|
-
const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
|
|
113
|
-
this.browserPool = new BrowserPool({
|
|
114
|
-
...rest,
|
|
115
|
-
preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
|
|
116
|
-
postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
|
|
117
|
-
});
|
|
142
|
+
this.#saveResponseCookies = saveResponseCookies;
|
|
143
|
+
this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
|
|
144
|
+
}
|
|
145
|
+
/** @internal */
|
|
146
|
+
getNavigationTimeoutMillis() {
|
|
147
|
+
return this.#navigationTimeoutMillis;
|
|
118
148
|
}
|
|
119
149
|
buildContextPipeline() {
|
|
120
|
-
return ContextPipeline.create().compose(
|
|
121
|
-
action: this.preparePage.bind(this),
|
|
122
|
-
cleanup: async (context) => {
|
|
123
|
-
context.registerDeferredCleanup(async () => {
|
|
124
|
-
await context.page
|
|
125
|
-
.close()
|
|
126
|
-
.catch((error) => this.log.debug('Error while closing page', { error }));
|
|
127
|
-
});
|
|
128
|
-
},
|
|
129
|
-
});
|
|
150
|
+
return ContextPipeline.create().compose(this.#preparePage.bind(this));
|
|
130
151
|
}
|
|
131
|
-
async containsSelectors(page, selectors) {
|
|
132
|
-
const
|
|
152
|
+
async #containsSelectors(page, selectors) {
|
|
153
|
+
const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
|
|
154
|
+
// a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
|
|
155
|
+
const elements = await query().catch(async () => {
|
|
156
|
+
await sleep(1000);
|
|
157
|
+
return query();
|
|
158
|
+
});
|
|
159
|
+
const foundSelectors = elements
|
|
133
160
|
.map((x, i) => [x, selectors[i]])
|
|
134
161
|
.filter(([x]) => x !== null)
|
|
135
162
|
.map(([, selector]) => selector);
|
|
136
163
|
return foundSelectors.length > 0 ? foundSelectors : null;
|
|
137
164
|
}
|
|
138
|
-
async
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
: DEFAULT_BLOCKED_STATUS_CODES;
|
|
146
|
-
// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
|
|
147
|
-
if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
|
|
148
|
-
await sleep(5000);
|
|
149
|
-
// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
|
|
150
|
-
const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
151
|
-
if (!foundSelectors)
|
|
152
|
-
return false;
|
|
153
|
-
return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
|
|
165
|
+
async #detectSelfResolvingChallenge({ page, response, }) {
|
|
166
|
+
const statusCode = response?.status();
|
|
167
|
+
if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
|
|
168
|
+
return 'Cloudflare';
|
|
169
|
+
}
|
|
170
|
+
if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
|
|
171
|
+
return 'AWS WAF';
|
|
154
172
|
}
|
|
155
|
-
|
|
156
|
-
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
async #isRequestBlocked(crawlingContext) {
|
|
176
|
+
const { page, response } = crawlingContext;
|
|
177
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
178
|
+
const statusCode = response?.status() ?? 0;
|
|
157
179
|
if (foundSelectors)
|
|
158
180
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
159
|
-
if (
|
|
160
|
-
return `Received blocked status code: ${
|
|
181
|
+
if (this.blockedStatusCodes.has(statusCode))
|
|
182
|
+
return `Received blocked status code: ${statusCode}`;
|
|
161
183
|
return false;
|
|
162
184
|
}
|
|
163
|
-
async preparePage(crawlingContext) {
|
|
164
|
-
const
|
|
185
|
+
async #preparePage(crawlingContext, onCleanup) {
|
|
186
|
+
const page = await this.browserPool.newPage({
|
|
165
187
|
id: crawlingContext.id,
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if (crawlingContext.session?.proxyInfo) {
|
|
169
|
-
const proxyInfo = crawlingContext.session.proxyInfo;
|
|
170
|
-
crawlingContext.proxyInfo = proxyInfo;
|
|
171
|
-
newPageOptions.proxyUrl = proxyInfo?.url;
|
|
172
|
-
newPageOptions.proxyTier = proxyInfo?.proxyTier;
|
|
173
|
-
if (proxyInfo?.ignoreTlsErrors) {
|
|
174
|
-
/**
|
|
175
|
-
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
176
|
-
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
177
|
-
*/
|
|
178
|
-
newPageOptions.pageOptions = {
|
|
179
|
-
ignoreHTTPSErrors: true,
|
|
180
|
-
acceptInsecureCerts: true,
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
const page = (await this.browserPool.newPage(newPageOptions));
|
|
188
|
+
session: crawlingContext.session,
|
|
189
|
+
});
|
|
185
190
|
tryCancel();
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
+
onCleanup((failure) => {
|
|
192
|
+
crawlingContext.registerDeferredCleanup(async () => {
|
|
193
|
+
const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
|
|
194
|
+
? failure.cause
|
|
195
|
+
: failure;
|
|
196
|
+
// Only tells the pool what to do with the page's browser state - the request's failure, if any, has
|
|
197
|
+
// already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
|
|
198
|
+
// unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
|
|
199
|
+
// count, so the block check must come first.
|
|
200
|
+
const { session } = crawlingContext;
|
|
201
|
+
const closeReason = cause instanceof SessionError
|
|
202
|
+
? cause
|
|
203
|
+
: session.isBlocked()
|
|
204
|
+
? new SessionError()
|
|
205
|
+
: session.isUsable()
|
|
206
|
+
? undefined
|
|
207
|
+
: new SessionRetiredError();
|
|
208
|
+
await this.browserPool
|
|
209
|
+
.closePage(page, { error: closeReason })
|
|
210
|
+
.catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
const addRequests = crawlingContext.addRequests;
|
|
214
|
+
const extractLinks = async (options) => {
|
|
215
|
+
return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
|
|
216
|
+
};
|
|
191
217
|
return {
|
|
192
218
|
page,
|
|
193
219
|
get response() {
|
|
194
220
|
throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
|
|
195
221
|
},
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
requestQueue: await this.getRequestQueue(),
|
|
204
|
-
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
|
|
205
|
-
onSkippedRequest: this.handleSkippedRequest,
|
|
206
|
-
originalRequestUrl: crawlingContext.request.url,
|
|
222
|
+
get gotoOptions() {
|
|
223
|
+
throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
|
|
224
|
+
},
|
|
225
|
+
extractLinks,
|
|
226
|
+
enqueueLinks: async (options = {}) => {
|
|
227
|
+
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
228
|
+
enqueueStrategy: options.strategy,
|
|
207
229
|
finalRequestUrl: crawlingContext.request.loadedUrl,
|
|
208
|
-
|
|
209
|
-
|
|
230
|
+
originalRequestUrl: crawlingContext.request.url,
|
|
231
|
+
userProvidedBaseUrl: options.baseUrl,
|
|
232
|
+
});
|
|
233
|
+
const urls = await extractLinks(options);
|
|
234
|
+
return addRequests(urls, {
|
|
235
|
+
...options,
|
|
236
|
+
baseUrl,
|
|
237
|
+
strategy: options.strategy ?? EnqueueStrategy.SameHostname,
|
|
238
|
+
});
|
|
210
239
|
},
|
|
211
240
|
};
|
|
212
241
|
}
|
|
213
|
-
async
|
|
242
|
+
async #prepareNavigation(crawlingContext) {
|
|
214
243
|
if (crawlingContext.request.skipNavigation) {
|
|
215
244
|
return {
|
|
216
|
-
request:
|
|
217
|
-
get(target, propertyName, receiver) {
|
|
218
|
-
if (propertyName === 'loadedUrl') {
|
|
219
|
-
throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
220
|
-
}
|
|
221
|
-
return Reflect.get(target, propertyName, receiver);
|
|
222
|
-
},
|
|
223
|
-
}),
|
|
245
|
+
request: skippedNavigationRequest(crawlingContext.request),
|
|
224
246
|
get response() {
|
|
225
|
-
throw new
|
|
247
|
+
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
226
248
|
},
|
|
227
249
|
};
|
|
228
250
|
}
|
|
229
|
-
const gotoOptions = { timeout: this.navigationTimeoutMillis };
|
|
230
|
-
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
|
|
231
251
|
crawlingContext.request.state = RequestState.BEFORE_NAV;
|
|
232
|
-
|
|
252
|
+
return {
|
|
253
|
+
// Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
|
|
254
|
+
// to the remaining shared window unless a hook overrode it (see there).
|
|
255
|
+
gotoOptions: { timeout: this.#navigationTimeoutMillis },
|
|
256
|
+
[COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
260
|
+
async navigate(crawlingContext) {
|
|
233
261
|
tryCancel();
|
|
234
|
-
const
|
|
235
|
-
|
|
262
|
+
const gotoOptions = crawlingContext.gotoOptions;
|
|
263
|
+
const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
|
|
264
|
+
if (remaining <= 0) {
|
|
265
|
+
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
266
|
+
}
|
|
267
|
+
// If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
|
|
268
|
+
// shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
|
|
269
|
+
// that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
|
|
270
|
+
// aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
|
|
271
|
+
const gotoTimeout = gotoOptions;
|
|
272
|
+
if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
|
|
273
|
+
gotoTimeout.timeout = remaining;
|
|
274
|
+
}
|
|
275
|
+
const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
|
|
276
|
+
const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
|
|
277
|
+
await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
|
|
236
278
|
let response;
|
|
237
279
|
try {
|
|
238
|
-
response = (await this.
|
|
280
|
+
response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
|
|
239
281
|
}
|
|
240
282
|
catch (error) {
|
|
241
|
-
await this
|
|
283
|
+
await this.#handleNavigationTimeout(crawlingContext, error);
|
|
242
284
|
crawlingContext.request.state = RequestState.ERROR;
|
|
243
|
-
this
|
|
285
|
+
this.#throwIfProxyError(error);
|
|
244
286
|
throw error;
|
|
245
287
|
}
|
|
246
288
|
tryCancel();
|
|
247
289
|
crawlingContext.request.state = RequestState.AFTER_NAV;
|
|
248
|
-
|
|
249
|
-
|
|
290
|
+
return { response };
|
|
291
|
+
}
|
|
292
|
+
async #finalizeNavigation(crawlingContext) {
|
|
293
|
+
tryCancel();
|
|
294
|
+
let response;
|
|
295
|
+
try {
|
|
296
|
+
response = crawlingContext.response;
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
// `preparePage` installs a throwing getter for `response`; reaching this branch means
|
|
300
|
+
// navigation produced no response and no hook overrode it. Treat as undefined.
|
|
301
|
+
}
|
|
302
|
+
await this.#processResponse(response, crawlingContext);
|
|
250
303
|
tryCancel();
|
|
251
|
-
//
|
|
252
|
-
//
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
304
|
+
// Persist cookies from the navigation response before the user handler runs.
|
|
305
|
+
// Cookies set during `requestHandler` are saved again afterwards.
|
|
306
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
307
|
+
return { request: crawlingContext.request };
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Copies cookies from the live browser page into the session cookie jar.
|
|
311
|
+
*/
|
|
312
|
+
async #persistCookiesFromPage(crawlingContext) {
|
|
313
|
+
if (!this.#saveResponseCookies || !crawlingContext.session) {
|
|
314
|
+
return;
|
|
257
315
|
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
316
|
+
const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
|
|
317
|
+
tryCancel();
|
|
318
|
+
// Prefer the live page URL — the handler may have navigated after the initial load.
|
|
319
|
+
const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
|
|
320
|
+
for (const cookie of cookies) {
|
|
321
|
+
try {
|
|
322
|
+
await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
|
|
323
|
+
ignoreError: false,
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
catch (e) {
|
|
327
|
+
this.log.debug(`Could not set cookie: ${e.message}`);
|
|
328
|
+
}
|
|
263
329
|
}
|
|
264
|
-
return {
|
|
265
|
-
request: crawlingContext.request,
|
|
266
|
-
};
|
|
267
330
|
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
331
|
+
/**
|
|
332
|
+
* Runs the user request handler, then re-reads browser cookies so login flows /
|
|
333
|
+
* `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
|
|
334
|
+
* @internal
|
|
335
|
+
*/
|
|
336
|
+
async runRequestHandler(crawlingContext) {
|
|
337
|
+
try {
|
|
338
|
+
await super.runRequestHandler(crawlingContext);
|
|
273
339
|
}
|
|
340
|
+
finally {
|
|
341
|
+
if (!crawlingContext.request.skipNavigation) {
|
|
342
|
+
try {
|
|
343
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// Page may already be closed on some failure paths; ignore.
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
352
|
+
if (!this.retryOnBlocked)
|
|
353
|
+
return {};
|
|
354
|
+
const page = crawlingContext.page;
|
|
355
|
+
// A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
|
|
356
|
+
let navigationResponse;
|
|
357
|
+
const onResponse = (response) => {
|
|
358
|
+
if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
|
|
359
|
+
navigationResponse = response;
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
page.on('response', onResponse);
|
|
363
|
+
try {
|
|
364
|
+
// Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
|
|
365
|
+
// Wait 5 seconds for the JS challenge to load / resolve before judging the page.
|
|
366
|
+
const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
|
|
367
|
+
if (challenge) {
|
|
368
|
+
await sleep(5000);
|
|
369
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
370
|
+
if (foundSelectors) {
|
|
371
|
+
throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
|
|
372
|
+
}
|
|
373
|
+
return navigationResponse ? { response: navigationResponse } : {};
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
page.off('response', onResponse);
|
|
378
|
+
}
|
|
379
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
380
|
+
if (error)
|
|
381
|
+
throw new SessionError(error);
|
|
274
382
|
return {};
|
|
275
383
|
}
|
|
276
|
-
async restoreRequestState(crawlingContext) {
|
|
384
|
+
async #restoreRequestState(crawlingContext) {
|
|
277
385
|
crawlingContext.request.state = RequestState.REQUEST_HANDLER;
|
|
278
386
|
return {};
|
|
279
387
|
}
|
|
280
|
-
async
|
|
281
|
-
const sessionCookie = session
|
|
388
|
+
async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
|
|
389
|
+
const sessionCookie = session
|
|
390
|
+
? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
|
|
391
|
+
: [];
|
|
282
392
|
const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
|
|
283
393
|
const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
|
|
284
|
-
|
|
394
|
+
const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
|
|
285
395
|
.filter((c) => typeof c !== 'undefined' && c !== null)
|
|
286
|
-
.map((c) => ({ ...c, url: c.domain ? undefined : request.url }))
|
|
396
|
+
.map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
|
|
397
|
+
await this.browserPool.injectPageState(page, { cookies });
|
|
287
398
|
}
|
|
288
399
|
/**
|
|
289
|
-
* Marks session bad in
|
|
400
|
+
* Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
|
|
290
401
|
*/
|
|
291
|
-
async
|
|
292
|
-
const { session } = crawlingContext;
|
|
293
|
-
|
|
294
|
-
|
|
402
|
+
async #handleNavigationTimeout(crawlingContext, error) {
|
|
403
|
+
const { session, page } = crawlingContext;
|
|
404
|
+
// Fire-and-forget: no user code will run on this page after a failed navigation.
|
|
405
|
+
// Swallow rejections: the page may already be detached.
|
|
406
|
+
void page.evaluate(() => window.stop()).catch(() => { });
|
|
407
|
+
if (isNavigationTimeoutError(error)) {
|
|
408
|
+
session?.markBad();
|
|
409
|
+
// The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
|
|
410
|
+
// hooks have run), so it names that value in its own error; report the configured window instead.
|
|
411
|
+
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
295
412
|
}
|
|
296
|
-
await crawlingContext.page.close();
|
|
297
413
|
}
|
|
298
414
|
/**
|
|
299
415
|
* Transforms proxy-related errors to `SessionError`.
|
|
300
416
|
*/
|
|
301
|
-
|
|
417
|
+
#throwIfProxyError(error) {
|
|
302
418
|
if (this.isProxyError(error)) {
|
|
303
|
-
throw new SessionError(this.
|
|
419
|
+
throw new SessionError(this.getMessageFromError(error));
|
|
304
420
|
}
|
|
305
421
|
}
|
|
306
|
-
async processResponse(response, crawlingContext) {
|
|
422
|
+
async #processResponse(response, crawlingContext) {
|
|
307
423
|
const { session, request, page } = crawlingContext;
|
|
308
424
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
309
425
|
const status = response.status();
|
|
310
|
-
this.
|
|
426
|
+
this.statistics.registerStatusCode(status);
|
|
427
|
+
// Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
|
|
428
|
+
// rate limit the domain should back off from.
|
|
429
|
+
if (status === 429) {
|
|
430
|
+
// Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
|
|
431
|
+
const retryAfter = response.headers?.()['retry-after'];
|
|
432
|
+
if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
|
|
433
|
+
throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
if (this.isErrorStatusCode(status)) {
|
|
437
|
+
if (this.additionalHttpErrorStatusCodes.has(status)) {
|
|
438
|
+
throw new Error(`${status} - Error status code was set by user.`);
|
|
439
|
+
}
|
|
440
|
+
throw new Error(`${status} - Internal Server Error`);
|
|
441
|
+
}
|
|
311
442
|
}
|
|
312
443
|
if (this.sessionPool && response && session) {
|
|
313
444
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
314
|
-
this.
|
|
445
|
+
this.throwOnBlockedRequest(response.status());
|
|
315
446
|
}
|
|
316
447
|
else {
|
|
317
448
|
this.log.debug('Got a malformed Browser response.', { request, response });
|
|
@@ -319,83 +450,18 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
319
450
|
}
|
|
320
451
|
request.loadedUrl = await page.url();
|
|
321
452
|
}
|
|
322
|
-
async _extendLaunchContext(_pageId, launchContext) {
|
|
323
|
-
const launchContextExtends = {};
|
|
324
|
-
if (this.sessionPool) {
|
|
325
|
-
launchContextExtends.session = await this.sessionPool.newSession({
|
|
326
|
-
proxyInfo: await this.proxyConfiguration?.newProxyInfo({
|
|
327
|
-
// cannot pass a request here, since session is created on browser launch
|
|
328
|
-
}),
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
if (!launchContext.proxyUrl && launchContextExtends.session?.proxyInfo) {
|
|
332
|
-
const proxyInfo = launchContextExtends.session.proxyInfo;
|
|
333
|
-
launchContext.proxyUrl = proxyInfo?.url;
|
|
334
|
-
launchContextExtends.proxyInfo = proxyInfo;
|
|
335
|
-
// Disable SSL verification for MITM proxies
|
|
336
|
-
if (proxyInfo?.ignoreTlsErrors) {
|
|
337
|
-
/**
|
|
338
|
-
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
339
|
-
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
340
|
-
*/
|
|
341
|
-
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
342
|
-
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
launchContext.extend(launchContextExtends);
|
|
346
|
-
}
|
|
347
|
-
_maybeAddSessionRetiredListener(_pageId, browserController) {
|
|
348
|
-
if (this.sessionPool) {
|
|
349
|
-
const listener = (session) => {
|
|
350
|
-
const { launchContext } = browserController;
|
|
351
|
-
if (session.id === launchContext.session.id) {
|
|
352
|
-
this.browserPool.retireBrowserController(browserController);
|
|
353
|
-
}
|
|
354
|
-
};
|
|
355
|
-
this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
|
|
356
|
-
browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
|
|
357
|
-
return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
|
|
358
|
-
});
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
453
|
/**
|
|
362
|
-
*
|
|
363
|
-
*
|
|
454
|
+
* Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
|
|
455
|
+
* crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
|
|
364
456
|
*/
|
|
365
457
|
async teardown() {
|
|
366
|
-
await this.
|
|
458
|
+
await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
|
|
367
459
|
await super.teardown();
|
|
368
460
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
return !!options.enqueueLinks;
|
|
373
|
-
}
|
|
374
|
-
/** @internal */
|
|
375
|
-
export async function browserCrawlerEnqueueLinks(options) {
|
|
376
|
-
const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
|
|
377
|
-
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
378
|
-
enqueueStrategy: enqueueLinksOptions?.strategy,
|
|
379
|
-
finalRequestUrl,
|
|
380
|
-
originalRequestUrl,
|
|
381
|
-
userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
|
|
382
|
-
});
|
|
383
|
-
const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
|
|
384
|
-
if (containsEnqueueLinks(options)) {
|
|
385
|
-
return options.enqueueLinks({
|
|
386
|
-
urls,
|
|
387
|
-
baseUrl,
|
|
388
|
-
...enqueueLinksOptions,
|
|
389
|
-
});
|
|
461
|
+
async destroy() {
|
|
462
|
+
await super.destroy();
|
|
463
|
+
await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
|
|
390
464
|
}
|
|
391
|
-
return enqueueLinks({
|
|
392
|
-
requestQueue: options.requestQueue,
|
|
393
|
-
robotsTxtFile: options.robotsTxtFile,
|
|
394
|
-
onSkippedRequest: options.onSkippedRequest,
|
|
395
|
-
urls,
|
|
396
|
-
baseUrl,
|
|
397
|
-
...enqueueLinksOptions,
|
|
398
|
-
});
|
|
399
465
|
}
|
|
400
466
|
/**
|
|
401
467
|
* Extracts URLs from a given page.
|
|
@@ -422,4 +488,3 @@ page, selector, baseUrl) {
|
|
|
422
488
|
})
|
|
423
489
|
.filter((href) => !!href);
|
|
424
490
|
}
|
|
425
|
-
//# sourceMappingURL=browser-crawler.js.map
|