@crawlee/browser 4.0.0-beta.29 → 4.0.0-beta.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,21 @@
1
- import { BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
- import { BrowserPool } from '@crawlee/browser-pool';
3
- import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
4
- import ow from 'ow';
5
- import { tryCancel } from '@apify/timeout';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
+ import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
+ import { sleep } from '@crawlee/utils';
4
+ import { z } from 'zod';
5
+ import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
6
+ const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
7
+ const readContextField = (ctx, key) => ctx[key];
8
+ /**
9
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
10
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
11
+ * naming the raw millisecond value rather than the configured window.
12
+ */
13
+ function isNavigationTimeoutError(error) {
14
+ return (error instanceof TimeoutError ||
15
+ error?.name === 'TimeoutError' ||
16
+ error?.constructor?.name === 'TimeoutError' ||
17
+ /timeout.*exceeded/i.test(error?.message ?? ''));
18
+ }
6
19
  /**
7
20
  * Provides a simple framework for parallel crawling of web pages
8
21
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,301 +28,421 @@ import { tryCancel } from '@apify/timeout';
15
28
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
16
29
  * which downloads the pages using raw HTTP requests and is about 10x faster.
17
30
  *
18
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
19
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
20
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
31
+ * The source URLs are represented by the {@link Request} objects that are fed from the
32
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
33
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
21
34
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
22
35
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
23
36
  *
24
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
25
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
26
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
37
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
38
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
39
+ * and pass the result as `requestManager`.
40
+ *
41
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
42
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
27
43
  *
28
44
  * The crawler finishes when there are no more {@link Request} objects to crawl.
29
45
  *
30
46
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
31
47
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
32
48
  *
33
- * New pages are only opened when there is enough free CPU and memory available,
34
- * using the functionality provided by the {@link AutoscaledPool} class.
35
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
36
- * parameter of the `BrowserCrawler` constructor.
37
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
38
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
39
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
49
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
50
+ * {@link ConcurrencySystem}.
51
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
52
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
53
+ * {@link ConcurrencySystem|`concurrencySystem`}.
40
54
  *
41
55
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
42
56
  *
43
57
  * @category Crawlers
44
58
  */
45
59
  export class BrowserCrawler extends BasicCrawler {
60
+ /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
61
+ #browserPoolDep;
46
62
  /**
47
- * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
63
+ * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
64
+ * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
48
65
  */
49
- browserPool;
50
- launchContext;
66
+ get browserPool() {
67
+ return this.#browserPoolDep.value;
68
+ }
51
69
  ignoreShadowRoots;
52
70
  ignoreIframes;
53
- navigationTimeoutMillis;
54
- preNavigationHooks;
55
- postNavigationHooks;
56
- persistCookiesPerSession;
71
+ #navigationTimeoutMillis;
72
+ #preNavigationHooks;
73
+ #postNavigationHooks;
74
+ #saveResponseCookies;
75
+ /**
76
+ * @internal
77
+ */
57
78
  static optionsShape = {
58
79
  ...BasicCrawler.optionsShape,
59
- navigationTimeoutSecs: ow.optional.number.greaterThan(0),
60
- preNavigationHooks: ow.optional.array,
61
- postNavigationHooks: ow.optional.array,
62
- launchContext: ow.optional.object,
63
- headless: ow.optional.any(ow.boolean, ow.string),
64
- browserPoolOptions: ow.object,
65
- sessionPoolOptions: ow.optional.object,
66
- persistCookiesPerSession: ow.optional.boolean,
67
- useSessionPool: ow.optional.boolean,
68
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
80
+ navigationTimeoutSecs: schemas.anyNumber
81
+ .refine((value) => value > 0, 'Expected a number greater than 0')
82
+ .default(60),
83
+ preNavigationHooks: schemas.anyArray.default(() => []),
84
+ postNavigationHooks: schemas.anyArray.default(() => []),
85
+ browserPool: validators.browserPool.optional(),
86
+ browserPoolBuilder: schemas.anyFunction.optional(),
87
+ remoteBrowser: schemas.anyObject.optional(),
88
+ saveResponseCookies: z.boolean().default(true),
89
+ proxyConfiguration: validators.proxyConfiguration.optional(),
90
+ ignoreIframes: z.boolean().default(false),
91
+ ignoreShadowRoots: z.boolean().default(false),
69
92
  };
93
+ /** @internal */
94
+ static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
70
95
  /**
71
96
  * All `BrowserCrawler` parameters are passed via an options object.
97
+ * @internal
72
98
  */
73
99
  constructor(options) {
74
- ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
75
- const { navigationTimeoutSecs = 60, persistCookiesPerSession, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, proxyConfiguration, ...basicCrawlerOptions } = options;
100
+ const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
101
+ if (browserPool) {
102
+ assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
103
+ }
104
+ const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
76
105
  super({
77
106
  ...basicCrawlerOptions,
78
- contextPipelineBuilder: () => contextPipelineBuilder()
79
- .compose({ action: this.performNavigation.bind(this) })
80
- .compose({ action: this.handleBlockedRequestByContent.bind(this) })
81
- .compose({ action: this.restoreRequestState.bind(this) }),
82
- extendContext: extendContext,
107
+ contextPipelineBuilder: () => {
108
+ // A single navigation window covers the pre-navigation hooks, the navigation, and the
109
+ // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
110
+ // hook eats into the same window the navigation uses. The navigation itself is bounded by
111
+ // capping its `gotoOptions.timeout` to the remaining budget.
112
+ const windowGuard = (step) => skipGuard(async (ctx) => {
113
+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
114
+ if (remaining <= 0) {
115
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
116
+ }
117
+ return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
118
+ });
119
+ let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
120
+ for (const hook of this.#preNavigationHooks) {
121
+ pipeline = pipeline.compose(windowGuard(hook));
122
+ }
123
+ pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
124
+ for (const hook of this.#postNavigationHooks) {
125
+ pipeline = pipeline.compose(windowGuard(hook));
126
+ }
127
+ return pipeline
128
+ .compose(skipGuard(this.#finalizeNavigation.bind(this)))
129
+ .compose(this.#handleBlockedRequestByContent.bind(this))
130
+ .compose(this.#restoreRequestState.bind(this));
131
+ },
132
+ extendContext,
83
133
  });
84
- // Cookies should be persisted per session only if session pool is used
85
- if (!this.useSessionPool && persistCookiesPerSession) {
86
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
87
- }
88
- this.launchContext = launchContext;
89
- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
90
- this.proxyConfiguration = proxyConfiguration;
91
- this.preNavigationHooks = preNavigationHooks;
92
- this.postNavigationHooks = postNavigationHooks;
134
+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
135
+ // The public option hooks are extension-aware; internal storage uses the base context type
136
+ // (the pipeline composes hooks against the concrete context, which does not statically carry
137
+ // `ContextExtension`). The extension members are present at runtime regardless.
138
+ this.#preNavigationHooks = preNavigationHooks;
139
+ this.#postNavigationHooks = postNavigationHooks;
93
140
  this.ignoreIframes = ignoreIframes;
94
141
  this.ignoreShadowRoots = ignoreShadowRoots;
95
- if (headless != null) {
96
- this.launchContext.launchOptions ??= {};
97
- this.launchContext.launchOptions.headless = headless;
98
- }
99
- if (this.useSessionPool) {
100
- this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
101
- }
102
- else {
103
- this.persistCookiesPerSession = false;
104
- }
105
- if (launchContext?.userAgent) {
106
- if (browserPoolOptions.useFingerprints)
107
- this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
108
- browserPoolOptions.useFingerprints = false;
109
- }
110
- const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
111
- this.browserPool = new BrowserPool({
112
- ...rest,
113
- preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
114
- postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
115
- });
142
+ this.#saveResponseCookies = saveResponseCookies;
143
+ this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
144
+ }
145
+ /** @internal */
146
+ getNavigationTimeoutMillis() {
147
+ return this.#navigationTimeoutMillis;
116
148
  }
117
149
  buildContextPipeline() {
118
- return ContextPipeline.create().compose({
119
- action: this.preparePage.bind(this),
120
- cleanup: async (context) => {
121
- context.registerDeferredCleanup(async () => {
122
- await context.page
123
- .close()
124
- .catch((error) => this.log.debug('Error while closing page', { error }));
125
- });
126
- },
127
- });
150
+ return ContextPipeline.create().compose(this.#preparePage.bind(this));
128
151
  }
129
- async containsSelectors(page, selectors) {
130
- const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
152
+ async #containsSelectors(page, selectors) {
153
+ const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
154
+ // a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
155
+ const elements = await query().catch(async () => {
156
+ await sleep(1000);
157
+ return query();
158
+ });
159
+ const foundSelectors = elements
131
160
  .map((x, i) => [x, selectors[i]])
132
161
  .filter(([x]) => x !== null)
133
162
  .map(([, selector]) => selector);
134
163
  return foundSelectors.length > 0 ? foundSelectors : null;
135
164
  }
136
- async isRequestBlocked(crawlingContext) {
137
- const { page, response } = crawlingContext;
138
- const blockedStatusCodes =
139
- // eslint-disable-next-line dot-notation
140
- (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
141
- ? // eslint-disable-next-line dot-notation
142
- this.sessionPool['blockedStatusCodes']
143
- : DEFAULT_BLOCKED_STATUS_CODES;
144
- // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
145
- if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
146
- await sleep(5000);
147
- // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
148
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
149
- if (!foundSelectors)
150
- return false;
151
- return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
165
+ async #detectSelfResolvingChallenge({ page, response, }) {
166
+ const statusCode = response?.status();
167
+ if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
168
+ return 'Cloudflare';
152
169
  }
153
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
154
- const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
170
+ if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
171
+ return 'AWS WAF';
172
+ }
173
+ return null;
174
+ }
175
+ async #isRequestBlocked(crawlingContext) {
176
+ const { page, response } = crawlingContext;
177
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
178
+ const statusCode = response?.status() ?? 0;
155
179
  if (foundSelectors)
156
180
  return `Found selectors: ${foundSelectors.join(', ')}`;
157
- if (blockedStatusCode)
158
- return `Received blocked status code: ${blockedStatusCode}`;
181
+ if (this.blockedStatusCodes.has(statusCode))
182
+ return `Received blocked status code: ${statusCode}`;
159
183
  return false;
160
184
  }
161
- async preparePage(crawlingContext) {
162
- const newPageOptions = {
185
+ async #preparePage(crawlingContext, onCleanup) {
186
+ const page = await this.browserPool.newPage({
163
187
  id: crawlingContext.id,
164
- };
165
- const useIncognitoPages = this.launchContext?.useIncognitoPages;
166
- if (crawlingContext.session?.proxyInfo) {
167
- const proxyInfo = crawlingContext.session.proxyInfo;
168
- crawlingContext.proxyInfo = proxyInfo;
169
- newPageOptions.proxyUrl = proxyInfo?.url;
170
- newPageOptions.proxyTier = proxyInfo?.proxyTier;
171
- if (proxyInfo?.ignoreTlsErrors) {
172
- /**
173
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
174
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
175
- */
176
- newPageOptions.pageOptions = {
177
- ignoreHTTPSErrors: true,
178
- acceptInsecureCerts: true,
179
- };
180
- }
181
- }
182
- const page = (await this.browserPool.newPage(newPageOptions));
188
+ session: crawlingContext.session,
189
+ });
183
190
  tryCancel();
184
- const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
185
- const contextEnqueueLinks = crawlingContext.enqueueLinks;
186
- const session = useIncognitoPages
187
- ? crawlingContext.session
188
- : browserControllerInstance.launchContext.session;
191
+ onCleanup((failure) => {
192
+ crawlingContext.registerDeferredCleanup(async () => {
193
+ const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
194
+ ? failure.cause
195
+ : failure;
196
+ // Only tells the pool what to do with the page's browser state - the request's failure, if any, has
197
+ // already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
198
+ // unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
199
+ // count, so the block check must come first.
200
+ const { session } = crawlingContext;
201
+ const closeReason = cause instanceof SessionError
202
+ ? cause
203
+ : session.isBlocked()
204
+ ? new SessionError()
205
+ : session.isUsable()
206
+ ? undefined
207
+ : new SessionRetiredError();
208
+ await this.browserPool
209
+ .closePage(page, { error: closeReason })
210
+ .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
211
+ });
212
+ });
213
+ const addRequests = crawlingContext.addRequests;
214
+ const extractLinks = async (options) => {
215
+ return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
216
+ };
189
217
  return {
190
218
  page,
191
219
  get response() {
192
220
  throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
193
221
  },
194
- browserController: browserControllerInstance,
195
- session,
196
- proxyInfo: session?.proxyInfo,
197
- enqueueLinks: async (enqueueOptions = {}) => {
198
- return (await browserCrawlerEnqueueLinks({
199
- options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
200
- page,
201
- requestQueue: await this.getRequestQueue(),
202
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
203
- onSkippedRequest: this.handleSkippedRequest,
204
- originalRequestUrl: crawlingContext.request.url,
222
+ get gotoOptions() {
223
+ throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
224
+ },
225
+ extractLinks,
226
+ enqueueLinks: async (options = {}) => {
227
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
228
+ enqueueStrategy: options.strategy,
205
229
  finalRequestUrl: crawlingContext.request.loadedUrl,
206
- enqueueLinks: contextEnqueueLinks,
207
- })); // TODO make this type safe
230
+ originalRequestUrl: crawlingContext.request.url,
231
+ userProvidedBaseUrl: options.baseUrl,
232
+ });
233
+ const urls = await extractLinks(options);
234
+ return addRequests(urls, {
235
+ ...options,
236
+ baseUrl,
237
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
238
+ });
208
239
  },
209
240
  };
210
241
  }
211
- async performNavigation(crawlingContext) {
242
+ async #prepareNavigation(crawlingContext) {
212
243
  if (crawlingContext.request.skipNavigation) {
213
244
  return {
214
- request: new Proxy(crawlingContext.request, {
215
- get(target, propertyName, receiver) {
216
- if (propertyName === 'loadedUrl') {
217
- throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
218
- }
219
- return Reflect.get(target, propertyName, receiver);
220
- },
221
- }),
245
+ request: skippedNavigationRequest(crawlingContext.request),
222
246
  get response() {
223
- throw new Error('The `response` property is not available - `skipNavigation` was used');
247
+ throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
224
248
  },
225
249
  };
226
250
  }
227
- const gotoOptions = { timeout: this.navigationTimeoutMillis };
228
- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
229
251
  crawlingContext.request.state = RequestState.BEFORE_NAV;
230
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
252
+ return {
253
+ // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
254
+ // to the remaining shared window unless a hook overrode it (see there).
255
+ gotoOptions: { timeout: this.#navigationTimeoutMillis },
256
+ [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
257
+ };
258
+ }
259
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
260
+ async navigate(crawlingContext) {
231
261
  tryCancel();
232
- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
233
- await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
262
+ const gotoOptions = crawlingContext.gotoOptions;
263
+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
264
+ if (remaining <= 0) {
265
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
266
+ }
267
+ // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
268
+ // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
269
+ // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
270
+ // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
271
+ const gotoTimeout = gotoOptions;
272
+ if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
273
+ gotoTimeout.timeout = remaining;
274
+ }
275
+ const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
276
+ const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
277
+ await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
234
278
  let response;
235
279
  try {
236
- response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
280
+ response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
237
281
  }
238
282
  catch (error) {
239
- await this._handleNavigationTimeout(crawlingContext, error);
283
+ await this.#handleNavigationTimeout(crawlingContext, error);
240
284
  crawlingContext.request.state = RequestState.ERROR;
241
- this._throwIfProxyError(error);
285
+ this.#throwIfProxyError(error);
242
286
  throw error;
243
287
  }
244
288
  tryCancel();
245
289
  crawlingContext.request.state = RequestState.AFTER_NAV;
246
- await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
247
- await this.processResponse(response, crawlingContext);
290
+ return { response };
291
+ }
292
+ async #finalizeNavigation(crawlingContext) {
248
293
  tryCancel();
249
- // save cookies
250
- // TODO: Should we save the cookies also after/only the handle page?
251
- if (this.persistCookiesPerSession) {
252
- const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
253
- tryCancel();
254
- crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl);
294
+ let response;
295
+ try {
296
+ response = crawlingContext.response;
255
297
  }
256
- if (response !== undefined) {
257
- return {
258
- request: crawlingContext.request,
259
- response,
260
- };
298
+ catch {
299
+ // `preparePage` installs a throwing getter for `response`; reaching this branch means
300
+ // navigation produced no response and no hook overrode it. Treat as undefined.
301
+ }
302
+ await this.#processResponse(response, crawlingContext);
303
+ tryCancel();
304
+ // Persist cookies from the navigation response before the user handler runs.
305
+ // Cookies set during `requestHandler` are saved again afterwards.
306
+ await this.#persistCookiesFromPage(crawlingContext);
307
+ return { request: crawlingContext.request };
308
+ }
309
+ /**
310
+ * Copies cookies from the live browser page into the session cookie jar.
311
+ */
312
+ async #persistCookiesFromPage(crawlingContext) {
313
+ if (!this.#saveResponseCookies || !crawlingContext.session) {
314
+ return;
315
+ }
316
+ const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
317
+ tryCancel();
318
+ // Prefer the live page URL — the handler may have navigated after the initial load.
319
+ const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
320
+ for (const cookie of cookies) {
321
+ try {
322
+ await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
323
+ ignoreError: false,
324
+ });
325
+ }
326
+ catch (e) {
327
+ this.log.debug(`Could not set cookie: ${e.message}`);
328
+ }
329
+ }
330
+ }
331
+ /**
332
+ * Runs the user request handler, then re-reads browser cookies so login flows /
333
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
334
+ * @internal
335
+ */
336
+ async runRequestHandler(crawlingContext) {
337
+ try {
338
+ await super.runRequestHandler(crawlingContext);
339
+ }
340
+ finally {
341
+ if (!crawlingContext.request.skipNavigation) {
342
+ try {
343
+ await this.#persistCookiesFromPage(crawlingContext);
344
+ }
345
+ catch {
346
+ // Page may already be closed on some failure paths; ignore.
347
+ }
348
+ }
261
349
  }
262
- return {
263
- request: crawlingContext.request,
264
- };
265
350
  }
266
- async handleBlockedRequestByContent(crawlingContext) {
267
- if (this.retryOnBlocked) {
268
- const error = await this.isRequestBlocked(crawlingContext);
269
- if (error)
270
- throw new SessionError(error);
351
+ async #handleBlockedRequestByContent(crawlingContext) {
352
+ if (!this.retryOnBlocked)
353
+ return {};
354
+ const page = crawlingContext.page;
355
+ // A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
356
+ let navigationResponse;
357
+ const onResponse = (response) => {
358
+ if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
359
+ navigationResponse = response;
360
+ }
361
+ };
362
+ page.on('response', onResponse);
363
+ try {
364
+ // Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
365
+ // Wait 5 seconds for the JS challenge to load / resolve before judging the page.
366
+ const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
367
+ if (challenge) {
368
+ await sleep(5000);
369
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
370
+ if (foundSelectors) {
371
+ throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
372
+ }
373
+ return navigationResponse ? { response: navigationResponse } : {};
374
+ }
375
+ }
376
+ finally {
377
+ page.off('response', onResponse);
271
378
  }
379
+ const error = await this.#isRequestBlocked(crawlingContext);
380
+ if (error)
381
+ throw new SessionError(error);
272
382
  return {};
273
383
  }
274
- async restoreRequestState(crawlingContext) {
384
+ async #restoreRequestState(crawlingContext) {
275
385
  crawlingContext.request.state = RequestState.REQUEST_HANDLER;
276
386
  return {};
277
387
  }
278
- async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
279
- const sessionCookie = session?.getCookies(request.url) ?? [];
388
+ async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
389
+ const sessionCookie = session
390
+ ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
391
+ : [];
280
392
  const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
281
393
  const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
282
- await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
394
+ const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
283
395
  .filter((c) => typeof c !== 'undefined' && c !== null)
284
- .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
396
+ .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
397
+ await this.browserPool.injectPageState(page, { cookies });
285
398
  }
286
399
  /**
287
- * Marks session bad in case of navigation timeout.
400
+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
288
401
  */
289
- async _handleNavigationTimeout(crawlingContext, error) {
290
- const { session } = crawlingContext;
291
- if (error && error.constructor.name === 'TimeoutError') {
292
- handleRequestTimeout({ session, errorMessage: error.message });
402
+ async #handleNavigationTimeout(crawlingContext, error) {
403
+ const { session, page } = crawlingContext;
404
+ // Fire-and-forget: no user code will run on this page after a failed navigation.
405
+ // Swallow rejections: the page may already be detached.
406
+ void page.evaluate(() => window.stop()).catch(() => { });
407
+ if (isNavigationTimeoutError(error)) {
408
+ session?.markBad();
409
+ // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
410
+ // hooks have run), so it names that value in its own error; report the configured window instead.
411
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
293
412
  }
294
- await crawlingContext.page.close();
295
413
  }
296
414
  /**
297
415
  * Transforms proxy-related errors to `SessionError`.
298
416
  */
299
- _throwIfProxyError(error) {
417
+ #throwIfProxyError(error) {
300
418
  if (this.isProxyError(error)) {
301
- throw new SessionError(this._getMessageFromError(error));
419
+ throw new SessionError(this.getMessageFromError(error));
302
420
  }
303
421
  }
304
- async processResponse(response, crawlingContext) {
422
+ async #processResponse(response, crawlingContext) {
305
423
  const { session, request, page } = crawlingContext;
306
424
  if (typeof response === 'object' && typeof response.status === 'function') {
307
425
  const status = response.status();
308
- this.stats.registerStatusCode(status);
426
+ this.statistics.registerStatusCode(status);
427
+ // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
428
+ // rate limit the domain should back off from.
429
+ if (status === 429) {
430
+ // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
431
+ const retryAfter = response.headers?.()['retry-after'];
432
+ if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
433
+ throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
434
+ }
435
+ }
436
+ if (this.isErrorStatusCode(status)) {
437
+ if (this.additionalHttpErrorStatusCodes.has(status)) {
438
+ throw new Error(`${status} - Error status code was set by user.`);
439
+ }
440
+ throw new Error(`${status} - Internal Server Error`);
441
+ }
309
442
  }
310
443
  if (this.sessionPool && response && session) {
311
444
  if (typeof response === 'object' && typeof response.status === 'function') {
312
- this._throwOnBlockedRequest(session, response.status());
445
+ this.throwOnBlockedRequest(response.status());
313
446
  }
314
447
  else {
315
448
  this.log.debug('Got a malformed Browser response.', { request, response });
@@ -317,83 +450,18 @@ export class BrowserCrawler extends BasicCrawler {
317
450
  }
318
451
  request.loadedUrl = await page.url();
319
452
  }
320
- async _extendLaunchContext(_pageId, launchContext) {
321
- const launchContextExtends = {};
322
- if (this.sessionPool) {
323
- launchContextExtends.session = await this.sessionPool.newSession({
324
- proxyInfo: await this.proxyConfiguration?.newProxyInfo({
325
- // cannot pass a request here, since session is created on browser launch
326
- }),
327
- });
328
- }
329
- if (!launchContext.proxyUrl && launchContextExtends.session?.proxyInfo) {
330
- const proxyInfo = launchContextExtends.session.proxyInfo;
331
- launchContext.proxyUrl = proxyInfo?.url;
332
- launchContextExtends.proxyInfo = proxyInfo;
333
- // Disable SSL verification for MITM proxies
334
- if (proxyInfo?.ignoreTlsErrors) {
335
- /**
336
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
337
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
338
- */
339
- launchContext.launchOptions.ignoreHTTPSErrors = true;
340
- launchContext.launchOptions.acceptInsecureCerts = true;
341
- }
342
- }
343
- launchContext.extend(launchContextExtends);
344
- }
345
- _maybeAddSessionRetiredListener(_pageId, browserController) {
346
- if (this.sessionPool) {
347
- const listener = (session) => {
348
- const { launchContext } = browserController;
349
- if (session.id === launchContext.session.id) {
350
- this.browserPool.retireBrowserController(browserController);
351
- }
352
- };
353
- this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
354
- browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
355
- return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
356
- });
357
- }
358
- }
359
453
  /**
360
- * Function for cleaning up after all requests are processed.
361
- * @ignore
454
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
455
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
362
456
  */
363
457
  async teardown() {
364
- await this.browserPool.destroy();
458
+ await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
365
459
  await super.teardown();
366
460
  }
367
- }
368
- /** @internal */
369
- function containsEnqueueLinks(options) {
370
- return !!options.enqueueLinks;
371
- }
372
- /** @internal */
373
- export async function browserCrawlerEnqueueLinks(options) {
374
- const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
375
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
376
- enqueueStrategy: enqueueLinksOptions?.strategy,
377
- finalRequestUrl,
378
- originalRequestUrl,
379
- userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
380
- });
381
- const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
382
- if (containsEnqueueLinks(options)) {
383
- return options.enqueueLinks({
384
- urls,
385
- baseUrl,
386
- ...enqueueLinksOptions,
387
- });
461
+ async destroy() {
462
+ await super.destroy();
463
+ await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
388
464
  }
389
- return enqueueLinks({
390
- requestQueue: options.requestQueue,
391
- robotsTxtFile: options.robotsTxtFile,
392
- onSkippedRequest: options.onSkippedRequest,
393
- urls,
394
- baseUrl,
395
- ...enqueueLinksOptions,
396
- });
397
465
  }
398
466
  /**
399
467
  * Extracts URLs from a given page.
@@ -420,4 +488,3 @@ page, selector, baseUrl) {
420
488
  })
421
489
  .filter((href) => !!href);
422
490
  }
423
- //# sourceMappingURL=browser-crawler.js.map