@crawlee/browser 4.0.0-beta.2 → 4.0.0-beta.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,21 @@
1
- import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
- import { BrowserPool } from '@crawlee/browser-pool';
3
- import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
4
- import ow from 'ow';
5
- import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
+ import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
+ import { sleep } from '@crawlee/utils';
4
+ import { z } from 'zod';
5
+ import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
6
+ const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
7
+ const readContextField = (ctx, key) => ctx[key];
8
+ /**
9
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
10
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
11
+ * naming the raw millisecond value rather than the configured window.
12
+ */
13
+ function isNavigationTimeoutError(error) {
14
+ return (error instanceof TimeoutError ||
15
+ error?.name === 'TimeoutError' ||
16
+ error?.constructor?.name === 'TimeoutError' ||
17
+ /timeout.*exceeded/i.test(error?.message ?? ''));
18
+ }
6
19
  /**
7
20
  * Provides a simple framework for parallel crawling of web pages
8
21
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,119 +28,126 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
15
28
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
16
29
  * which downloads the pages using raw HTTP requests and is about 10x faster.
17
30
  *
18
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
19
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
20
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
31
+ * The source URLs are represented by the {@link Request} objects that are fed from the
32
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
33
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
21
34
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
22
35
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
23
36
  *
24
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
25
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
26
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
37
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
38
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
39
+ * and pass the result as `requestManager`.
40
+ *
41
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
42
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
27
43
  *
28
44
  * The crawler finishes when there are no more {@link Request} objects to crawl.
29
45
  *
30
46
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
31
47
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
32
48
  *
33
- * New pages are only opened when there is enough free CPU and memory available,
34
- * using the functionality provided by the {@link AutoscaledPool} class.
35
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
36
- * parameter of the `BrowserCrawler` constructor.
37
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
38
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
39
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
49
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
50
+ * {@link ConcurrencySystem}.
51
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
52
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
53
+ * {@link ConcurrencySystem|`concurrencySystem`}.
40
54
  *
41
55
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
42
56
  *
43
57
  * @category Crawlers
44
58
  */
45
59
  export class BrowserCrawler extends BasicCrawler {
46
- config;
60
+ /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
61
+ #browserPoolDep;
47
62
  /**
48
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
49
- * Only available if used by the crawler.
63
+ * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
64
+ * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
50
65
  */
51
- proxyConfiguration;
66
+ get browserPool() {
67
+ return this.#browserPoolDep.value;
68
+ }
69
+ ignoreShadowRoots;
70
+ ignoreIframes;
71
+ #navigationTimeoutMillis;
72
+ #preNavigationHooks;
73
+ #postNavigationHooks;
74
+ #saveResponseCookies;
52
75
  /**
53
- * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
76
+ * @internal
54
77
  */
55
- browserPool;
56
- launchContext;
57
- userProvidedRequestHandler;
58
- navigationTimeoutMillis;
59
- requestHandlerTimeoutInnerMillis;
60
- preNavigationHooks;
61
- postNavigationHooks;
62
- persistCookiesPerSession;
63
78
  static optionsShape = {
64
79
  ...BasicCrawler.optionsShape,
65
- navigationTimeoutSecs: ow.optional.number.greaterThan(0),
66
- preNavigationHooks: ow.optional.array,
67
- postNavigationHooks: ow.optional.array,
68
- launchContext: ow.optional.object,
69
- headless: ow.optional.any(ow.boolean, ow.string),
70
- browserPoolOptions: ow.object,
71
- sessionPoolOptions: ow.optional.object,
72
- persistCookiesPerSession: ow.optional.boolean,
73
- useSessionPool: ow.optional.boolean,
74
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
75
- ignoreShadowRoots: ow.optional.boolean,
76
- ignoreIframes: ow.optional.boolean,
80
+ navigationTimeoutSecs: schemas.anyNumber
81
+ .refine((value) => value > 0, 'Expected a number greater than 0')
82
+ .default(60),
83
+ preNavigationHooks: schemas.anyArray.default(() => []),
84
+ postNavigationHooks: schemas.anyArray.default(() => []),
85
+ browserPool: validators.browserPool.optional(),
86
+ browserPoolBuilder: schemas.anyFunction.optional(),
87
+ remoteBrowser: schemas.anyObject.optional(),
88
+ saveResponseCookies: z.boolean().default(true),
89
+ proxyConfiguration: validators.proxyConfiguration.optional(),
90
+ ignoreIframes: z.boolean().default(false),
91
+ ignoreShadowRoots: z.boolean().default(false),
77
92
  };
93
+ /** @internal */
94
+ static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
78
95
  /**
79
96
  * All `BrowserCrawler` parameters are passed via an options object.
97
+ * @internal
80
98
  */
81
- constructor(options = {}, config = Configuration.getGlobalConfig()) {
82
- ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
83
- const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], requestHandler, failedRequestHandler, headless, ignoreShadowRoots, ignoreIframes, ...basicCrawlerOptions } = options;
99
+ constructor(options) {
100
+ const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
101
+ if (browserPool) {
102
+ assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
103
+ }
104
+ const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
84
105
  super({
85
106
  ...basicCrawlerOptions,
86
- requestHandler: async (...args) => this._runRequestHandler(...args),
87
- requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
88
- }, config);
89
- this.config = config;
90
- // FIXME any
91
- this.userProvidedRequestHandler = requestHandler ?? this.router;
92
- this.failedRequestHandler = failedRequestHandler; // FIXME is this even needed?
93
- // Cookies should be persisted per session only if session pool is used
94
- if (!this.useSessionPool && persistCookiesPerSession) {
95
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
96
- }
97
- this.launchContext = launchContext;
98
- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
99
- this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
100
- this.proxyConfiguration = proxyConfiguration;
101
- this.preNavigationHooks = preNavigationHooks;
102
- this.postNavigationHooks = postNavigationHooks;
103
- if (headless != null) {
104
- this.launchContext.launchOptions ??= {};
105
- this.launchContext.launchOptions.headless = headless;
106
- }
107
- if (this.useSessionPool) {
108
- this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
109
- }
110
- else {
111
- this.persistCookiesPerSession = false;
112
- }
113
- if (launchContext?.userAgent) {
114
- if (browserPoolOptions.useFingerprints)
115
- this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
116
- browserPoolOptions.useFingerprints = false;
117
- }
118
- const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
119
- this.browserPool = new BrowserPool({
120
- ...rest,
121
- preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
122
- postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
107
+ contextPipelineBuilder: () => {
108
+ // A single navigation window covers the pre-navigation hooks, the navigation, and the
109
+ // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
110
+ // hook eats into the same window the navigation uses. The navigation itself is bounded by
111
+ // capping its `gotoOptions.timeout` to the remaining budget.
112
+ const windowGuard = (step) => skipGuard(async (ctx) => {
113
+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
114
+ if (remaining <= 0) {
115
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
116
+ }
117
+ return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
118
+ });
119
+ let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));
120
+ for (const hook of this.#preNavigationHooks) {
121
+ pipeline = pipeline.compose(windowGuard(hook));
122
+ }
123
+ pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
124
+ for (const hook of this.#postNavigationHooks) {
125
+ pipeline = pipeline.compose(windowGuard(hook));
126
+ }
127
+ return pipeline
128
+ .compose(skipGuard(this.finalizeNavigation.bind(this)))
129
+ .compose(this.handleBlockedRequestByContent.bind(this))
130
+ .compose(this.restoreRequestState.bind(this));
131
+ },
132
+ extendContext,
123
133
  });
134
+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
135
+ // The public option hooks are extension-aware; internal storage uses the base context type
136
+ // (the pipeline composes hooks against the concrete context, which does not statically carry
137
+ // `ContextExtension`). The extension members are present at runtime regardless.
138
+ this.#preNavigationHooks = preNavigationHooks;
139
+ this.#postNavigationHooks = postNavigationHooks;
140
+ this.ignoreIframes = ignoreIframes;
141
+ this.ignoreShadowRoots = ignoreShadowRoots;
142
+ this.#saveResponseCookies = saveResponseCookies;
143
+ this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
124
144
  }
125
- async _cleanupContext(crawlingContext) {
126
- const { page } = crawlingContext;
127
- // Page creation may be aborted
128
- if (page) {
129
- await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
130
- }
145
+ /** @internal */
146
+ getNavigationTimeoutMillis() {
147
+ return this.#navigationTimeoutMillis;
148
+ }
149
+ buildContextPipeline() {
150
+ return ContextPipeline.create().compose(this.preparePage.bind(this));
131
151
  }
132
152
  async containsSelectors(page, selectors) {
133
153
  const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
@@ -138,12 +158,6 @@ export class BrowserCrawler extends BasicCrawler {
138
158
  }
139
159
  async isRequestBlocked(crawlingContext) {
140
160
  const { page, response } = crawlingContext;
141
- const blockedStatusCodes =
142
- // eslint-disable-next-line dot-notation
143
- (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
144
- ? // eslint-disable-next-line dot-notation
145
- this.sessionPool['blockedStatusCodes']
146
- : DEFAULT_BLOCKED_STATUS_CODES;
147
161
  // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
148
162
  if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
149
163
  await sleep(5000);
@@ -154,170 +168,244 @@ export class BrowserCrawler extends BasicCrawler {
154
168
  return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
155
169
  }
156
170
  const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
157
- const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
171
+ const statusCode = response?.status() ?? 0;
158
172
  if (foundSelectors)
159
173
  return `Found selectors: ${foundSelectors.join(', ')}`;
160
- if (blockedStatusCode)
161
- return `Received blocked status code: ${blockedStatusCode}`;
174
+ if (this.blockedStatusCodes.has(statusCode))
175
+ return `Received blocked status code: ${statusCode}`;
162
176
  return false;
163
177
  }
164
- /**
165
- * Wrapper around requestHandler that opens and closes pages etc.
166
- */
167
- async _runRequestHandler(crawlingContext) {
168
- const newPageOptions = {
178
+ async preparePage(crawlingContext, onCleanup) {
179
+ const page = await this.browserPool.newPage({
169
180
  id: crawlingContext.id,
170
- };
171
- const useIncognitoPages = this.launchContext?.useIncognitoPages;
172
- if (this.proxyConfiguration) {
173
- const { session } = crawlingContext;
174
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, {
175
- request: crawlingContext.request,
181
+ session: crawlingContext.session,
182
+ });
183
+ tryCancel();
184
+ onCleanup(() => {
185
+ crawlingContext.registerDeferredCleanup(async () => {
186
+ const error = !crawlingContext.session.isUsable()
187
+ ? new SessionError('Session is no longer usable')
188
+ : undefined;
189
+ await this.browserPool
190
+ .closePage(page, { error })
191
+ .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
176
192
  });
177
- crawlingContext.proxyInfo = proxyInfo;
178
- newPageOptions.proxyUrl = proxyInfo?.url;
179
- newPageOptions.proxyTier = proxyInfo?.proxyTier;
180
- if (this.proxyConfiguration.isManInTheMiddle) {
181
- /**
182
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
183
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
184
- */
185
- newPageOptions.pageOptions = {
186
- ignoreHTTPSErrors: true,
187
- acceptInsecureCerts: true,
188
- };
189
- }
193
+ });
194
+ const addRequests = crawlingContext.addRequests;
195
+ const extractLinks = async (options) => {
196
+ return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
197
+ };
198
+ return {
199
+ page,
200
+ get response() {
201
+ throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
202
+ },
203
+ get gotoOptions() {
204
+ throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
205
+ },
206
+ extractLinks,
207
+ enqueueLinks: async (options = {}) => {
208
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
209
+ enqueueStrategy: options.strategy,
210
+ finalRequestUrl: crawlingContext.request.loadedUrl,
211
+ originalRequestUrl: crawlingContext.request.url,
212
+ userProvidedBaseUrl: options.baseUrl,
213
+ });
214
+ const urls = await extractLinks(options);
215
+ return addRequests(urls, {
216
+ ...options,
217
+ baseUrl,
218
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
219
+ });
220
+ },
221
+ };
222
+ }
223
+ async prepareNavigation(crawlingContext) {
224
+ if (crawlingContext.request.skipNavigation) {
225
+ return {
226
+ request: new Proxy(crawlingContext.request, {
227
+ get(target, propertyName, receiver) {
228
+ if (propertyName === 'loadedUrl') {
229
+ throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
230
+ }
231
+ return Reflect.get(target, propertyName, receiver);
232
+ },
233
+ }),
234
+ get response() {
235
+ throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
236
+ },
237
+ };
190
238
  }
191
- const page = (await this.browserPool.newPage(newPageOptions));
239
+ crawlingContext.request.state = RequestState.BEFORE_NAV;
240
+ return {
241
+ // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
242
+ // to the remaining shared window unless a hook overrode it (see there).
243
+ gotoOptions: { timeout: this.#navigationTimeoutMillis },
244
+ [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
245
+ };
246
+ }
247
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
248
+ async navigate(crawlingContext) {
192
249
  tryCancel();
193
- this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
194
- // DO NOT MOVE THIS LINE ABOVE!
195
- // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
196
- // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
197
- // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
198
- const { request, session } = crawlingContext;
199
- if (!request.skipNavigation) {
200
- await this._handleNavigation(crawlingContext);
201
- tryCancel();
202
- await this._responseHandler(crawlingContext);
203
- tryCancel();
204
- // save cookies
205
- // TODO: Should we save the cookies also after/only the handle page?
206
- if (this.persistCookiesPerSession) {
207
- const cookies = await crawlingContext.browserController.getCookies(page);
208
- tryCancel();
209
- session?.setCookies(cookies, request.loadedUrl);
210
- }
211
- }
212
- if (!this.requestMatchesEnqueueStrategy(request)) {
213
- this.log.debug(
214
- // eslint-disable-next-line dot-notation
215
- `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
216
- request.noRetry = true;
217
- request.state = RequestState.SKIPPED;
218
- return;
250
+ const gotoOptions = crawlingContext.gotoOptions;
251
+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
252
+ if (remaining <= 0) {
253
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
219
254
  }
220
- if (this.retryOnBlocked) {
221
- const error = await this.isRequestBlocked(crawlingContext);
222
- if (error)
223
- throw new SessionError(error);
255
+ // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
256
+ // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
257
+ // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
258
+ // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
259
+ const gotoTimeout = gotoOptions;
260
+ if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
261
+ gotoTimeout.timeout = remaining;
224
262
  }
225
- request.state = RequestState.REQUEST_HANDLER;
263
+ const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
264
+ const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
265
+ await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
266
+ let response;
226
267
  try {
227
- await addTimeoutToPromise(async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutInnerMillis, `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`);
228
- request.state = RequestState.DONE;
268
+ response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
229
269
  }
230
- catch (e) {
231
- request.state = RequestState.ERROR;
232
- throw e;
270
+ catch (error) {
271
+ await this.handleNavigationTimeout(crawlingContext, error);
272
+ crawlingContext.request.state = RequestState.ERROR;
273
+ this.throwIfProxyError(error);
274
+ throw error;
233
275
  }
234
276
  tryCancel();
277
+ crawlingContext.request.state = RequestState.AFTER_NAV;
278
+ return { response };
235
279
  }
236
- _enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession) {
237
- crawlingContext.page = page;
238
- // This switch is because the crawlingContexts are created on per request basis.
239
- // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
240
- // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
241
- // which configures proxy and session for every new request
242
- const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
243
- crawlingContext.browserController = browserControllerInstance;
244
- if (!createNewSession) {
245
- crawlingContext.session = browserControllerInstance.launchContext.session;
280
+ async finalizeNavigation(crawlingContext) {
281
+ tryCancel();
282
+ let response;
283
+ try {
284
+ response = crawlingContext.response;
246
285
  }
247
- if (!crawlingContext.proxyInfo) {
248
- crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
286
+ catch {
287
+ // `preparePage` installs a throwing getter for `response`; reaching this branch means
288
+ // navigation produced no response and no hook overrode it. Treat as undefined.
249
289
  }
250
- crawlingContext.enqueueLinks = async (enqueueOptions) => {
251
- return browserCrawlerEnqueueLinks({
252
- options: enqueueOptions,
253
- page,
254
- requestQueue: await this.getRequestQueue(),
255
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
256
- onSkippedRequest: this.onSkippedRequest,
257
- originalRequestUrl: crawlingContext.request.url,
258
- finalRequestUrl: crawlingContext.request.loadedUrl,
259
- });
260
- };
290
+ await this.processResponse(response, crawlingContext);
291
+ tryCancel();
292
+ // Persist cookies from the navigation response before the user handler runs.
293
+ // Cookies set during `requestHandler` are saved again afterwards.
294
+ await this.persistCookiesFromPage(crawlingContext);
295
+ return { request: crawlingContext.request };
261
296
  }
262
- async _handleNavigation(crawlingContext) {
263
- const gotoOptions = { timeout: this.navigationTimeoutMillis };
264
- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
265
- crawlingContext.request.state = RequestState.BEFORE_NAV;
266
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
297
+ /**
298
+ * Copies cookies from the live browser page into the session cookie jar.
299
+ */
300
+ async persistCookiesFromPage(crawlingContext) {
301
+ if (!this.#saveResponseCookies || !crawlingContext.session) {
302
+ return;
303
+ }
304
+ const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
267
305
  tryCancel();
268
- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
269
- await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
306
+ // Prefer the live page URL — the handler may have navigated after the initial load.
307
+ const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
308
+ for (const cookie of cookies) {
309
+ try {
310
+ await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
311
+ ignoreError: false,
312
+ });
313
+ }
314
+ catch (e) {
315
+ this.log.debug(`Could not set cookie: ${e.message}`);
316
+ }
317
+ }
318
+ }
319
+ /**
320
+ * Runs the user request handler, then re-reads browser cookies so login flows /
321
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
322
+ * @internal
323
+ */
324
+ async runRequestHandler(crawlingContext) {
270
325
  try {
271
- crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
326
+ await super.runRequestHandler(crawlingContext);
272
327
  }
273
- catch (error) {
274
- await this._handleNavigationTimeout(crawlingContext, error);
275
- crawlingContext.request.state = RequestState.ERROR;
276
- this._throwIfProxyError(error);
277
- throw error;
328
+ finally {
329
+ if (!crawlingContext.request.skipNavigation) {
330
+ try {
331
+ await this.persistCookiesFromPage(crawlingContext);
332
+ }
333
+ catch {
334
+ // Page may already be closed on some failure paths; ignore.
335
+ }
336
+ }
278
337
  }
279
- tryCancel();
280
- crawlingContext.request.state = RequestState.AFTER_NAV;
281
- await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
282
338
  }
283
- async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
284
- const sessionCookie = session?.getCookies(request.url) ?? [];
339
+ async handleBlockedRequestByContent(crawlingContext) {
340
+ if (this.retryOnBlocked) {
341
+ const error = await this.isRequestBlocked(crawlingContext);
342
+ if (error)
343
+ throw new SessionError(error);
344
+ }
345
+ return {};
346
+ }
347
+ async restoreRequestState(crawlingContext) {
348
+ crawlingContext.request.state = RequestState.REQUEST_HANDLER;
349
+ return {};
350
+ }
351
+ async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
352
+ const sessionCookie = session
353
+ ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
354
+ : [];
285
355
  const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
286
356
  const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
287
- await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
357
+ const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
288
358
  .filter((c) => typeof c !== 'undefined' && c !== null)
289
- .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
359
+ .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
360
+ await this.browserPool.injectPageState(page, { cookies });
290
361
  }
291
362
  /**
292
- * Marks session bad in case of navigation timeout.
363
+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
293
364
  */
294
- async _handleNavigationTimeout(crawlingContext, error) {
295
- const { session } = crawlingContext;
296
- if (error && error.constructor.name === 'TimeoutError') {
297
- handleRequestTimeout({ session, errorMessage: error.message });
365
+ async handleNavigationTimeout(crawlingContext, error) {
366
+ const { session, page } = crawlingContext;
367
+ // Fire-and-forget: no user code will run on this page after a failed navigation.
368
+ // Swallow rejections: the page may already be detached.
369
+ void page.evaluate(() => window.stop()).catch(() => { });
370
+ if (isNavigationTimeoutError(error)) {
371
+ session?.markBad();
372
+ // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
373
+ // hooks have run), so it names that value in its own error; report the configured window instead.
374
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
298
375
  }
299
- await crawlingContext.page.close();
300
376
  }
301
377
  /**
302
378
  * Transforms proxy-related errors to `SessionError`.
303
379
  */
304
- _throwIfProxyError(error) {
380
+ throwIfProxyError(error) {
305
381
  if (this.isProxyError(error)) {
306
- throw new SessionError(this._getMessageFromError(error));
382
+ throw new SessionError(this.getMessageFromError(error));
307
383
  }
308
384
  }
309
- /**
310
- * Should be overridden in case of different automation library that does not support this response API.
311
- */
312
- async _responseHandler(crawlingContext) {
313
- const { response, session, request, page } = crawlingContext;
385
+ async processResponse(response, crawlingContext) {
386
+ const { session, request, page } = crawlingContext;
314
387
  if (typeof response === 'object' && typeof response.status === 'function') {
315
388
  const status = response.status();
316
- this.stats.registerStatusCode(status);
389
+ this.statistics.registerStatusCode(status);
390
+ // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
391
+ // rate limit the domain should back off from.
392
+ if (status === 429) {
393
+ // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
394
+ const retryAfter = response.headers?.()['retry-after'];
395
+ if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
396
+ throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
397
+ }
398
+ }
399
+ if (this.isErrorStatusCode(status)) {
400
+ if (this.additionalHttpErrorStatusCodes.has(status)) {
401
+ throw new Error(`${status} - Error status code was set by user.`);
402
+ }
403
+ throw new Error(`${status} - Internal Server Error`);
404
+ }
317
405
  }
318
406
  if (this.sessionPool && response && session) {
319
407
  if (typeof response === 'object' && typeof response.status === 'function') {
320
- this._throwOnBlockedRequest(session, response.status());
408
+ this.throwOnBlockedRequest(response.status());
321
409
  }
322
410
  else {
323
411
  this.log.debug('Got a malformed Browser response.', { request, response });
@@ -325,69 +413,18 @@ export class BrowserCrawler extends BasicCrawler {
325
413
  }
326
414
  request.loadedUrl = await page.url();
327
415
  }
328
- async _extendLaunchContext(_pageId, launchContext) {
329
- const launchContextExtends = {};
330
- if (this.sessionPool) {
331
- launchContextExtends.session = await this.sessionPool.getSession();
332
- }
333
- if (this.proxyConfiguration && !launchContext.proxyUrl) {
334
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, {
335
- proxyTier: launchContext.proxyTier ?? undefined,
336
- });
337
- launchContext.proxyUrl = proxyInfo?.url;
338
- launchContextExtends.proxyInfo = proxyInfo;
339
- // Disable SSL verification for MITM proxies
340
- if (this.proxyConfiguration.isManInTheMiddle) {
341
- /**
342
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
343
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
344
- */
345
- launchContext.launchOptions.ignoreHTTPSErrors = true;
346
- launchContext.launchOptions.acceptInsecureCerts = true;
347
- }
348
- }
349
- launchContext.extend(launchContextExtends);
350
- }
351
- _maybeAddSessionRetiredListener(_pageId, browserController) {
352
- if (this.sessionPool) {
353
- const listener = (session) => {
354
- const { launchContext } = browserController;
355
- if (session.id === launchContext.session.id) {
356
- this.browserPool.retireBrowserController(browserController);
357
- }
358
- };
359
- this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
360
- browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
361
- return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
362
- });
363
- }
364
- }
365
416
  /**
366
- * Function for cleaning up after all requests are processed.
367
- * @ignore
417
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
418
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
368
419
  */
369
420
  async teardown() {
370
- await this.browserPool.destroy();
421
+ await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
371
422
  await super.teardown();
372
423
  }
373
- }
374
- /** @internal */
375
- export async function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
376
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
377
- enqueueStrategy: options?.strategy,
378
- finalRequestUrl,
379
- originalRequestUrl,
380
- userProvidedBaseUrl: options?.baseUrl,
381
- });
382
- const urls = await extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
383
- return enqueueLinks({
384
- requestQueue,
385
- robotsTxtFile,
386
- onSkippedRequest,
387
- urls,
388
- baseUrl,
389
- ...options,
390
- });
424
+ async destroy() {
425
+ await super.destroy();
426
+ await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
427
+ }
391
428
  }
392
429
  /**
393
430
  * Extracts URLs from a given page.
@@ -414,4 +451,3 @@ page, selector, baseUrl) {
414
451
  })
415
452
  .filter((href) => !!href);
416
453
  }
417
- //# sourceMappingURL=browser-crawler.js.map