@crawlee/browser 4.0.0-beta.19 → 4.0.0-beta.190

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,35 @@
1
- import { BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
- import { BrowserPool } from '@crawlee/browser-pool';
3
- import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
4
- import ow from 'ow';
5
- import { tryCancel } from '@apify/timeout';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
+ import { CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
+ import { sleep } from '@crawlee/utils';
4
+ import { z } from 'zod';
5
+ import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
6
+ /**
7
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
8
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
9
+ * into a second, half-working way of configuring the same pool.
10
+ */
11
+ export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
12
+ const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
13
+ if (names.length === 0) {
14
+ return;
15
+ }
16
+ throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
17
+ `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
18
+ 'itself. Configure the pool you pass in instead.');
19
+ }
20
+ const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
21
+ const readContextField = (ctx, key) => ctx[key];
22
+ /**
23
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
24
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
25
+ * naming the raw millisecond value rather than the configured window.
26
+ */
27
+ function isNavigationTimeoutError(error) {
28
+ return (error instanceof TimeoutError ||
29
+ error?.name === 'TimeoutError' ||
30
+ error?.constructor?.name === 'TimeoutError' ||
31
+ /timeout.*exceeded/i.test(error?.message ?? ''));
32
+ }
6
33
  /**
7
34
  * Provides a simple framework for parallel crawling of web pages
8
35
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,118 +42,127 @@ import { tryCancel } from '@apify/timeout';
15
42
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
16
43
  * which downloads the pages using raw HTTP requests and is about 10x faster.
17
44
  *
18
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
19
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
20
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
45
+ * The source URLs are represented by the {@link Request} objects that are fed from the
46
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
47
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
21
48
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
22
49
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
23
50
  *
24
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
25
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
26
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
51
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
52
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
53
+ * and pass the result as `requestManager`.
54
+ *
55
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
56
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
27
57
  *
28
58
  * The crawler finishes when there are no more {@link Request} objects to crawl.
29
59
  *
30
60
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
31
61
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
32
62
  *
33
- * New pages are only opened when there is enough free CPU and memory available,
34
- * using the functionality provided by the {@link AutoscaledPool} class.
35
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
36
- * parameter of the `BrowserCrawler` constructor.
37
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
38
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
39
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
63
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
64
+ * {@link ConcurrencySystem}.
65
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
66
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
67
+ * {@link ConcurrencySystem|`concurrencySystem`}.
40
68
  *
41
69
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
42
70
  *
43
71
  * @category Crawlers
44
72
  */
45
73
  export class BrowserCrawler extends BasicCrawler {
46
- config;
74
+ /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
75
+ #browserPoolDep;
47
76
  /**
48
- * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
77
+ * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
78
+ * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
49
79
  */
50
- browserPool;
80
+ get browserPool() {
81
+ return this.#browserPoolDep.value;
82
+ }
51
83
  launchContext;
52
84
  ignoreShadowRoots;
53
85
  ignoreIframes;
54
- navigationTimeoutMillis;
55
- preNavigationHooks;
56
- postNavigationHooks;
57
- persistCookiesPerSession;
86
+ #navigationTimeoutMillis;
87
+ #preNavigationHooks;
88
+ #postNavigationHooks;
89
+ #saveResponseCookies;
90
+ /**
91
+ * @internal
92
+ */
58
93
  static optionsShape = {
59
94
  ...BasicCrawler.optionsShape,
60
- navigationTimeoutSecs: ow.optional.number.greaterThan(0),
61
- preNavigationHooks: ow.optional.array,
62
- postNavigationHooks: ow.optional.array,
63
- launchContext: ow.optional.object,
64
- headless: ow.optional.any(ow.boolean, ow.string),
65
- browserPoolOptions: ow.object,
66
- sessionPoolOptions: ow.optional.object,
67
- persistCookiesPerSession: ow.optional.boolean,
68
- useSessionPool: ow.optional.boolean,
69
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
95
+ navigationTimeoutSecs: schemas.anyNumber
96
+ .refine((value) => value > 0, 'Expected a number greater than 0')
97
+ .default(60),
98
+ preNavigationHooks: schemas.anyArray.default(() => []),
99
+ postNavigationHooks: schemas.anyArray.default(() => []),
100
+ launchContext: schemas.anyObject.default(() => ({})),
101
+ browserPool: validators.browserPool.optional(),
102
+ browserPoolBuilder: schemas.anyFunction.optional(),
103
+ remoteBrowser: schemas.anyObject.optional(),
104
+ saveResponseCookies: z.boolean().default(true),
105
+ proxyConfiguration: validators.proxyConfiguration.optional(),
106
+ ignoreIframes: z.boolean().default(false),
107
+ ignoreShadowRoots: z.boolean().default(false),
70
108
  };
109
+ /** @internal */
110
+ static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
71
111
  /**
72
112
  * All `BrowserCrawler` parameters are passed via an options object.
73
113
  */
74
- constructor(options, config = Configuration.getGlobalConfig()) {
75
- ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
76
- const { navigationTimeoutSecs = 60, persistCookiesPerSession, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, proxyConfiguration, ...basicCrawlerOptions } = options;
114
+ constructor(options) {
115
+ const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
116
+ if (browserPool) {
117
+ assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
118
+ }
119
+ const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
77
120
  super({
78
121
  ...basicCrawlerOptions,
79
- contextPipelineBuilder: () => contextPipelineBuilder()
80
- .compose({ action: this.performNavigation.bind(this) })
81
- .compose({ action: this.handleBlockedRequestByContent.bind(this) })
82
- .compose({ action: this.restoreRequestState.bind(this) }),
83
- extendContext: extendContext,
84
- }, config);
85
- this.config = config;
86
- // Cookies should be persisted per session only if session pool is used
87
- if (!this.useSessionPool && persistCookiesPerSession) {
88
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
89
- }
122
+ contextPipelineBuilder: () => {
123
+ // A single navigation window covers the pre-navigation hooks, the navigation, and the
124
+ // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
125
+ // hook eats into the same window the navigation uses. The navigation itself is bounded by
126
+ // capping its `gotoOptions.timeout` to the remaining budget.
127
+ const windowGuard = (step) => skipGuard(async (ctx) => {
128
+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
129
+ if (remaining <= 0) {
130
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
131
+ }
132
+ return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
133
+ });
134
+ let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));
135
+ for (const hook of this.#preNavigationHooks) {
136
+ pipeline = pipeline.compose(windowGuard(hook));
137
+ }
138
+ pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
139
+ for (const hook of this.#postNavigationHooks) {
140
+ pipeline = pipeline.compose(windowGuard(hook));
141
+ }
142
+ return pipeline
143
+ .compose(skipGuard(this.finalizeNavigation.bind(this)))
144
+ .compose(this.handleBlockedRequestByContent.bind(this))
145
+ .compose(this.restoreRequestState.bind(this));
146
+ },
147
+ extendContext,
148
+ });
90
149
  this.launchContext = launchContext;
91
- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
92
- this.proxyConfiguration = proxyConfiguration;
93
- this.preNavigationHooks = preNavigationHooks;
94
- this.postNavigationHooks = postNavigationHooks;
150
+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
151
+ // The public option hooks are extension-aware; internal storage uses the base context type
152
+ // (the pipeline composes hooks against the concrete context, which does not statically carry
153
+ // `ContextExtension`). The extension members are present at runtime regardless.
154
+ this.#preNavigationHooks = preNavigationHooks;
155
+ this.#postNavigationHooks = postNavigationHooks;
95
156
  this.ignoreIframes = ignoreIframes;
96
157
  this.ignoreShadowRoots = ignoreShadowRoots;
97
- if (headless != null) {
98
- this.launchContext.launchOptions ??= {};
99
- this.launchContext.launchOptions.headless = headless;
100
- }
101
- if (this.useSessionPool) {
102
- this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
103
- }
104
- else {
105
- this.persistCookiesPerSession = false;
106
- }
107
- if (launchContext?.userAgent) {
108
- if (browserPoolOptions.useFingerprints)
109
- this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
110
- browserPoolOptions.useFingerprints = false;
111
- }
112
- const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
113
- this.browserPool = new BrowserPool({
114
- ...rest,
115
- preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
116
- postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
117
- });
158
+ this.#saveResponseCookies = saveResponseCookies;
159
+ this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
160
+ }
161
+ getNavigationTimeoutMillis() {
162
+ return this.#navigationTimeoutMillis;
118
163
  }
119
164
  buildContextPipeline() {
120
- return ContextPipeline.create().compose({
121
- action: this.preparePage.bind(this),
122
- cleanup: async (context) => {
123
- context.registerDeferredCleanup(async () => {
124
- await context.page
125
- .close()
126
- .catch((error) => this.log.debug('Error while closing page', { error }));
127
- });
128
- },
129
- });
165
+ return ContextPipeline.create().compose(this.preparePage.bind(this));
130
166
  }
131
167
  async containsSelectors(page, selectors) {
132
168
  const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
@@ -137,12 +173,6 @@ export class BrowserCrawler extends BasicCrawler {
137
173
  }
138
174
  async isRequestBlocked(crawlingContext) {
139
175
  const { page, response } = crawlingContext;
140
- const blockedStatusCodes =
141
- // eslint-disable-next-line dot-notation
142
- (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
143
- ? // eslint-disable-next-line dot-notation
144
- this.sessionPool['blockedStatusCodes']
145
- : DEFAULT_BLOCKED_STATUS_CODES;
146
176
  // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
147
177
  if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
148
178
  await sleep(5000);
@@ -153,117 +183,172 @@ export class BrowserCrawler extends BasicCrawler {
153
183
  return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
154
184
  }
155
185
  const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
156
- const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
186
+ const statusCode = response?.status() ?? 0;
157
187
  if (foundSelectors)
158
188
  return `Found selectors: ${foundSelectors.join(', ')}`;
159
- if (blockedStatusCode)
160
- return `Received blocked status code: ${blockedStatusCode}`;
189
+ if (this.blockedStatusCodes.has(statusCode))
190
+ return `Received blocked status code: ${statusCode}`;
161
191
  return false;
162
192
  }
163
- async preparePage(crawlingContext) {
164
- const newPageOptions = {
193
+ async preparePage(crawlingContext, onCleanup) {
194
+ const page = await this.browserPool.newPage({
165
195
  id: crawlingContext.id,
166
- };
167
- const useIncognitoPages = this.launchContext?.useIncognitoPages;
168
- if (crawlingContext.session?.proxyInfo) {
169
- const proxyInfo = crawlingContext.session.proxyInfo;
170
- crawlingContext.proxyInfo = proxyInfo;
171
- newPageOptions.proxyUrl = proxyInfo?.url;
172
- newPageOptions.proxyTier = proxyInfo?.proxyTier;
173
- if (proxyInfo?.ignoreTlsErrors) {
174
- /**
175
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
176
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
177
- */
178
- newPageOptions.pageOptions = {
179
- ignoreHTTPSErrors: true,
180
- acceptInsecureCerts: true,
181
- };
182
- }
183
- }
184
- const page = (await this.browserPool.newPage(newPageOptions));
196
+ session: crawlingContext.session,
197
+ });
185
198
  tryCancel();
186
- const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
187
- const contextEnqueueLinks = crawlingContext.enqueueLinks;
188
- const session = useIncognitoPages
189
- ? crawlingContext.session
190
- : browserControllerInstance.launchContext.session;
199
+ onCleanup(() => {
200
+ crawlingContext.registerDeferredCleanup(async () => {
201
+ const error = !crawlingContext.session.isUsable()
202
+ ? new SessionError('Session is no longer usable')
203
+ : undefined;
204
+ await this.browserPool
205
+ .closePage(page, { error })
206
+ .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
207
+ });
208
+ });
209
+ const addRequests = crawlingContext.addRequests;
210
+ const extractLinks = async (options) => {
211
+ return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
212
+ };
191
213
  return {
192
214
  page,
193
215
  get response() {
194
216
  throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
195
217
  },
196
- browserController: browserControllerInstance,
197
- session,
198
- proxyInfo: session?.proxyInfo,
199
- enqueueLinks: async (enqueueOptions = {}) => {
200
- return (await browserCrawlerEnqueueLinks({
201
- options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
202
- page,
203
- requestQueue: await this.getRequestQueue(),
204
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
205
- onSkippedRequest: this.handleSkippedRequest,
206
- originalRequestUrl: crawlingContext.request.url,
218
+ get gotoOptions() {
219
+ throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
220
+ },
221
+ extractLinks,
222
+ enqueueLinks: async (options = {}) => {
223
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
224
+ enqueueStrategy: options.strategy,
207
225
  finalRequestUrl: crawlingContext.request.loadedUrl,
208
- enqueueLinks: contextEnqueueLinks,
209
- })); // TODO make this type safe
226
+ originalRequestUrl: crawlingContext.request.url,
227
+ userProvidedBaseUrl: options.baseUrl,
228
+ });
229
+ const urls = await extractLinks(options);
230
+ return addRequests(urls, {
231
+ ...options,
232
+ baseUrl,
233
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
234
+ });
210
235
  },
211
236
  };
212
237
  }
213
- async performNavigation(crawlingContext) {
238
+ async prepareNavigation(crawlingContext) {
214
239
  if (crawlingContext.request.skipNavigation) {
215
240
  return {
216
241
  request: new Proxy(crawlingContext.request, {
217
242
  get(target, propertyName, receiver) {
218
243
  if (propertyName === 'loadedUrl') {
219
- throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
244
+ throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
220
245
  }
221
246
  return Reflect.get(target, propertyName, receiver);
222
247
  },
223
248
  }),
224
249
  get response() {
225
- throw new Error('The `response` property is not available - `skipNavigation` was used');
250
+ throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
226
251
  },
227
252
  };
228
253
  }
229
- const gotoOptions = { timeout: this.navigationTimeoutMillis };
230
- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
231
254
  crawlingContext.request.state = RequestState.BEFORE_NAV;
232
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
255
+ return {
256
+ // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
257
+ // to the remaining shared window unless a hook overrode it (see there).
258
+ gotoOptions: { timeout: this.#navigationTimeoutMillis },
259
+ [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
260
+ };
261
+ }
262
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
263
+ async navigate(crawlingContext) {
233
264
  tryCancel();
234
- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
235
- await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
265
+ const gotoOptions = crawlingContext.gotoOptions;
266
+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
267
+ if (remaining <= 0) {
268
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
269
+ }
270
+ // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
271
+ // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
272
+ // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
273
+ // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
274
+ const gotoTimeout = gotoOptions;
275
+ if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
276
+ gotoTimeout.timeout = remaining;
277
+ }
278
+ const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
279
+ const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
280
+ await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
236
281
  let response;
237
282
  try {
238
- response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
283
+ response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
239
284
  }
240
285
  catch (error) {
241
- await this._handleNavigationTimeout(crawlingContext, error);
286
+ await this.handleNavigationTimeout(crawlingContext, error);
242
287
  crawlingContext.request.state = RequestState.ERROR;
243
- this._throwIfProxyError(error);
288
+ this.throwIfProxyError(error);
244
289
  throw error;
245
290
  }
246
291
  tryCancel();
247
292
  crawlingContext.request.state = RequestState.AFTER_NAV;
248
- await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
293
+ return { response };
294
+ }
295
+ async finalizeNavigation(crawlingContext) {
296
+ tryCancel();
297
+ let response;
298
+ try {
299
+ response = crawlingContext.response;
300
+ }
301
+ catch {
302
+ // `preparePage` installs a throwing getter for `response`; reaching this branch means
303
+ // navigation produced no response and no hook overrode it. Treat as undefined.
304
+ }
249
305
  await this.processResponse(response, crawlingContext);
250
306
  tryCancel();
251
- // save cookies
252
- // TODO: Should we save the cookies also after/only the handle page?
253
- if (this.persistCookiesPerSession) {
254
- const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
255
- tryCancel();
256
- crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl);
307
+ // Persist cookies from the navigation response before the user handler runs.
308
+ // Cookies set during `requestHandler` are saved again afterwards.
309
+ await this.persistCookiesFromPage(crawlingContext);
310
+ return { request: crawlingContext.request };
311
+ }
312
+ /**
313
+ * Copies cookies from the live browser page into the session cookie jar.
314
+ */
315
+ async persistCookiesFromPage(crawlingContext) {
316
+ if (!this.#saveResponseCookies || !crawlingContext.session) {
317
+ return;
257
318
  }
258
- if (response !== undefined) {
259
- return {
260
- request: crawlingContext.request,
261
- response,
262
- };
319
+ const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
320
+ tryCancel();
321
+ // Prefer the live page URL — the handler may have navigated after the initial load.
322
+ const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
323
+ for (const cookie of cookies) {
324
+ try {
325
+ await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
326
+ ignoreError: false,
327
+ });
328
+ }
329
+ catch (e) {
330
+ this.log.debug(`Could not set cookie: ${e.message}`);
331
+ }
332
+ }
333
+ }
334
+ /**
335
+ * Runs the user request handler, then re-reads browser cookies so login flows /
336
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
337
+ */
338
+ async runRequestHandler(crawlingContext) {
339
+ try {
340
+ await super.runRequestHandler(crawlingContext);
341
+ }
342
+ finally {
343
+ if (!crawlingContext.request.skipNavigation) {
344
+ try {
345
+ await this.persistCookiesFromPage(crawlingContext);
346
+ }
347
+ catch {
348
+ // Page may already be closed on some failure paths; ignore.
349
+ }
350
+ }
263
351
  }
264
- return {
265
- request: crawlingContext.request,
266
- };
267
352
  }
268
353
  async handleBlockedRequestByContent(crawlingContext) {
269
354
  if (this.retryOnBlocked) {
@@ -277,41 +362,64 @@ export class BrowserCrawler extends BasicCrawler {
277
362
  crawlingContext.request.state = RequestState.REQUEST_HANDLER;
278
363
  return {};
279
364
  }
280
- async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
281
- const sessionCookie = session?.getCookies(request.url) ?? [];
365
+ async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
366
+ const sessionCookie = session
367
+ ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
368
+ : [];
282
369
  const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
283
370
  const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
284
- await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
371
+ const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
285
372
  .filter((c) => typeof c !== 'undefined' && c !== null)
286
- .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
373
+ .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
374
+ await this.browserPool.injectPageState(page, { cookies });
287
375
  }
288
376
  /**
289
- * Marks session bad in case of navigation timeout.
377
+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
290
378
  */
291
- async _handleNavigationTimeout(crawlingContext, error) {
292
- const { session } = crawlingContext;
293
- if (error && error.constructor.name === 'TimeoutError') {
294
- handleRequestTimeout({ session, errorMessage: error.message });
379
+ async handleNavigationTimeout(crawlingContext, error) {
380
+ const { session, page } = crawlingContext;
381
+ // Fire-and-forget: no user code will run on this page after a failed navigation.
382
+ // Swallow rejections: the page may already be detached.
383
+ void page.evaluate(() => window.stop()).catch(() => { });
384
+ if (isNavigationTimeoutError(error)) {
385
+ session?.markBad();
386
+ // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
387
+ // hooks have run), so it names that value in its own error; report the configured window instead.
388
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
295
389
  }
296
- await crawlingContext.page.close();
297
390
  }
298
391
  /**
299
392
  * Transforms proxy-related errors to `SessionError`.
300
393
  */
301
- _throwIfProxyError(error) {
394
+ throwIfProxyError(error) {
302
395
  if (this.isProxyError(error)) {
303
- throw new SessionError(this._getMessageFromError(error));
396
+ throw new SessionError(this.getMessageFromError(error));
304
397
  }
305
398
  }
306
399
  async processResponse(response, crawlingContext) {
307
400
  const { session, request, page } = crawlingContext;
308
401
  if (typeof response === 'object' && typeof response.status === 'function') {
309
402
  const status = response.status();
310
- this.stats.registerStatusCode(status);
403
+ this.statistics.registerStatusCode(status);
404
+ // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
405
+ // rate limit the domain should back off from.
406
+ if (status === 429) {
407
+ // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
408
+ const retryAfter = response.headers?.()['retry-after'];
409
+ if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
410
+ throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
411
+ }
412
+ }
413
+ if (this.isErrorStatusCode(status)) {
414
+ if (this.additionalHttpErrorStatusCodes.has(status)) {
415
+ throw new Error(`${status} - Error status code was set by user.`);
416
+ }
417
+ throw new Error(`${status} - Internal Server Error`);
418
+ }
311
419
  }
312
420
  if (this.sessionPool && response && session) {
313
421
  if (typeof response === 'object' && typeof response.status === 'function') {
314
- this._throwOnBlockedRequest(session, response.status());
422
+ this.throwOnBlockedRequest(response.status());
315
423
  }
316
424
  else {
317
425
  this.log.debug('Got a malformed Browser response.', { request, response });
@@ -319,83 +427,18 @@ export class BrowserCrawler extends BasicCrawler {
319
427
  }
320
428
  request.loadedUrl = await page.url();
321
429
  }
322
- async _extendLaunchContext(_pageId, launchContext) {
323
- const launchContextExtends = {};
324
- if (this.sessionPool) {
325
- launchContextExtends.session = await this.sessionPool.newSession({
326
- proxyInfo: await this.proxyConfiguration?.newProxyInfo({
327
- // cannot pass a request here, since session is created on browser launch
328
- }),
329
- });
330
- }
331
- if (!launchContext.proxyUrl && launchContextExtends.session?.proxyInfo) {
332
- const proxyInfo = launchContextExtends.session.proxyInfo;
333
- launchContext.proxyUrl = proxyInfo?.url;
334
- launchContextExtends.proxyInfo = proxyInfo;
335
- // Disable SSL verification for MITM proxies
336
- if (proxyInfo?.ignoreTlsErrors) {
337
- /**
338
- * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
339
- * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
340
- */
341
- launchContext.launchOptions.ignoreHTTPSErrors = true;
342
- launchContext.launchOptions.acceptInsecureCerts = true;
343
- }
344
- }
345
- launchContext.extend(launchContextExtends);
346
- }
347
- _maybeAddSessionRetiredListener(_pageId, browserController) {
348
- if (this.sessionPool) {
349
- const listener = (session) => {
350
- const { launchContext } = browserController;
351
- if (session.id === launchContext.session.id) {
352
- this.browserPool.retireBrowserController(browserController);
353
- }
354
- };
355
- this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
356
- browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
357
- return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
358
- });
359
- }
360
- }
361
430
  /**
362
- * Function for cleaning up after all requests are processed.
363
- * @ignore
431
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
432
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
364
433
  */
365
434
  async teardown() {
366
- await this.browserPool.destroy();
435
+ await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
367
436
  await super.teardown();
368
437
  }
369
- }
370
- /** @internal */
371
- function containsEnqueueLinks(options) {
372
- return !!options.enqueueLinks;
373
- }
374
- /** @internal */
375
- export async function browserCrawlerEnqueueLinks(options) {
376
- const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
377
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
378
- enqueueStrategy: enqueueLinksOptions?.strategy,
379
- finalRequestUrl,
380
- originalRequestUrl,
381
- userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
382
- });
383
- const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
384
- if (containsEnqueueLinks(options)) {
385
- return options.enqueueLinks({
386
- urls,
387
- baseUrl,
388
- ...enqueueLinksOptions,
389
- });
438
+ async destroy() {
439
+ await super.destroy();
440
+ await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
390
441
  }
391
- return enqueueLinks({
392
- requestQueue: options.requestQueue,
393
- robotsTxtFile: options.robotsTxtFile,
394
- onSkippedRequest: options.onSkippedRequest,
395
- urls,
396
- baseUrl,
397
- ...enqueueLinksOptions,
398
- });
399
442
  }
400
443
  /**
401
444
  * Extracts URLs from a given page.
@@ -422,4 +465,3 @@ page, selector, baseUrl) {
422
465
  })
423
466
  .filter((href) => !!href);
424
467
  }
425
- //# sourceMappingURL=browser-crawler.js.map