@crawlee/browser 4.0.0-beta.210 → 4.0.0-beta.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -381,35 +381,14 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
381
381
  /** @internal */
382
382
  protected getNavigationTimeoutMillis(): number;
383
383
  protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
384
- private containsSelectors;
385
- private isRequestBlocked;
386
- private preparePage;
387
- private prepareNavigation;
388
384
  private navigate;
389
- private finalizeNavigation;
390
- /**
391
- * Copies cookies from the live browser page into the session cookie jar.
392
- */
393
- private persistCookiesFromPage;
394
385
  /**
395
386
  * Runs the user request handler, then re-reads browser cookies so login flows /
396
387
  * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
397
388
  * @internal
398
389
  */
399
390
  protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
400
- private handleBlockedRequestByContent;
401
- private restoreRequestState;
402
- private applyCookies;
403
- /**
404
- * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
405
- */
406
- private handleNavigationTimeout;
407
- /**
408
- * Transforms proxy-related errors to `SessionError`.
409
- */
410
- private throwIfProxyError;
411
391
  protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
412
- private processResponse;
413
392
  /**
414
393
  * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
415
394
  * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
@@ -116,7 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
116
116
  }
117
117
  return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
118
118
  });
119
- let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));
119
+ let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
120
120
  for (const hook of this.#preNavigationHooks) {
121
121
  pipeline = pipeline.compose(windowGuard(hook));
122
122
  }
@@ -125,9 +125,9 @@ export class BrowserCrawler extends BasicCrawler {
125
125
  pipeline = pipeline.compose(windowGuard(hook));
126
126
  }
127
127
  return pipeline
128
- .compose(skipGuard(this.finalizeNavigation.bind(this)))
129
- .compose(this.handleBlockedRequestByContent.bind(this))
130
- .compose(this.restoreRequestState.bind(this));
128
+ .compose(skipGuard(this.#finalizeNavigation.bind(this)))
129
+ .compose(this.#handleBlockedRequestByContent.bind(this))
130
+ .compose(this.#restoreRequestState.bind(this));
131
131
  },
132
132
  extendContext,
133
133
  });
@@ -147,27 +147,27 @@ export class BrowserCrawler extends BasicCrawler {
147
147
  return this.#navigationTimeoutMillis;
148
148
  }
149
149
  buildContextPipeline() {
150
- return ContextPipeline.create().compose(this.preparePage.bind(this));
150
+ return ContextPipeline.create().compose(this.#preparePage.bind(this));
151
151
  }
152
- async containsSelectors(page, selectors) {
152
+ async #containsSelectors(page, selectors) {
153
153
  const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
154
154
  .map((x, i) => [x, selectors[i]])
155
155
  .filter(([x]) => x !== null)
156
156
  .map(([, selector]) => selector);
157
157
  return foundSelectors.length > 0 ? foundSelectors : null;
158
158
  }
159
- async isRequestBlocked(crawlingContext) {
159
+ async #isRequestBlocked(crawlingContext) {
160
160
  const { page, response } = crawlingContext;
161
161
  // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
162
- if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
162
+ if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
163
163
  await sleep(5000);
164
164
  // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
165
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
165
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
166
166
  if (!foundSelectors)
167
167
  return false;
168
168
  return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
169
169
  }
170
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
170
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
171
171
  const statusCode = response?.status() ?? 0;
172
172
  if (foundSelectors)
173
173
  return `Found selectors: ${foundSelectors.join(', ')}`;
@@ -175,7 +175,7 @@ export class BrowserCrawler extends BasicCrawler {
175
175
  return `Received blocked status code: ${statusCode}`;
176
176
  return false;
177
177
  }
178
- async preparePage(crawlingContext, onCleanup) {
178
+ async #preparePage(crawlingContext, onCleanup) {
179
179
  const page = await this.browserPool.newPage({
180
180
  id: crawlingContext.id,
181
181
  session: crawlingContext.session,
@@ -232,16 +232,19 @@ export class BrowserCrawler extends BasicCrawler {
232
232
  },
233
233
  };
234
234
  }
235
- async prepareNavigation(crawlingContext) {
235
+ async #prepareNavigation(crawlingContext) {
236
236
  if (crawlingContext.request.skipNavigation) {
237
237
  return {
238
238
  request: new Proxy(crawlingContext.request, {
239
- get(target, propertyName, receiver) {
239
+ get(target, propertyName) {
240
240
  if (propertyName === 'loadedUrl') {
241
241
  throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
242
242
  }
243
- return Reflect.get(target, propertyName, receiver);
243
+ // `target` as receiver and bound methods, or `#` members throw on the proxy
244
+ const value = Reflect.get(target, propertyName, target);
245
+ return typeof value === 'function' ? value.bind(target) : value;
244
246
  },
247
+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
245
248
  }),
246
249
  get response() {
247
250
  throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -274,22 +277,22 @@ export class BrowserCrawler extends BasicCrawler {
274
277
  }
275
278
  const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
276
279
  const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
277
- await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
280
+ await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
278
281
  let response;
279
282
  try {
280
283
  response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
281
284
  }
282
285
  catch (error) {
283
- await this.handleNavigationTimeout(crawlingContext, error);
286
+ await this.#handleNavigationTimeout(crawlingContext, error);
284
287
  crawlingContext.request.state = RequestState.ERROR;
285
- this.throwIfProxyError(error);
288
+ this.#throwIfProxyError(error);
286
289
  throw error;
287
290
  }
288
291
  tryCancel();
289
292
  crawlingContext.request.state = RequestState.AFTER_NAV;
290
293
  return { response };
291
294
  }
292
- async finalizeNavigation(crawlingContext) {
295
+ async #finalizeNavigation(crawlingContext) {
293
296
  tryCancel();
294
297
  let response;
295
298
  try {
@@ -299,17 +302,17 @@ export class BrowserCrawler extends BasicCrawler {
299
302
  // `preparePage` installs a throwing getter for `response`; reaching this branch means
300
303
  // navigation produced no response and no hook overrode it. Treat as undefined.
301
304
  }
302
- await this.processResponse(response, crawlingContext);
305
+ await this.#processResponse(response, crawlingContext);
303
306
  tryCancel();
304
307
  // Persist cookies from the navigation response before the user handler runs.
305
308
  // Cookies set during `requestHandler` are saved again afterwards.
306
- await this.persistCookiesFromPage(crawlingContext);
309
+ await this.#persistCookiesFromPage(crawlingContext);
307
310
  return { request: crawlingContext.request };
308
311
  }
309
312
  /**
310
313
  * Copies cookies from the live browser page into the session cookie jar.
311
314
  */
312
- async persistCookiesFromPage(crawlingContext) {
315
+ async #persistCookiesFromPage(crawlingContext) {
313
316
  if (!this.#saveResponseCookies || !crawlingContext.session) {
314
317
  return;
315
318
  }
@@ -340,7 +343,7 @@ export class BrowserCrawler extends BasicCrawler {
340
343
  finally {
341
344
  if (!crawlingContext.request.skipNavigation) {
342
345
  try {
343
- await this.persistCookiesFromPage(crawlingContext);
346
+ await this.#persistCookiesFromPage(crawlingContext);
344
347
  }
345
348
  catch {
346
349
  // Page may already be closed on some failure paths; ignore.
@@ -348,19 +351,19 @@ export class BrowserCrawler extends BasicCrawler {
348
351
  }
349
352
  }
350
353
  }
351
- async handleBlockedRequestByContent(crawlingContext) {
354
+ async #handleBlockedRequestByContent(crawlingContext) {
352
355
  if (this.retryOnBlocked) {
353
- const error = await this.isRequestBlocked(crawlingContext);
356
+ const error = await this.#isRequestBlocked(crawlingContext);
354
357
  if (error)
355
358
  throw new SessionError(error);
356
359
  }
357
360
  return {};
358
361
  }
359
- async restoreRequestState(crawlingContext) {
362
+ async #restoreRequestState(crawlingContext) {
360
363
  crawlingContext.request.state = RequestState.REQUEST_HANDLER;
361
364
  return {};
362
365
  }
363
- async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
366
+ async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
364
367
  const sessionCookie = session
365
368
  ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
366
369
  : [];
@@ -374,7 +377,7 @@ export class BrowserCrawler extends BasicCrawler {
374
377
  /**
375
378
  * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
376
379
  */
377
- async handleNavigationTimeout(crawlingContext, error) {
380
+ async #handleNavigationTimeout(crawlingContext, error) {
378
381
  const { session, page } = crawlingContext;
379
382
  // Fire-and-forget: no user code will run on this page after a failed navigation.
380
383
  // Swallow rejections: the page may already be detached.
@@ -389,12 +392,12 @@ export class BrowserCrawler extends BasicCrawler {
389
392
  /**
390
393
  * Transforms proxy-related errors to `SessionError`.
391
394
  */
392
- throwIfProxyError(error) {
395
+ #throwIfProxyError(error) {
393
396
  if (this.isProxyError(error)) {
394
397
  throw new SessionError(this.getMessageFromError(error));
395
398
  }
396
399
  }
397
- async processResponse(response, crawlingContext) {
400
+ async #processResponse(response, crawlingContext) {
398
401
  const { session, request, page } = crawlingContext;
399
402
  if (typeof response === 'object' && typeof response.status === 'function') {
400
403
  const status = response.status();
@@ -84,6 +84,7 @@ type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserP
84
84
  * @ignore
85
85
  */
86
86
  export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
87
+ #private;
87
88
  readonly configuration: Configuration;
88
89
  launcher: Launcher;
89
90
  proxyUrl?: string;
@@ -140,11 +141,6 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
140
141
  * @internal
141
142
  */
142
143
  createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
143
- /**
144
- * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
145
- * explicitly requested user agent wins.
146
- */
147
- private resolveFingerprinting;
148
144
  /**
149
145
  * Launches a browser instance based on the plugin.
150
146
  * @returns Browser instance.
@@ -152,11 +148,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
152
148
  launch(): LaunchResult;
153
149
  createLaunchOptions(): Dictionary;
154
150
  protected getDefaultHeadlessOption(): boolean;
155
- private getChromeExecutablePath;
156
- /**
157
- * Gets a typical path to Chrome executable, depending on the current operating system.
158
- */
159
- private getTypicalChromeExecutablePath;
160
- private validateProxyUrlProtocol;
161
151
  }
162
152
  export {};
@@ -61,7 +61,7 @@ export class BrowserLauncher {
61
61
  constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
62
62
  this.configuration = configuration;
63
63
  const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
64
- this.validateProxyUrlProtocol(proxyUrl);
64
+ this.#validateProxyUrlProtocol(proxyUrl);
65
65
  // those need to be reassigned otherwise they are {} in types
66
66
  this.launcher = launcher;
67
67
  this.proxyUrl = proxyUrl;
@@ -91,7 +91,7 @@ export class BrowserLauncher {
91
91
  // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
92
92
  // caller-facing hook types get pinned down.
93
93
  return new BrowserPool({
94
- ...this.resolveFingerprinting(options),
94
+ ...this.#resolveFingerprinting(options),
95
95
  browserPlugins: [this.createBrowserPlugin()],
96
96
  });
97
97
  }
@@ -104,14 +104,14 @@ export class BrowserLauncher {
104
104
  return new RemoteBrowserPool({
105
105
  ...options,
106
106
  browserPlugins: [this.createBrowserPlugin()],
107
- browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),
107
+ browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
108
108
  });
109
109
  }
110
110
  /**
111
111
  * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
112
112
  * explicitly requested user agent wins.
113
113
  */
114
- resolveFingerprinting(options) {
114
+ #resolveFingerprinting(options) {
115
115
  if (!this.userAgent) {
116
116
  return options;
117
117
  }
@@ -148,20 +148,20 @@ export class BrowserLauncher {
148
148
  launchOptions.headless = this.getDefaultHeadlessOption();
149
149
  }
150
150
  if (this.useChrome && !launchOptions.executablePath) {
151
- launchOptions.executablePath = this.getChromeExecutablePath();
151
+ launchOptions.executablePath = this.#getChromeExecutablePath();
152
152
  }
153
153
  return launchOptions;
154
154
  }
155
155
  getDefaultHeadlessOption() {
156
156
  return this.configuration.headless && !this.configuration.xvfb;
157
157
  }
158
- getChromeExecutablePath() {
159
- return this.configuration.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();
158
+ #getChromeExecutablePath() {
159
+ return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
160
160
  }
161
161
  /**
162
162
  * Gets a typical path to Chrome executable, depending on the current operating system.
163
163
  */
164
- getTypicalChromeExecutablePath() {
164
+ #getTypicalChromeExecutablePath() {
165
165
  /**
166
166
  * Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
167
167
  * Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
@@ -189,7 +189,7 @@ export class BrowserLauncher {
189
189
  return '/usr/bin/google-chrome';
190
190
  }
191
191
  }
192
- validateProxyUrlProtocol(proxyUrl) {
192
+ #validateProxyUrlProtocol(proxyUrl) {
193
193
  if (!proxyUrl)
194
194
  return;
195
195
  if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-beta.210",
3
+ "version": "4.0.0-beta.212",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.210",
52
- "@crawlee/browser-pool": "4.0.0-beta.210",
53
- "@crawlee/types": "4.0.0-beta.210",
54
- "@crawlee/utils": "4.0.0-beta.210",
51
+ "@crawlee/basic": "4.0.0-beta.212",
52
+ "@crawlee/browser-pool": "4.0.0-beta.212",
53
+ "@crawlee/types": "4.0.0-beta.212",
54
+ "@crawlee/utils": "4.0.0-beta.212",
55
55
  "tslib": "^2.8.1",
56
56
  "type-fest": "^4.41.0",
57
57
  "zod": "^4.5.4"
@@ -75,5 +75,5 @@
75
75
  }
76
76
  }
77
77
  },
78
- "gitHead": "037dff5d651d88e96749cc79d17bc6d1e154f4e1"
78
+ "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
79
79
  }