@crawlee/browser 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,9 @@
1
1
  import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
- import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
3
+ import type { CommonPage } from '@crawlee/browser-pool';
4
4
  import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
5
  import { z } from 'zod';
6
+ import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
6
7
  interface BaseResponse {
7
8
  status(): number;
8
9
  /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
@@ -65,14 +66,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
65
66
  /**
66
67
  * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
67
68
  *
68
- * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
69
- * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
70
- * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
71
- * or a {@link RemoteBrowserProvider}.
69
+ * The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
70
+ * for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
71
+ * per launch, or a {@link RemoteBrowserProvider}.
72
72
  *
73
- * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it
74
- * across crawlers, build it with the remote factory for your crawler ({@link remotePlaywrightBrowserPool},
75
- * {@link remotePuppeteerBrowserPool}, {@link remoteStagehandBrowserPool}) and pass it as `browserPool`.
73
+ * Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
74
+ * `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
75
+ * {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
76
76
  */
77
77
  remoteBrowser?: CrawlerRemoteBrowserOptions;
78
78
  /**
@@ -1,5 +1,5 @@
1
- import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
- import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
+ import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
3
  import { sleep } from '@crawlee/utils';
4
4
  import { z } from 'zod';
5
5
  import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
@@ -150,23 +150,30 @@ export class BrowserCrawler extends BasicCrawler {
150
150
  return ContextPipeline.create().compose(this.#preparePage.bind(this));
151
151
  }
152
152
  async #containsSelectors(page, selectors) {
153
- const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
153
+ const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
154
+ // a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
155
+ const elements = await query().catch(async () => {
156
+ await sleep(1000);
157
+ return query();
158
+ });
159
+ const foundSelectors = elements
154
160
  .map((x, i) => [x, selectors[i]])
155
161
  .filter(([x]) => x !== null)
156
162
  .map(([, selector]) => selector);
157
163
  return foundSelectors.length > 0 ? foundSelectors : null;
158
164
  }
165
+ async #detectSelfResolvingChallenge({ page, response, }) {
166
+ const statusCode = response?.status();
167
+ if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
168
+ return 'Cloudflare';
169
+ }
170
+ if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
171
+ return 'AWS WAF';
172
+ }
173
+ return null;
174
+ }
159
175
  async #isRequestBlocked(crawlingContext) {
160
176
  const { page, response } = crawlingContext;
161
- // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
162
- if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
163
- await sleep(5000);
164
- // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
165
- const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
166
- if (!foundSelectors)
167
- return false;
168
- return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
169
- }
170
177
  const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
171
178
  const statusCode = response?.status() ?? 0;
172
179
  if (foundSelectors)
@@ -235,17 +242,7 @@ export class BrowserCrawler extends BasicCrawler {
235
242
  async #prepareNavigation(crawlingContext) {
236
243
  if (crawlingContext.request.skipNavigation) {
237
244
  return {
238
- request: new Proxy(crawlingContext.request, {
239
- get(target, propertyName) {
240
- if (propertyName === 'loadedUrl') {
241
- throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
242
- }
243
- // `target` as receiver and bound methods, or `#` members throw on the proxy
244
- const value = Reflect.get(target, propertyName, target);
245
- return typeof value === 'function' ? value.bind(target) : value;
246
- },
247
- set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
248
- }),
245
+ request: skippedNavigationRequest(crawlingContext.request),
249
246
  get response() {
250
247
  throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
251
248
  },
@@ -352,11 +349,36 @@ export class BrowserCrawler extends BasicCrawler {
352
349
  }
353
350
  }
354
351
  async #handleBlockedRequestByContent(crawlingContext) {
355
- if (this.retryOnBlocked) {
356
- const error = await this.#isRequestBlocked(crawlingContext);
357
- if (error)
358
- throw new SessionError(error);
352
+ if (!this.retryOnBlocked)
353
+ return {};
354
+ const page = crawlingContext.page;
355
+ // A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
356
+ let navigationResponse;
357
+ const onResponse = (response) => {
358
+ if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
359
+ navigationResponse = response;
360
+ }
361
+ };
362
+ page.on('response', onResponse);
363
+ try {
364
+ // Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
365
+ // Wait 5 seconds for the JS challenge to load / resolve before judging the page.
366
+ const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
367
+ if (challenge) {
368
+ await sleep(5000);
369
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
370
+ if (foundSelectors) {
371
+ throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
372
+ }
373
+ return navigationResponse ? { response: navigationResponse } : {};
374
+ }
375
+ }
376
+ finally {
377
+ page.off('response', onResponse);
359
378
  }
379
+ const error = await this.#isRequestBlocked(crawlingContext);
380
+ if (error)
381
+ throw new SessionError(error);
360
382
  return {};
361
383
  }
362
384
  async #restoreRequestState(crawlingContext) {
@@ -1,6 +1,5 @@
1
1
  import { Configuration } from '@crawlee/basic';
2
- import type { BrowserPlugin, BrowserPluginOptions, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
3
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
2
+ import { type BrowserPlugin, type BrowserPluginOptions, BrowserPool, type BrowserPoolHooks, type BrowserPoolOptions, type RemoteBrowserOptions } from '@crawlee/browser-pool';
4
3
  import type { Constructor, Dictionary } from '@crawlee/types';
5
4
  import { z } from 'zod';
6
5
  export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
@@ -76,9 +75,16 @@ type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
76
75
  [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
77
76
  };
78
77
  /**
79
- * The {@link RemoteBrowserPool} counterpart of {@link LauncherBrowserPoolOptions}.
78
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. Library-specific
79
+ * connect settings (e.g. Playwright's `protocol`) are added by the concrete crawler's options.
80
80
  */
81
- type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
81
+ export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
82
+ /**
83
+ * Maximum number of remote browsers open at once; see {@link BrowserPoolOptions.maxOpenBrowsers}. Defaults
84
+ * to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or `Infinity`.
85
+ */
86
+ maxOpenBrowsers?: number;
87
+ }
82
88
  /**
83
89
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
84
90
  * @ignore
@@ -92,6 +98,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
92
98
  launchOptions: Dictionary;
93
99
  otherLaunchContextProps: Dictionary;
94
100
  Plugin: T;
101
+ RemotePlugin?: T;
95
102
  userAgent?: string;
96
103
  /**
97
104
  * @internal
@@ -125,22 +132,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
125
132
  /**
126
133
  * @ignore
127
134
  */
128
- createBrowserPlugin(): Plugin;
135
+ createBrowserPlugin(remoteBrowser?: CrawlerRemoteBrowserOptions): Plugin;
129
136
  /**
130
137
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
131
138
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
132
139
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
140
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
133
141
  * @internal
134
142
  */
135
- createBrowserPool(options?: LauncherBrowserPoolOptions): BrowserPool<{
143
+ createBrowserPool({ remoteBrowser, ...options }?: LauncherBrowserPoolOptions & {
144
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
145
+ }): BrowserPool<{
136
146
  browserPlugins: [Plugin];
137
- }, [Plugin]>;
138
- /**
139
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
140
- * supplies the plugin, the caller supplies the remote connection details.
141
- * @internal
142
- */
143
- createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
147
+ }, [
148
+ Plugin
149
+ ]>;
144
150
  /**
145
151
  * Launches a browser instance based on the plugin.
146
152
  * @returns Browser instance.
@@ -2,7 +2,7 @@ import fs from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import os from 'node:os';
4
4
  import { Configuration, serviceLocator } from '@crawlee/basic';
5
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
5
+ import { BrowserPool, RemoteBrowserProvider, } from '@crawlee/browser-pool';
6
6
  import { schemas } from '@crawlee/utils/internal';
7
7
  import { z } from 'zod';
8
8
  const DEFAULT_VIEWPORT = {
@@ -21,8 +21,9 @@ export class BrowserLauncher {
21
21
  useChrome;
22
22
  launchOptions;
23
23
  otherLaunchContextProps;
24
- // to be provided by child classes;
24
+ // to be provided by child classes; `RemotePlugin` only by those supporting remote browsers
25
25
  Plugin;
26
+ RemotePlugin;
26
27
  userAgent;
27
28
  /**
28
29
  * @internal
@@ -73,41 +74,42 @@ export class BrowserLauncher {
73
74
  /**
74
75
  * @ignore
75
76
  */
76
- createBrowserPlugin() {
77
- return new this.Plugin(this.launcher, {
77
+ createBrowserPlugin(remoteBrowser) {
78
+ const options = {
78
79
  proxyUrl: this.proxyUrl,
79
80
  launchOptions: this.createLaunchOptions(),
80
81
  ...this.otherLaunchContextProps,
81
- });
82
+ };
83
+ if (!remoteBrowser) {
84
+ return new this.Plugin(this.launcher, options);
85
+ }
86
+ if (!this.RemotePlugin) {
87
+ throw new Error(`${this.constructor.name} does not support remote browsers.`);
88
+ }
89
+ const { maxOpenBrowsers: _, ...remoteOptions } = remoteBrowser;
90
+ return new this.RemotePlugin(this.launcher, { ...options, ...remoteOptions });
82
91
  }
83
92
  /**
84
93
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
85
94
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
86
95
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
96
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
87
97
  * @internal
88
98
  */
89
- createBrowserPool(options = {}) {
99
+ createBrowserPool({ remoteBrowser, ...options } = {}) {
100
+ const provider = remoteBrowser?.endpoint instanceof RemoteBrowserProvider ? remoteBrowser.endpoint : undefined;
101
+ const maxOpenBrowsers = options.maxOpenBrowsers ?? remoteBrowser?.maxOpenBrowsers ?? provider?.maxOpenBrowsers;
90
102
  // The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
91
103
  // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
92
104
  // caller-facing hook types get pinned down.
93
105
  return new BrowserPool({
94
106
  ...this.#resolveFingerprinting(options),
95
- browserPlugins: [this.createBrowserPlugin()],
96
- });
97
- }
98
- /**
99
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
100
- * supplies the plugin, the caller supplies the remote connection details.
101
- * @internal
102
- */
103
- createRemoteBrowserPool(options) {
104
- return new RemoteBrowserPool({
105
- ...options,
106
- browserPlugins: [this.createBrowserPlugin()],
107
- browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
107
+ ...(maxOpenBrowsers !== undefined && { maxOpenBrowsers }),
108
+ browserPlugins: [this.createBrowserPlugin(remoteBrowser)],
108
109
  });
109
110
  }
110
111
  /**
112
+
111
113
  * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
112
114
  * explicitly requested user agent wins.
113
115
  */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-rc.1",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,12 +48,12 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-rc.1",
52
- "@crawlee/browser-pool": "4.0.0-rc.1",
53
- "@crawlee/types": "4.0.0-rc.1",
54
- "@crawlee/utils": "4.0.0-rc.1",
55
- "tslib": "^2.8.1",
56
- "zod": "^4.5.4"
51
+ "@crawlee/basic": "4.0.0-rc.2",
52
+ "@crawlee/browser-pool": "4.0.0-rc.2",
53
+ "@crawlee/types": "4.0.0-rc.2",
54
+ "@crawlee/utils": "4.0.0-rc.2",
55
+ "tslib": "^2.9.1",
56
+ "zod": "^4.6.7"
57
57
  },
58
58
  "peerDependencies": {
59
59
  "playwright": "*",
@@ -74,5 +74,5 @@
74
74
  }
75
75
  }
76
76
  },
77
- "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
77
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
78
78
  }