@crawlee/browser 4.0.0-beta.223 → 4.0.0-beta.225

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,9 @@
1
1
  import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
- import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
3
+ import type { CommonPage } from '@crawlee/browser-pool';
4
4
  import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
5
  import { z } from 'zod';
6
+ import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
6
7
  interface BaseResponse {
7
8
  status(): number;
8
9
  /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
@@ -65,14 +66,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
65
66
  /**
66
67
  * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
67
68
  *
68
- * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
69
- * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
70
- * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
71
- * or a {@link RemoteBrowserProvider}.
69
+ * The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
70
+ * for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
71
+ * per launch, or a {@link RemoteBrowserProvider}.
72
72
  *
73
- * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it
74
- * across crawlers, build it with the remote factory for your crawler ({@link remotePlaywrightBrowserPool},
75
- * {@link remotePuppeteerBrowserPool}, {@link remoteStagehandBrowserPool}) and pass it as `browserPool`.
73
+ * Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
74
+ * `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
75
+ * {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
76
76
  */
77
77
  remoteBrowser?: CrawlerRemoteBrowserOptions;
78
78
  /**
@@ -1,6 +1,5 @@
1
1
  import { Configuration } from '@crawlee/basic';
2
- import type { BrowserPlugin, BrowserPluginOptions, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
3
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
2
+ import { type BrowserPlugin, type BrowserPluginOptions, BrowserPool, type BrowserPoolHooks, type BrowserPoolOptions, type RemoteBrowserOptions } from '@crawlee/browser-pool';
4
3
  import type { Constructor, Dictionary } from '@crawlee/types';
5
4
  import { z } from 'zod';
6
5
  export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
@@ -76,9 +75,16 @@ type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
76
75
  [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
77
76
  };
78
77
  /**
79
- * The {@link RemoteBrowserPool} counterpart of {@link LauncherBrowserPoolOptions}.
78
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. Library-specific
79
+ * connect settings (e.g. Playwright's `protocol`) are added by the concrete crawler's options.
80
80
  */
81
- type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
81
+ export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
82
+ /**
83
+ * Maximum number of remote browsers open at once; see {@link BrowserPoolOptions.maxOpenBrowsers}. Defaults
84
+ * to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or `Infinity`.
85
+ */
86
+ maxOpenBrowsers?: number;
87
+ }
82
88
  /**
83
89
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
84
90
  * @ignore
@@ -92,6 +98,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
92
98
  launchOptions: Dictionary;
93
99
  otherLaunchContextProps: Dictionary;
94
100
  Plugin: T;
101
+ RemotePlugin?: T;
95
102
  userAgent?: string;
96
103
  /**
97
104
  * @internal
@@ -125,22 +132,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
125
132
  /**
126
133
  * @ignore
127
134
  */
128
- createBrowserPlugin(): Plugin;
135
+ createBrowserPlugin(remoteBrowser?: CrawlerRemoteBrowserOptions): Plugin;
129
136
  /**
130
137
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
131
138
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
132
139
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
140
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
133
141
  * @internal
134
142
  */
135
- createBrowserPool(options?: LauncherBrowserPoolOptions): BrowserPool<{
143
+ createBrowserPool({ remoteBrowser, ...options }?: LauncherBrowserPoolOptions & {
144
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
145
+ }): BrowserPool<{
136
146
  browserPlugins: [Plugin];
137
- }, [Plugin]>;
138
- /**
139
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
140
- * supplies the plugin, the caller supplies the remote connection details.
141
- * @internal
142
- */
143
- createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
147
+ }, [
148
+ Plugin
149
+ ]>;
144
150
  /**
145
151
  * Launches a browser instance based on the plugin.
146
152
  * @returns Browser instance.
@@ -2,7 +2,7 @@ import fs from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import os from 'node:os';
4
4
  import { Configuration, serviceLocator } from '@crawlee/basic';
5
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
5
+ import { BrowserPool, RemoteBrowserProvider, } from '@crawlee/browser-pool';
6
6
  import { schemas } from '@crawlee/utils/internal';
7
7
  import { z } from 'zod';
8
8
  const DEFAULT_VIEWPORT = {
@@ -21,8 +21,9 @@ export class BrowserLauncher {
21
21
  useChrome;
22
22
  launchOptions;
23
23
  otherLaunchContextProps;
24
- // to be provided by child classes;
24
+ // to be provided by child classes; `RemotePlugin` only by those supporting remote browsers
25
25
  Plugin;
26
+ RemotePlugin;
26
27
  userAgent;
27
28
  /**
28
29
  * @internal
@@ -73,41 +74,42 @@ export class BrowserLauncher {
73
74
  /**
74
75
  * @ignore
75
76
  */
76
- createBrowserPlugin() {
77
- return new this.Plugin(this.launcher, {
77
+ createBrowserPlugin(remoteBrowser) {
78
+ const options = {
78
79
  proxyUrl: this.proxyUrl,
79
80
  launchOptions: this.createLaunchOptions(),
80
81
  ...this.otherLaunchContextProps,
81
- });
82
+ };
83
+ if (!remoteBrowser) {
84
+ return new this.Plugin(this.launcher, options);
85
+ }
86
+ if (!this.RemotePlugin) {
87
+ throw new Error(`${this.constructor.name} does not support remote browsers.`);
88
+ }
89
+ const { maxOpenBrowsers: _, ...remoteOptions } = remoteBrowser;
90
+ return new this.RemotePlugin(this.launcher, { ...options, ...remoteOptions });
82
91
  }
83
92
  /**
84
93
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
85
94
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
86
95
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
96
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
87
97
  * @internal
88
98
  */
89
- createBrowserPool(options = {}) {
99
+ createBrowserPool({ remoteBrowser, ...options } = {}) {
100
+ const provider = remoteBrowser?.endpoint instanceof RemoteBrowserProvider ? remoteBrowser.endpoint : undefined;
101
+ const maxOpenBrowsers = options.maxOpenBrowsers ?? remoteBrowser?.maxOpenBrowsers ?? provider?.maxOpenBrowsers;
90
102
  // The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
91
103
  // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
92
104
  // caller-facing hook types get pinned down.
93
105
  return new BrowserPool({
94
106
  ...this.#resolveFingerprinting(options),
95
- browserPlugins: [this.createBrowserPlugin()],
96
- });
97
- }
98
- /**
99
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
100
- * supplies the plugin, the caller supplies the remote connection details.
101
- * @internal
102
- */
103
- createRemoteBrowserPool(options) {
104
- return new RemoteBrowserPool({
105
- ...options,
106
- browserPlugins: [this.createBrowserPlugin()],
107
- browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
107
+ ...(maxOpenBrowsers !== undefined && { maxOpenBrowsers }),
108
+ browserPlugins: [this.createBrowserPlugin(remoteBrowser)],
108
109
  });
109
110
  }
110
111
  /**
112
+
111
113
  * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
112
114
  * explicitly requested user agent wins.
113
115
  */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-beta.223",
3
+ "version": "4.0.0-beta.225",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.223",
52
- "@crawlee/browser-pool": "4.0.0-beta.223",
53
- "@crawlee/types": "4.0.0-beta.223",
54
- "@crawlee/utils": "4.0.0-beta.223",
51
+ "@crawlee/basic": "4.0.0-beta.225",
52
+ "@crawlee/browser-pool": "4.0.0-beta.225",
53
+ "@crawlee/types": "4.0.0-beta.225",
54
+ "@crawlee/utils": "4.0.0-beta.225",
55
55
  "tslib": "^2.8.1",
56
56
  "zod": "^4.5.4"
57
57
  },
@@ -74,5 +74,5 @@
74
74
  }
75
75
  }
76
76
  },
77
- "gitHead": "cfcbcbebeabd9ca2dfd7a17afe559db16307250d"
77
+ "gitHead": "5d71fa0f785d7dc5736d95bdf5f4ec907e5a1c42"
78
78
  }