@crawlee/browser 4.0.0-beta.223 → 4.0.0-beta.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
|
-
import type { CommonPage
|
|
3
|
+
import type { CommonPage } from '@crawlee/browser-pool';
|
|
4
4
|
import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
|
|
5
5
|
import { z } from 'zod';
|
|
6
|
+
import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
|
|
6
7
|
interface BaseResponse {
|
|
7
8
|
status(): number;
|
|
8
9
|
/** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
|
|
@@ -65,14 +66,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
65
66
|
/**
|
|
66
67
|
* Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
|
|
67
68
|
*
|
|
68
|
-
* The crawler builds a
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* or a {@link RemoteBrowserProvider}.
|
|
69
|
+
* The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
|
|
70
|
+
* for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
|
|
71
|
+
* per launch, or a {@link RemoteBrowserProvider}.
|
|
72
72
|
*
|
|
73
|
-
* Cannot be combined with `browserPool`. To tune the pool
|
|
74
|
-
*
|
|
75
|
-
* {@link
|
|
73
|
+
* Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
|
|
74
|
+
* `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
|
|
75
|
+
* {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
|
|
76
76
|
*/
|
|
77
77
|
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
78
78
|
/**
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { Configuration } from '@crawlee/basic';
|
|
2
|
-
import type
|
|
3
|
-
import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
|
|
2
|
+
import { type BrowserPlugin, type BrowserPluginOptions, BrowserPool, type BrowserPoolHooks, type BrowserPoolOptions, type RemoteBrowserOptions } from '@crawlee/browser-pool';
|
|
4
3
|
import type { Constructor, Dictionary } from '@crawlee/types';
|
|
5
4
|
import { z } from 'zod';
|
|
6
5
|
export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
|
|
@@ -76,9 +75,16 @@ type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
|
|
|
76
75
|
[Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
|
|
77
76
|
};
|
|
78
77
|
/**
|
|
79
|
-
* The
|
|
78
|
+
* The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. Library-specific
|
|
79
|
+
* connect settings (e.g. Playwright's `protocol`) are added by the concrete crawler's options.
|
|
80
80
|
*/
|
|
81
|
-
|
|
81
|
+
export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
|
|
82
|
+
/**
|
|
83
|
+
* Maximum number of remote browsers open at once; see {@link BrowserPoolOptions.maxOpenBrowsers}. Defaults
|
|
84
|
+
* to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or `Infinity`.
|
|
85
|
+
*/
|
|
86
|
+
maxOpenBrowsers?: number;
|
|
87
|
+
}
|
|
82
88
|
/**
|
|
83
89
|
* Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
|
|
84
90
|
* @ignore
|
|
@@ -92,6 +98,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
92
98
|
launchOptions: Dictionary;
|
|
93
99
|
otherLaunchContextProps: Dictionary;
|
|
94
100
|
Plugin: T;
|
|
101
|
+
RemotePlugin?: T;
|
|
95
102
|
userAgent?: string;
|
|
96
103
|
/**
|
|
97
104
|
* @internal
|
|
@@ -125,22 +132,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
125
132
|
/**
|
|
126
133
|
* @ignore
|
|
127
134
|
*/
|
|
128
|
-
createBrowserPlugin(): Plugin;
|
|
135
|
+
createBrowserPlugin(remoteBrowser?: CrawlerRemoteBrowserOptions): Plugin;
|
|
129
136
|
/**
|
|
130
137
|
* Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
|
|
131
138
|
* per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
|
|
132
139
|
* a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
|
|
140
|
+
* With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
|
|
133
141
|
* @internal
|
|
134
142
|
*/
|
|
135
|
-
createBrowserPool(options?: LauncherBrowserPoolOptions
|
|
143
|
+
createBrowserPool({ remoteBrowser, ...options }?: LauncherBrowserPoolOptions & {
|
|
144
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
145
|
+
}): BrowserPool<{
|
|
136
146
|
browserPlugins: [Plugin];
|
|
137
|
-
}, [
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
* supplies the plugin, the caller supplies the remote connection details.
|
|
141
|
-
* @internal
|
|
142
|
-
*/
|
|
143
|
-
createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
147
|
+
}, [
|
|
148
|
+
Plugin
|
|
149
|
+
]>;
|
|
144
150
|
/**
|
|
145
151
|
* Launches a browser instance based on the plugin.
|
|
146
152
|
* @returns Browser instance.
|
|
@@ -2,7 +2,7 @@ import fs from 'node:fs';
|
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import os from 'node:os';
|
|
4
4
|
import { Configuration, serviceLocator } from '@crawlee/basic';
|
|
5
|
-
import { BrowserPool,
|
|
5
|
+
import { BrowserPool, RemoteBrowserProvider, } from '@crawlee/browser-pool';
|
|
6
6
|
import { schemas } from '@crawlee/utils/internal';
|
|
7
7
|
import { z } from 'zod';
|
|
8
8
|
const DEFAULT_VIEWPORT = {
|
|
@@ -21,8 +21,9 @@ export class BrowserLauncher {
|
|
|
21
21
|
useChrome;
|
|
22
22
|
launchOptions;
|
|
23
23
|
otherLaunchContextProps;
|
|
24
|
-
// to be provided by child classes;
|
|
24
|
+
// to be provided by child classes; `RemotePlugin` only by those supporting remote browsers
|
|
25
25
|
Plugin;
|
|
26
|
+
RemotePlugin;
|
|
26
27
|
userAgent;
|
|
27
28
|
/**
|
|
28
29
|
* @internal
|
|
@@ -73,41 +74,42 @@ export class BrowserLauncher {
|
|
|
73
74
|
/**
|
|
74
75
|
* @ignore
|
|
75
76
|
*/
|
|
76
|
-
createBrowserPlugin() {
|
|
77
|
-
|
|
77
|
+
createBrowserPlugin(remoteBrowser) {
|
|
78
|
+
const options = {
|
|
78
79
|
proxyUrl: this.proxyUrl,
|
|
79
80
|
launchOptions: this.createLaunchOptions(),
|
|
80
81
|
...this.otherLaunchContextProps,
|
|
81
|
-
}
|
|
82
|
+
};
|
|
83
|
+
if (!remoteBrowser) {
|
|
84
|
+
return new this.Plugin(this.launcher, options);
|
|
85
|
+
}
|
|
86
|
+
if (!this.RemotePlugin) {
|
|
87
|
+
throw new Error(`${this.constructor.name} does not support remote browsers.`);
|
|
88
|
+
}
|
|
89
|
+
const { maxOpenBrowsers: _, ...remoteOptions } = remoteBrowser;
|
|
90
|
+
return new this.RemotePlugin(this.launcher, { ...options, ...remoteOptions });
|
|
82
91
|
}
|
|
83
92
|
/**
|
|
84
93
|
* Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
|
|
85
94
|
* per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
|
|
86
95
|
* a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
|
|
96
|
+
* With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
|
|
87
97
|
* @internal
|
|
88
98
|
*/
|
|
89
|
-
createBrowserPool(options = {}) {
|
|
99
|
+
createBrowserPool({ remoteBrowser, ...options } = {}) {
|
|
100
|
+
const provider = remoteBrowser?.endpoint instanceof RemoteBrowserProvider ? remoteBrowser.endpoint : undefined;
|
|
101
|
+
const maxOpenBrowsers = options.maxOpenBrowsers ?? remoteBrowser?.maxOpenBrowsers ?? provider?.maxOpenBrowsers;
|
|
90
102
|
// The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
|
|
91
103
|
// parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
|
|
92
104
|
// caller-facing hook types get pinned down.
|
|
93
105
|
return new BrowserPool({
|
|
94
106
|
...this.#resolveFingerprinting(options),
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
|
|
100
|
-
* supplies the plugin, the caller supplies the remote connection details.
|
|
101
|
-
* @internal
|
|
102
|
-
*/
|
|
103
|
-
createRemoteBrowserPool(options) {
|
|
104
|
-
return new RemoteBrowserPool({
|
|
105
|
-
...options,
|
|
106
|
-
browserPlugins: [this.createBrowserPlugin()],
|
|
107
|
-
browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
|
|
107
|
+
...(maxOpenBrowsers !== undefined && { maxOpenBrowsers }),
|
|
108
|
+
browserPlugins: [this.createBrowserPlugin(remoteBrowser)],
|
|
108
109
|
});
|
|
109
110
|
}
|
|
110
111
|
/**
|
|
112
|
+
|
|
111
113
|
* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
|
|
112
114
|
* explicitly requested user agent wins.
|
|
113
115
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.225",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.225",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.225",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.225",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.225",
|
|
55
55
|
"tslib": "^2.8.1",
|
|
56
56
|
"zod": "^4.5.4"
|
|
57
57
|
},
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "5d71fa0f785d7dc5736d95bdf5f4ec907e5a1c42"
|
|
78
78
|
}
|