@crawlee/browser-pool 4.0.0-beta.223 → 4.0.0-beta.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstract-classes/browser-plugin.d.ts +1 -29
- package/abstract-classes/browser-plugin.js +3 -57
- package/browser-pool.d.ts +10 -13
- package/browser-pool.js +33 -38
- package/index.d.ts +3 -2
- package/index.js +2 -1
- package/launch-context.d.ts +0 -6
- package/launch-context.js +1 -7
- package/package.json +5 -5
- package/playwright/playwright-plugin.d.ts +0 -8
- package/playwright/playwright-plugin.js +0 -25
- package/playwright/remote-playwright-plugin.d.ts +29 -0
- package/playwright/remote-playwright-plugin.js +38 -0
- package/puppeteer/puppeteer-plugin.d.ts +7 -5
- package/puppeteer/puppeteer-plugin.js +57 -71
- package/puppeteer/remote-puppeteer-plugin.d.ts +22 -0
- package/puppeteer/remote-puppeteer-plugin.js +31 -0
- package/remote-browser-plugin.d.ts +49 -0
- package/remote-browser-plugin.js +63 -0
- package/remote-browser-provider.d.ts +8 -10
- package/remote-browser-provider.js +8 -10
- package/remote-browser-pool.d.ts +0 -170
- package/remote-browser-pool.js +0 -206
|
@@ -5,82 +5,69 @@ import { noop } from '../utils.js';
|
|
|
5
5
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
6
6
|
const PROXY_SERVER_ARG = '--proxy-server=';
|
|
7
7
|
export class PuppeteerPlugin extends BrowserPlugin {
|
|
8
|
-
/**
|
|
9
|
-
* Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
|
|
10
|
-
*
|
|
11
|
-
* @internal
|
|
12
|
-
*/
|
|
13
|
-
useRemoteConnection(connection, parameters = {}) {
|
|
14
|
-
super.useRemoteConnection(connection, parameters);
|
|
15
|
-
if (!this.useIncognitoPages) {
|
|
16
|
-
this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
|
|
17
|
-
'browser instance (useIncognitoPages defaults to false).');
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
8
|
async _launch(launchContext) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const version = +parsed.version.split('.')[0];
|
|
26
|
-
oldPuppeteerVersion = version < 22;
|
|
9
|
+
const oldPuppeteerVersion = await this.isOldPuppeteerVersion();
|
|
10
|
+
const { launchOptions, userDataDir, experimentalContainers, proxyUrl } = launchContext;
|
|
11
|
+
if (experimentalContainers) {
|
|
12
|
+
throw new Error('Experimental containers are only available with Playwright');
|
|
27
13
|
}
|
|
28
|
-
|
|
29
|
-
|
|
14
|
+
launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
|
|
15
|
+
if (launchOptions.headless === false) {
|
|
16
|
+
if (Array.isArray(launchOptions.args)) {
|
|
17
|
+
launchOptions.args.push('--disable-site-isolation-trials');
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
launchOptions.args = ['--disable-site-isolation-trials'];
|
|
21
|
+
}
|
|
30
22
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
if (this.remoteConnection) {
|
|
34
|
-
browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
35
|
-
const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
|
|
36
|
-
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
37
|
-
return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
|
|
38
|
-
});
|
|
23
|
+
if (launchOptions.headless === true && oldPuppeteerVersion) {
|
|
24
|
+
launchOptions.headless = 'new';
|
|
39
25
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if (Array.isArray(launchOptions.args)) {
|
|
48
|
-
launchOptions.args.push('--disable-site-isolation-trials');
|
|
49
|
-
}
|
|
50
|
-
else {
|
|
51
|
-
launchOptions.args = ['--disable-site-isolation-trials'];
|
|
52
|
-
}
|
|
26
|
+
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
|
|
27
|
+
ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
|
|
28
|
+
});
|
|
29
|
+
if (proxyUrl) {
|
|
30
|
+
const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
|
|
31
|
+
if (Array.isArray(launchOptions.args)) {
|
|
32
|
+
launchOptions.args.push(proxyArg);
|
|
53
33
|
}
|
|
54
|
-
|
|
55
|
-
launchOptions.
|
|
34
|
+
else {
|
|
35
|
+
launchOptions.args = [proxyArg];
|
|
56
36
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if (Array.isArray(launchOptions.args)) {
|
|
64
|
-
launchOptions.args.push(proxyArg);
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
launchOptions.args = [proxyArg];
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
try {
|
|
71
|
-
browser = await this.library.launch(launchOptions);
|
|
72
|
-
if (anonymizedProxyUrl) {
|
|
73
|
-
browser.on('disconnected', async () => {
|
|
74
|
-
await close();
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
catch (error) {
|
|
37
|
+
}
|
|
38
|
+
let browser;
|
|
39
|
+
try {
|
|
40
|
+
browser = await this.library.launch(launchOptions);
|
|
41
|
+
if (anonymizedProxyUrl) {
|
|
42
|
+
browser.on('disconnected', async () => {
|
|
79
43
|
await close();
|
|
80
|
-
|
|
81
|
-
}
|
|
44
|
+
});
|
|
82
45
|
}
|
|
83
46
|
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
await close();
|
|
49
|
+
this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
|
|
50
|
+
}
|
|
51
|
+
return this.wrapBrowser(browser, launchContext, oldPuppeteerVersion);
|
|
52
|
+
}
|
|
53
|
+
/** @internal */
|
|
54
|
+
async isOldPuppeteerVersion() {
|
|
55
|
+
try {
|
|
56
|
+
const jsonPath = require.resolve('puppeteer/package.json');
|
|
57
|
+
const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
|
|
58
|
+
return +parsed.version.split('.')[0] < 22;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
|
|
66
|
+
* and {@link RemotePuppeteerPlugin}.
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
wrapBrowser(browser, launchContext, oldPuppeteerVersion) {
|
|
70
|
+
const { useIncognitoPages, proxyUrl, ignoreProxyCertificate, isRemote } = launchContext;
|
|
84
71
|
const targetCreatedHandler = async (target) => {
|
|
85
72
|
try {
|
|
86
73
|
const page = await target.page();
|
|
@@ -96,8 +83,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
96
83
|
}
|
|
97
84
|
};
|
|
98
85
|
browser.on('targetcreated', targetCreatedHandler);
|
|
99
|
-
|
|
100
|
-
if (this.remoteConnection) {
|
|
86
|
+
if (isRemote) {
|
|
101
87
|
browser.once('disconnected', () => {
|
|
102
88
|
browser.off('targetcreated', targetCreatedHandler);
|
|
103
89
|
});
|
|
@@ -123,8 +109,8 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
123
109
|
return async (...args) => {
|
|
124
110
|
let page;
|
|
125
111
|
if (useIncognitoPages) {
|
|
126
|
-
//
|
|
127
|
-
const effectiveProxyUrl =
|
|
112
|
+
// Proxy is managed by the remote service.
|
|
113
|
+
const effectiveProxyUrl = isRemote ? undefined : proxyUrl;
|
|
128
114
|
const [anonymizedProxyUrl, close] = effectiveProxyUrl
|
|
129
115
|
? await anonymizeProxySugar(effectiveProxyUrl, undefined, undefined, {
|
|
130
116
|
ignoreProxyCertificate,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
+
import type Puppeteer from 'puppeteer';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type * as PuppeteerTypes from 'puppeteer';
|
|
5
|
+
import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
|
|
6
|
+
import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
|
|
7
|
+
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
8
|
+
import { type PuppeteerLaunchContext, PuppeteerPlugin } from './puppeteer-plugin.js';
|
|
9
|
+
export interface RemotePuppeteerPluginOptions extends BrowserPluginOptions<PuppeteerTypes.LaunchOptions>, RemoteBrowserOptions {
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
13
|
+
* over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
14
|
+
*/
|
|
15
|
+
export declare class RemotePuppeteerPlugin extends PuppeteerPlugin {
|
|
16
|
+
#private;
|
|
17
|
+
constructor(library: typeof Puppeteer, options: RemotePuppeteerPluginOptions);
|
|
18
|
+
createLaunchContext(options?: CreateLaunchContextOptions<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): PuppeteerLaunchContext;
|
|
19
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
20
|
+
launch(launchContext?: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
21
|
+
protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { connectRemoteBrowser } from '../remote-browser-plugin.js';
|
|
2
|
+
import { PuppeteerPlugin } from './puppeteer-plugin.js';
|
|
3
|
+
/**
|
|
4
|
+
* A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
5
|
+
* over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
6
|
+
*/
|
|
7
|
+
export class RemotePuppeteerPlugin extends PuppeteerPlugin {
|
|
8
|
+
#remote;
|
|
9
|
+
constructor(library, options) {
|
|
10
|
+
super(library, options);
|
|
11
|
+
this.#remote = options;
|
|
12
|
+
if (!this.useIncognitoPages) {
|
|
13
|
+
this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
|
|
14
|
+
'browser instance (useIncognitoPages defaults to false).');
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
createLaunchContext(options = {}) {
|
|
18
|
+
return super.createLaunchContext({ ...options, isRemote: true });
|
|
19
|
+
}
|
|
20
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
21
|
+
async launch(launchContext = this.createLaunchContext()) {
|
|
22
|
+
return this._launch(launchContext);
|
|
23
|
+
}
|
|
24
|
+
async _launch(launchContext) {
|
|
25
|
+
const browser = await connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
|
|
26
|
+
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
27
|
+
return this.library.connect({ ...this.#remote.connectOptions, browserWSEndpoint: url });
|
|
28
|
+
});
|
|
29
|
+
return this.wrapBrowser(browser, launchContext, await this.isOldPuppeteerVersion());
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
3
|
+
/**
|
|
4
|
+
* The result of resolving a remote browser endpoint: the URL to connect to plus an optional opaque
|
|
5
|
+
* `context` object that is handed back to `release`.
|
|
6
|
+
*/
|
|
7
|
+
export interface ResolvedRemoteEndpoint {
|
|
8
|
+
/** The WebSocket / CDP URL to connect to. */
|
|
9
|
+
url: string;
|
|
10
|
+
/** Opaque per-session data (e.g. a session ID) passed back to `release`. */
|
|
11
|
+
context?: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A remote browser endpoint: either a static URL string, or a function called once per browser launch
|
|
15
|
+
* that returns the URL (optionally with a `context` for `release`). The function receives the proxy URL
|
|
16
|
+
* Crawlee resolved for the browser, so it can be forwarded to services that accept an external proxy.
|
|
17
|
+
*/
|
|
18
|
+
export type RemoteBrowserEndpoint = string | ((options?: {
|
|
19
|
+
proxyUrl?: string;
|
|
20
|
+
}) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);
|
|
21
|
+
/** Options shared by all remote browser plugins. */
|
|
22
|
+
export interface RemoteBrowserOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The remote browser endpoint: a static URL, a function returning one per launch, or a
|
|
25
|
+
* {@link RemoteBrowserProvider} instance encapsulating a session create/release lifecycle.
|
|
26
|
+
*/
|
|
27
|
+
endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;
|
|
28
|
+
/**
|
|
29
|
+
* Cleanup callback invoked when the browser disconnects or the connection fails right after the endpoint
|
|
30
|
+
* resolved. Receives the `context` returned by a function endpoint. Errors are caught and logged. Ignored
|
|
31
|
+
* when `endpoint` is a {@link RemoteBrowserProvider} (its own `release()` is used instead).
|
|
32
|
+
*/
|
|
33
|
+
release?: (info: {
|
|
34
|
+
endpoint: string;
|
|
35
|
+
context?: Record<string, unknown>;
|
|
36
|
+
}) => unknown;
|
|
37
|
+
/** Extra options forwarded to the library `connect()` call (endpoint excluded). */
|
|
38
|
+
connectOptions?: Record<string, unknown>;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Resolves `endpoint` for one launch, runs the library-specific `connect`, and ties the session's `release` to
|
|
42
|
+
* the browser's `disconnected` event. On connect failure the session is released and the error is wrapped in a
|
|
43
|
+
* {@link BrowserLaunchError}.
|
|
44
|
+
*
|
|
45
|
+
* @internal
|
|
46
|
+
*/
|
|
47
|
+
export declare function connectRemoteBrowser<Browser extends {
|
|
48
|
+
on(event: 'disconnected', listener: () => void): unknown;
|
|
49
|
+
}>(options: RemoteBrowserOptions, proxyUrl: string | undefined, log: CrawleeLogger, connect: (url: string) => Promise<Browser>): Promise<Browser>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { BrowserLaunchError } from './abstract-classes/browser-plugin.js';
|
|
2
|
+
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
3
|
+
import { sanitizeEndpointForLog } from './utils.js';
|
|
4
|
+
/**
|
|
5
|
+
* Resolves `endpoint` for one launch, runs the library-specific `connect`, and ties the session's `release` to
|
|
6
|
+
* the browser's `disconnected` event. On connect failure the session is released and the error is wrapped in a
|
|
7
|
+
* {@link BrowserLaunchError}.
|
|
8
|
+
*
|
|
9
|
+
* @internal
|
|
10
|
+
*/
|
|
11
|
+
export async function connectRemoteBrowser(options, proxyUrl, log, connect) {
|
|
12
|
+
let resolved;
|
|
13
|
+
try {
|
|
14
|
+
resolved = await resolveEndpoint(options.endpoint, proxyUrl);
|
|
15
|
+
}
|
|
16
|
+
catch (cause) {
|
|
17
|
+
throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
|
|
18
|
+
}
|
|
19
|
+
let released = false;
|
|
20
|
+
const release = async () => {
|
|
21
|
+
if (released)
|
|
22
|
+
return;
|
|
23
|
+
released = true;
|
|
24
|
+
try {
|
|
25
|
+
if (options.endpoint instanceof RemoteBrowserProvider) {
|
|
26
|
+
await options.endpoint.release(resolved.context ?? {});
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
await options.release?.({ endpoint: resolved.url, context: resolved.context });
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
log.warning('Remote browser release() failed.', { error: err?.message });
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
let browser;
|
|
37
|
+
try {
|
|
38
|
+
browser = await connect(resolved.url);
|
|
39
|
+
}
|
|
40
|
+
catch (cause) {
|
|
41
|
+
await release();
|
|
42
|
+
throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(resolved.url)}". ` +
|
|
43
|
+
'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
|
|
44
|
+
}
|
|
45
|
+
browser.on('disconnected', release);
|
|
46
|
+
return browser;
|
|
47
|
+
}
|
|
48
|
+
async function resolveEndpoint(endpoint, proxyUrl) {
|
|
49
|
+
const resolved = endpoint instanceof RemoteBrowserProvider
|
|
50
|
+
? await endpoint.connect({ proxyUrl })
|
|
51
|
+
: typeof endpoint === 'function'
|
|
52
|
+
? await endpoint({ proxyUrl })
|
|
53
|
+
: endpoint;
|
|
54
|
+
if (typeof resolved === 'string') {
|
|
55
|
+
if (!resolved)
|
|
56
|
+
throw new Error('Remote browser endpoint resolved to an empty string.');
|
|
57
|
+
return { url: resolved };
|
|
58
|
+
}
|
|
59
|
+
if (!resolved?.url) {
|
|
60
|
+
throw new Error("Remote browser endpoint() must return a URL string or an object with a non-empty 'url'.");
|
|
61
|
+
}
|
|
62
|
+
return resolved;
|
|
63
|
+
}
|
|
@@ -2,20 +2,18 @@
|
|
|
2
2
|
* Abstract base class for remote browser service providers.
|
|
3
3
|
*
|
|
4
4
|
* Implement this class to encapsulate the lifecycle of a remote browser session
|
|
5
|
-
* (creation, connection URL resolution, and cleanup).
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* (creation, connection URL resolution, and cleanup). A remote browser plugin
|
|
6
|
+
* ({@link RemotePlaywrightPlugin}, {@link RemotePuppeteerPlugin}) calls {@link connect}
|
|
7
|
+
* once per browser launch and {@link release} when the browser disconnects or the connection
|
|
8
|
+
* fails during launch.
|
|
8
9
|
*
|
|
9
|
-
* Pass the provider instance as the `endpoint` of a
|
|
10
|
-
*
|
|
10
|
+
* Pass the provider instance as the `endpoint` of a remote plugin, or as the crawler's
|
|
11
|
+
* `remoteBrowser.endpoint`:
|
|
11
12
|
*
|
|
12
13
|
* ```typescript
|
|
13
|
-
* const
|
|
14
|
-
*
|
|
15
|
-
* endpoint: new MyProvider(),
|
|
14
|
+
* const crawler = new PlaywrightCrawler({
|
|
15
|
+
* remoteBrowser: { endpoint: new MyProvider() },
|
|
16
16
|
* });
|
|
17
|
-
*
|
|
18
|
-
* const crawler = new PlaywrightCrawler({ browserPool });
|
|
19
17
|
* ```
|
|
20
18
|
*
|
|
21
19
|
* **Example — simple static endpoint (e.g. Browserless):**
|
|
@@ -2,20 +2,18 @@
|
|
|
2
2
|
* Abstract base class for remote browser service providers.
|
|
3
3
|
*
|
|
4
4
|
* Implement this class to encapsulate the lifecycle of a remote browser session
|
|
5
|
-
* (creation, connection URL resolution, and cleanup).
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* (creation, connection URL resolution, and cleanup). A remote browser plugin
|
|
6
|
+
* ({@link RemotePlaywrightPlugin}, {@link RemotePuppeteerPlugin}) calls {@link connect}
|
|
7
|
+
* once per browser launch and {@link release} when the browser disconnects or the connection
|
|
8
|
+
* fails during launch.
|
|
8
9
|
*
|
|
9
|
-
* Pass the provider instance as the `endpoint` of a
|
|
10
|
-
*
|
|
10
|
+
* Pass the provider instance as the `endpoint` of a remote plugin, or as the crawler's
|
|
11
|
+
* `remoteBrowser.endpoint`:
|
|
11
12
|
*
|
|
12
13
|
* ```typescript
|
|
13
|
-
* const
|
|
14
|
-
*
|
|
15
|
-
* endpoint: new MyProvider(),
|
|
14
|
+
* const crawler = new PlaywrightCrawler({
|
|
15
|
+
* remoteBrowser: { endpoint: new MyProvider() },
|
|
16
16
|
* });
|
|
17
|
-
*
|
|
18
|
-
* const crawler = new PlaywrightCrawler({ browserPool });
|
|
19
17
|
* ```
|
|
20
18
|
*
|
|
21
19
|
* **Example — simple static endpoint (e.g. Browserless):**
|
package/remote-browser-pool.d.ts
DELETED
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
|
|
2
|
-
import type { BrowserPlugin } from './abstract-classes/browser-plugin.js';
|
|
3
|
-
import { BrowserPool } from './browser-pool.js';
|
|
4
|
-
import type { BrowserPoolHooks, BrowserPoolOptions } from './browser-pool.js';
|
|
5
|
-
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
6
|
-
/**
|
|
7
|
-
* The result of resolving a remote browser endpoint: the URL to connect to plus an optional opaque
|
|
8
|
-
* `context` object that is handed back to `release`.
|
|
9
|
-
*/
|
|
10
|
-
export interface ResolvedRemoteEndpoint {
|
|
11
|
-
/** The browser endpoint URL to connect to. */
|
|
12
|
-
url: string;
|
|
13
|
-
/** Opaque metadata passed back to `release()` — e.g. session IDs, API tokens. */
|
|
14
|
-
context?: Record<string, unknown>;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* A remote browser endpoint: either a static URL string, or a function called once per browser launch
|
|
18
|
-
* that returns a URL (optionally with a `context` for `release`).
|
|
19
|
-
*
|
|
20
|
-
* The function receives the `proxyUrl` resolved by Crawlee's proxy configuration for the launch, so it
|
|
21
|
-
* can forward it to the remote service's proxy API.
|
|
22
|
-
*/
|
|
23
|
-
export type RemoteBrowserEndpoint = string | ((options?: {
|
|
24
|
-
proxyUrl?: string;
|
|
25
|
-
}) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);
|
|
26
|
-
/**
|
|
27
|
-
* The bridge a {@link RemoteBrowserPool} injects into a {@link BrowserPlugin} so the plugin can
|
|
28
|
-
* connect to a remote browser without owning any remote-session policy.
|
|
29
|
-
*
|
|
30
|
-
* The plugin only knows how to make the library-specific `connect()` call; everything else — resolving
|
|
31
|
-
* the endpoint, calling the user's `release()`, and guaranteeing release fires at most once — lives in
|
|
32
|
-
* the pool. The plugin calls {@link RemoteConnection.resolve|resolve} before connecting, stores the
|
|
33
|
-
* returned `token` on its launch context, and the controller later calls
|
|
34
|
-
* {@link RemoteConnection.release|release} with that token when the browser closes.
|
|
35
|
-
*/
|
|
36
|
-
export interface RemoteConnection {
|
|
37
|
-
/** Resolves the endpoint for a single browser launch. The `token` identifies the session for release. */
|
|
38
|
-
resolve(options?: {
|
|
39
|
-
proxyUrl?: string;
|
|
40
|
-
}): Promise<{
|
|
41
|
-
url: string;
|
|
42
|
-
token: number;
|
|
43
|
-
}>;
|
|
44
|
-
/** Releases the remote session for `token`. Idempotent — safe to call from both `close()` and `kill()`. */
|
|
45
|
-
release(token: number): Promise<void>;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Per-plugin remote connection parameters, passed to {@link BrowserPlugin.useRemoteConnection}.
|
|
49
|
-
* The endpoint is supplied per-launch via {@link RemoteConnection}; these are the static connect()
|
|
50
|
-
* parameters (protocol, headers, timeouts, …).
|
|
51
|
-
*/
|
|
52
|
-
export interface RemoteConnectionParameters {
|
|
53
|
-
/**
|
|
54
|
-
* Playwright only: which protocol to connect with. `'cdp'` uses `connectOverCDP()` (the default),
|
|
55
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
56
|
-
* `'playwright'` uses `connect()` (Playwright's own WebSocket protocol). Ignored by Puppeteer.
|
|
57
|
-
*/
|
|
58
|
-
protocol?: 'cdp' | 'playwright';
|
|
59
|
-
/** Extra options forwarded to the library `connect()` / `connectOverCDP()` call (endpoint excluded). */
|
|
60
|
-
connectOptions?: Record<string, unknown>;
|
|
61
|
-
}
|
|
62
|
-
export interface RemoteBrowserPoolOptions {
|
|
63
|
-
/**
|
|
64
|
-
* The browser plugin(s) used to connect to the remote service — e.g. `new PlaywrightPlugin(playwright.chromium)`
|
|
65
|
-
* or `new PuppeteerPlugin(puppeteer)`. The pool configures them for remote connection; do not set a local
|
|
66
|
-
* `launchOptions` on them.
|
|
67
|
-
*/
|
|
68
|
-
browserPlugins: BrowserPlugin[];
|
|
69
|
-
/**
|
|
70
|
-
* The remote browser endpoint: a static URL, a function returning one per launch, or a
|
|
71
|
-
* {@link RemoteBrowserProvider} instance encapsulating a session create/release lifecycle.
|
|
72
|
-
*/
|
|
73
|
-
endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;
|
|
74
|
-
/**
|
|
75
|
-
* Cleanup callback invoked when a browser closes, crashes, or the pool is destroyed. Receives the
|
|
76
|
-
* `context` returned by a function endpoint. Errors are caught and logged. Ignored when `endpoint`
|
|
77
|
-
* is a {@link RemoteBrowserProvider} (its own `release()` is used instead).
|
|
78
|
-
*/
|
|
79
|
-
release?: (info: {
|
|
80
|
-
endpoint: string;
|
|
81
|
-
context?: Record<string, unknown>;
|
|
82
|
-
}) => unknown;
|
|
83
|
-
/**
|
|
84
|
-
* Maximum number of remote browsers open at once. When reached, {@link RemoteBrowserPool.newPage|newPage}
|
|
85
|
-
* waits for a browser to close before connecting a new one. Set it to your service's concurrent-session limit
|
|
86
|
-
* to avoid `429` errors. Defaults to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or
|
|
87
|
-
* `Infinity`.
|
|
88
|
-
*/
|
|
89
|
-
maxOpenBrowsers?: number;
|
|
90
|
-
/** Static connect() parameters (Playwright protocol selection, headers, timeouts, …). */
|
|
91
|
-
connection?: RemoteConnectionParameters;
|
|
92
|
-
/** Extra {@link BrowserPool} options (lifecycle hooks, page limits, fingerprinting, …). */
|
|
93
|
-
browserPoolOptions?: Omit<BrowserPoolOptions, 'browserPlugins'> & BrowserPoolHooks<any, any, any>;
|
|
94
|
-
/**
|
|
95
|
-
* Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it.
|
|
96
|
-
* @default 500
|
|
97
|
-
* @internal
|
|
98
|
-
*/
|
|
99
|
-
slotPollIntervalMillis?: number;
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the
|
|
103
|
-
* {@link RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its
|
|
104
|
-
* browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` — tuning the wrapping
|
|
105
|
-
* pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.
|
|
106
|
-
*/
|
|
107
|
-
export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
|
|
108
|
-
/**
|
|
109
|
-
* An {@link IBrowserPool} implementation for remote browser services.
|
|
110
|
-
*
|
|
111
|
-
* Unlike configuring a remote browser through a crawler's `launchContext`, this pool is the single owner
|
|
112
|
-
* of all remote-session concerns:
|
|
113
|
-
* - **endpoint resolution** — static URL, per-launch function, or {@link RemoteBrowserProvider};
|
|
114
|
-
* - **release lifecycle** — `release()` fires exactly once per session on close/crash/teardown (no leaks,
|
|
115
|
-
* no double-release);
|
|
116
|
-
* - **concurrency** — {@link RemoteBrowserPoolOptions.maxOpenBrowsers|maxOpenBrowsers} is enforced inside
|
|
117
|
-
* {@link RemoteBrowserPool.newPage|newPage}, which waits for a free slot rather than overshooting.
|
|
118
|
-
*
|
|
119
|
-
* The wrapped {@link BrowserPool} and its plugin only perform the library-specific `connect()` call.
|
|
120
|
-
*
|
|
121
|
-
* Pass an instance as the crawler's `browserPool` option:
|
|
122
|
-
*
|
|
123
|
-
* ```typescript
|
|
124
|
-
* import { PlaywrightPlugin, RemoteBrowserPool } from '@crawlee/browser-pool';
|
|
125
|
-
* import { PlaywrightCrawler } from 'crawlee';
|
|
126
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
127
|
-
* import playwright from 'playwright';
|
|
128
|
-
*
|
|
129
|
-
* const browserPool = new RemoteBrowserPool({
|
|
130
|
-
* browserPlugins: [new PlaywrightPlugin(playwright.chromium)],
|
|
131
|
-
* endpoint: 'wss://production-sfo.browserless.io?token=xxx',
|
|
132
|
-
* maxOpenBrowsers: 2,
|
|
133
|
-
* });
|
|
134
|
-
*
|
|
135
|
-
* const crawler = new PlaywrightCrawler({ browserPool });
|
|
136
|
-
* ```
|
|
137
|
-
*
|
|
138
|
-
* @category Browser management
|
|
139
|
-
*/
|
|
140
|
-
export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
|
|
141
|
-
#private;
|
|
142
|
-
/**
|
|
143
|
-
* The wrapped pool that performs the remote connections and serves pages.
|
|
144
|
-
*
|
|
145
|
-
* @internal
|
|
146
|
-
*/
|
|
147
|
-
readonly browserPool: BrowserPool;
|
|
148
|
-
constructor(options: RemoteBrowserPoolOptions);
|
|
149
|
-
/** Maximum number of remote browsers that may be open at the same time. */
|
|
150
|
-
get maxOpenBrowsers(): number;
|
|
151
|
-
set maxOpenBrowsers(value: number);
|
|
152
|
-
/**
|
|
153
|
-
* Opens a new page, waiting first until {@link RemoteBrowserPoolOptions.maxOpenBrowsers|maxOpenBrowsers}
|
|
154
|
-
* allows it (either a new browser slot is free, or an active browser still has page capacity).
|
|
155
|
-
*/
|
|
156
|
-
newPage(options?: NewPageOptions): Promise<Page>;
|
|
157
|
-
closePage(page: Page, options?: {
|
|
158
|
-
error?: Error;
|
|
159
|
-
}): Promise<void>;
|
|
160
|
-
extractPageState(page: Page): Promise<PageState>;
|
|
161
|
-
injectPageState(page: Page, state: PageState): Promise<void>;
|
|
162
|
-
/**
|
|
163
|
-
* Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
|
|
164
|
-
* releases its session (see the constructor), so this needs no separate release pass.
|
|
165
|
-
*/
|
|
166
|
-
releaseAllBrowsers(): Promise<void>;
|
|
167
|
-
[Symbol.asyncDispose](): Promise<void>;
|
|
168
|
-
/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
|
|
169
|
-
destroy(): Promise<void>;
|
|
170
|
-
}
|