@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstract-classes/browser-controller.d.ts +1 -3
- package/abstract-classes/browser-controller.js +13 -11
- package/abstract-classes/browser-plugin.d.ts +5 -5
- package/abstract-classes/browser-plugin.js +7 -7
- package/browser-pool.d.ts +13 -15
- package/browser-pool.js +75 -70
- package/events.d.ts +2 -2
- package/fingerprinting/hooks.js +2 -1
- package/fingerprinting/types.d.ts +2 -2
- package/fingerprinting/utils.js +5 -5
- package/launch-context.d.ts +2 -3
- package/launch-context.js +11 -8
- package/package.json +6 -6
- package/playwright/playwright-browser.d.ts +2 -5
- package/playwright/playwright-browser.js +16 -16
- package/playwright/playwright-plugin.d.ts +4 -4
- package/playwright/playwright-plugin.js +14 -12
- package/puppeteer/puppeteer-plugin.d.ts +2 -2
- package/puppeteer/puppeteer-plugin.js +4 -4
- package/remote-browser-pool.d.ts +7 -13
- package/remote-browser-pool.js +48 -47
|
@@ -36,7 +36,7 @@ export interface BrowserSpecification {
|
|
|
36
36
|
*/
|
|
37
37
|
httpVersion?: HttpVersion;
|
|
38
38
|
}
|
|
39
|
-
export declare
|
|
39
|
+
export declare enum OperatingSystemsName {
|
|
40
40
|
linux = "linux",
|
|
41
41
|
macos = "macos",
|
|
42
42
|
windows = "windows",
|
|
@@ -49,7 +49,7 @@ export declare const enum OperatingSystemsName {
|
|
|
49
49
|
*/
|
|
50
50
|
ios = "ios"
|
|
51
51
|
}
|
|
52
|
-
export declare
|
|
52
|
+
export declare enum DeviceCategory {
|
|
53
53
|
/**
|
|
54
54
|
* Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
|
|
55
55
|
* > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
|
package/fingerprinting/utils.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
|
-
import { BrowserName } from './types.js';
|
|
3
|
+
import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
|
|
4
4
|
export const getGeneratorDefaultOptions = (launchContext) => {
|
|
5
5
|
const { browserPlugin, launchOptions } = launchContext;
|
|
6
6
|
const options = {
|
|
7
|
-
devices: [
|
|
7
|
+
devices: [DeviceCategory.desktop],
|
|
8
8
|
locales: ['en-US'],
|
|
9
9
|
browsers: [getBrowserName(browserPlugin, launchOptions)],
|
|
10
10
|
operatingSystems: [getOperatingSystem()],
|
|
@@ -34,11 +34,11 @@ const getOperatingSystem = () => {
|
|
|
34
34
|
switch (platform) {
|
|
35
35
|
case 'win32':
|
|
36
36
|
// platform is win32 even for 64-bit
|
|
37
|
-
return
|
|
37
|
+
return OperatingSystemsName.windows;
|
|
38
38
|
case 'darwin':
|
|
39
|
-
return
|
|
39
|
+
return OperatingSystemsName.macos;
|
|
40
40
|
default:
|
|
41
41
|
// consider everything else a linux
|
|
42
|
-
return
|
|
42
|
+
return OperatingSystemsName.linux;
|
|
43
43
|
}
|
|
44
44
|
};
|
package/launch-context.d.ts
CHANGED
|
@@ -56,6 +56,7 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
56
56
|
isRemote?: boolean;
|
|
57
57
|
}
|
|
58
58
|
export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
59
|
+
#private;
|
|
59
60
|
id?: string;
|
|
60
61
|
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
61
62
|
launchOptions: LibraryOptions;
|
|
@@ -64,15 +65,13 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
64
65
|
userDataDir: string;
|
|
65
66
|
readonly isRemote: boolean;
|
|
66
67
|
ignoreProxyCertificate?: boolean;
|
|
67
|
-
private _proxyUrl?;
|
|
68
|
-
private readonly _reservedFieldNames;
|
|
69
68
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
70
69
|
/**
|
|
71
70
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
72
71
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
73
72
|
* @internal
|
|
74
73
|
*/
|
|
75
|
-
|
|
74
|
+
remoteToken?: number;
|
|
76
75
|
[K: PropertyKey]: unknown;
|
|
77
76
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
78
77
|
/**
|
package/launch-context.js
CHANGED
|
@@ -7,15 +7,15 @@ export class LaunchContext {
|
|
|
7
7
|
userDataDir;
|
|
8
8
|
isRemote;
|
|
9
9
|
ignoreProxyCertificate;
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
#proxyUrl;
|
|
11
|
+
#reservedFieldNames;
|
|
12
12
|
fingerprint;
|
|
13
13
|
/**
|
|
14
14
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
15
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
16
|
* @internal
|
|
17
17
|
*/
|
|
18
|
-
|
|
18
|
+
remoteToken;
|
|
19
19
|
constructor(options) {
|
|
20
20
|
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
21
21
|
this.id = id;
|
|
@@ -26,7 +26,10 @@ export class LaunchContext {
|
|
|
26
26
|
this.userDataDir = userDataDir;
|
|
27
27
|
this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
|
|
28
28
|
this.isRemote = isRemote ?? false;
|
|
29
|
-
this
|
|
29
|
+
this.#proxyUrl = proxyUrl;
|
|
30
|
+
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
|
+
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* Extend the launch context with any extra fields.
|
|
@@ -37,7 +40,7 @@ export class LaunchContext {
|
|
|
37
40
|
*/
|
|
38
41
|
extend(fields) {
|
|
39
42
|
Object.entries(fields).forEach(([key, value]) => {
|
|
40
|
-
if (this.
|
|
43
|
+
if (this.#reservedFieldNames.includes(key)) {
|
|
41
44
|
throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
|
|
42
45
|
}
|
|
43
46
|
else {
|
|
@@ -51,7 +54,7 @@ export class LaunchContext {
|
|
|
51
54
|
*/
|
|
52
55
|
set proxyUrl(url) {
|
|
53
56
|
if (!url) {
|
|
54
|
-
this
|
|
57
|
+
this.#proxyUrl = undefined;
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
60
|
const urlInstance = new URL(url);
|
|
@@ -59,12 +62,12 @@ export class LaunchContext {
|
|
|
59
62
|
urlInstance.search = '';
|
|
60
63
|
urlInstance.hash = '';
|
|
61
64
|
// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
|
|
62
|
-
this
|
|
65
|
+
this.#proxyUrl = urlInstance.href.slice(0, -1);
|
|
63
66
|
}
|
|
64
67
|
/**
|
|
65
68
|
* Returns the proxy URL of the browser.
|
|
66
69
|
*/
|
|
67
70
|
get proxyUrl() {
|
|
68
|
-
return this
|
|
71
|
+
return this.#proxyUrl;
|
|
69
72
|
}
|
|
70
73
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-
|
|
3
|
+
"version": "4.0.0-rc.0",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -31,18 +31,18 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@apify/timeout": "^0.4.4",
|
|
34
|
-
"@crawlee/core": "4.0.0-
|
|
35
|
-
"@crawlee/types": "4.0.0-
|
|
34
|
+
"@crawlee/core": "4.0.0-rc.0",
|
|
35
|
+
"@crawlee/types": "4.0.0-rc.0",
|
|
36
36
|
"fingerprint-generator": "^2.1.68",
|
|
37
37
|
"fingerprint-injector": "^2.1.68",
|
|
38
38
|
"lodash.merge": "^4.6.2",
|
|
39
39
|
"nanoid": "^5.1.5",
|
|
40
|
-
"ow": "^2.0.0",
|
|
41
40
|
"p-limit": "^6.2.0",
|
|
42
41
|
"proxy-chain": "^2.5.8",
|
|
43
42
|
"quick-lru": "^7.0.1",
|
|
44
43
|
"tiny-typed-emitter": "^2.1.0",
|
|
45
|
-
"tslib": "^2.8.1"
|
|
44
|
+
"tslib": "^2.8.1",
|
|
45
|
+
"zod": "^4.4.3"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"playwright": "*",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
|
|
67
67
|
}
|
|
@@ -9,10 +9,7 @@ export interface BrowserOptions {
|
|
|
9
9
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
10
10
|
*/
|
|
11
11
|
export declare class PlaywrightBrowser extends EventEmitter {
|
|
12
|
-
private
|
|
13
|
-
private _version;
|
|
14
|
-
private _isConnected;
|
|
15
|
-
private _browserType?;
|
|
12
|
+
#private;
|
|
16
13
|
constructor(options: BrowserOptions);
|
|
17
14
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
15
|
close(): Promise<void>;
|
|
@@ -20,7 +17,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
20
17
|
isConnected(): boolean;
|
|
21
18
|
version(): string;
|
|
22
19
|
/** @internal */
|
|
23
|
-
|
|
20
|
+
setBrowserType(browserType: BrowserType): void;
|
|
24
21
|
browserType(): BrowserType;
|
|
25
22
|
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
|
26
23
|
newContext(): Promise<never>;
|
|
@@ -3,17 +3,17 @@ import { EventEmitter } from 'node:events';
|
|
|
3
3
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
4
4
|
*/
|
|
5
5
|
export class PlaywrightBrowser extends EventEmitter {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
#browserContext;
|
|
7
|
+
#version;
|
|
8
|
+
#isConnected = true;
|
|
9
|
+
#browserType;
|
|
10
10
|
constructor(options) {
|
|
11
11
|
super();
|
|
12
12
|
const { browserContext, version } = options;
|
|
13
|
-
this
|
|
14
|
-
this
|
|
15
|
-
this.
|
|
16
|
-
this
|
|
13
|
+
this.#browserContext = browserContext;
|
|
14
|
+
this.#version = version;
|
|
15
|
+
this.#browserContext.once('close', () => {
|
|
16
|
+
this.#isConnected = false;
|
|
17
17
|
this.emit('disconnected');
|
|
18
18
|
});
|
|
19
19
|
}
|
|
@@ -21,26 +21,26 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
21
21
|
await this.close();
|
|
22
22
|
}
|
|
23
23
|
async close() {
|
|
24
|
-
await this.
|
|
24
|
+
await this.#browserContext.close();
|
|
25
25
|
}
|
|
26
26
|
contexts() {
|
|
27
|
-
return [this
|
|
27
|
+
return [this.#browserContext];
|
|
28
28
|
}
|
|
29
29
|
isConnected() {
|
|
30
|
-
return this
|
|
30
|
+
return this.#isConnected;
|
|
31
31
|
}
|
|
32
32
|
version() {
|
|
33
|
-
return this
|
|
33
|
+
return this.#version;
|
|
34
34
|
}
|
|
35
35
|
/** @internal */
|
|
36
|
-
|
|
37
|
-
this
|
|
36
|
+
setBrowserType(browserType) {
|
|
37
|
+
this.#browserType = browserType;
|
|
38
38
|
}
|
|
39
39
|
browserType() {
|
|
40
|
-
return this
|
|
40
|
+
return this.#browserType;
|
|
41
41
|
}
|
|
42
42
|
async newPage(...args) {
|
|
43
|
-
return this.
|
|
43
|
+
return this.#browserContext.newPage(...args);
|
|
44
44
|
}
|
|
45
45
|
async newContext() {
|
|
46
46
|
throw new Error('Function `newContext()` is not available in incognito mode');
|
|
@@ -6,15 +6,15 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
|
|
|
6
6
|
import type { SafeParameters } from '../utils.js';
|
|
7
7
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
8
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
|
-
private
|
|
9
|
+
#private;
|
|
10
10
|
/**
|
|
11
11
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
12
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
13
|
*/
|
|
14
14
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
15
15
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
16
|
-
private
|
|
16
|
+
private throwOnFailedLaunch;
|
|
17
17
|
createController(): PlaywrightController;
|
|
18
|
-
protected
|
|
19
|
-
protected
|
|
18
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
19
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
20
20
|
}
|
|
@@ -5,7 +5,7 @@ import { getLocalProxyAddress } from '../proxy-server.js';
|
|
|
5
5
|
import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
|
|
6
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
7
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
8
|
-
|
|
8
|
+
#browserVersion;
|
|
9
9
|
/**
|
|
10
10
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
11
11
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
@@ -20,7 +20,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
20
20
|
}
|
|
21
21
|
async _launch(launchContext) {
|
|
22
22
|
if (this.remoteConnection) {
|
|
23
|
-
return this.
|
|
23
|
+
return this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
24
24
|
const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
|
|
25
25
|
if (this.remoteConnectionParameters?.protocol === 'playwright') {
|
|
26
26
|
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
@@ -53,7 +53,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
53
53
|
try {
|
|
54
54
|
if (useIncognitoPages) {
|
|
55
55
|
browser = await this.library.launch(launchOptions).catch((error) => {
|
|
56
|
-
return this.
|
|
56
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
57
57
|
});
|
|
58
58
|
if (anonymizedProxyUrl) {
|
|
59
59
|
browser.on('disconnected', async () => {
|
|
@@ -65,7 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
65
65
|
const browserContext = await this.library
|
|
66
66
|
.launchPersistentContext(userDataDir, launchOptions)
|
|
67
67
|
.catch((error) => {
|
|
68
|
-
return this.
|
|
68
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
69
69
|
});
|
|
70
70
|
browserContext.once('close', () => {
|
|
71
71
|
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
|
|
@@ -80,18 +80,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
80
80
|
await close();
|
|
81
81
|
});
|
|
82
82
|
}
|
|
83
|
-
if (!this
|
|
83
|
+
if (!this.#browserVersion) {
|
|
84
84
|
// Launches unused browser just to get the browser version.
|
|
85
85
|
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
86
|
-
this
|
|
86
|
+
this.#browserVersion = inactiveBrowser.version();
|
|
87
87
|
inactiveBrowser.close().catch((error) => {
|
|
88
88
|
this.log.exception(error, 'Failed to close browser.');
|
|
89
89
|
});
|
|
90
90
|
}
|
|
91
|
-
|
|
91
|
+
const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
|
|
92
92
|
browserContext,
|
|
93
|
-
version: this
|
|
93
|
+
version: this.#browserVersion,
|
|
94
94
|
});
|
|
95
|
+
persistentBrowser.setBrowserType(this.library);
|
|
96
|
+
browser = persistentBrowser;
|
|
95
97
|
}
|
|
96
98
|
}
|
|
97
99
|
catch (error) {
|
|
@@ -100,13 +102,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
100
102
|
}
|
|
101
103
|
return browser;
|
|
102
104
|
}
|
|
103
|
-
|
|
104
|
-
this.
|
|
105
|
+
throwOnFailedLaunch(launchContext, cause) {
|
|
106
|
+
this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
|
|
105
107
|
}
|
|
106
108
|
createController() {
|
|
107
109
|
return new PlaywrightController(this);
|
|
108
110
|
}
|
|
109
|
-
async
|
|
111
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
110
112
|
launchContext.launchOptions ??= {};
|
|
111
113
|
const { launchOptions, proxyUrl } = launchContext;
|
|
112
114
|
if (proxyUrl) {
|
|
@@ -118,7 +120,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
118
120
|
};
|
|
119
121
|
}
|
|
120
122
|
}
|
|
121
|
-
|
|
123
|
+
isChromiumBasedBrowser() {
|
|
122
124
|
const name = this.library.name();
|
|
123
125
|
return name === 'chromium';
|
|
124
126
|
}
|
|
@@ -12,6 +12,6 @@ export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, Pup
|
|
|
12
12
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
13
13
|
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
|
14
14
|
createController(): PuppeteerController;
|
|
15
|
-
protected
|
|
16
|
-
protected
|
|
15
|
+
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
16
|
+
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
|
17
17
|
}
|
|
@@ -27,7 +27,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
27
27
|
const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
|
|
28
28
|
let browser;
|
|
29
29
|
if (this.remoteConnection) {
|
|
30
|
-
browser = await this.
|
|
30
|
+
browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
31
31
|
const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
|
|
32
32
|
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
33
33
|
return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
|
|
@@ -73,7 +73,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
73
73
|
}
|
|
74
74
|
catch (error) {
|
|
75
75
|
await close();
|
|
76
|
-
this.
|
|
76
|
+
this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -177,7 +177,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
177
177
|
createController() {
|
|
178
178
|
return new PuppeteerController(this);
|
|
179
179
|
}
|
|
180
|
-
async
|
|
180
|
+
async addProxyToLaunchOptions(_launchContext) {
|
|
181
181
|
/*
|
|
182
182
|
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
183
183
|
launchContext.launchOptions ??= {};
|
|
@@ -204,7 +204,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
204
204
|
}
|
|
205
205
|
*/
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
isChromiumBasedBrowser(_launchContext) {
|
|
208
208
|
return true;
|
|
209
209
|
}
|
|
210
210
|
}
|
package/remote-browser-pool.d.ts
CHANGED
|
@@ -95,10 +95,10 @@ export interface RemoteBrowserPoolOptions {
|
|
|
95
95
|
slotPollIntervalMillis?: number;
|
|
96
96
|
}
|
|
97
97
|
/**
|
|
98
|
-
* The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option
|
|
99
|
-
* {@link RemoteBrowserPoolOptions}
|
|
100
|
-
*
|
|
101
|
-
*
|
|
98
|
+
* The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the
|
|
99
|
+
* {@link RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its
|
|
100
|
+
* browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` — tuning the wrapping
|
|
101
|
+
* pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.
|
|
102
102
|
*/
|
|
103
103
|
export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
|
|
104
104
|
/**
|
|
@@ -134,15 +134,9 @@ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browse
|
|
|
134
134
|
* @category Browser management
|
|
135
135
|
*/
|
|
136
136
|
export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
|
|
137
|
+
#private;
|
|
137
138
|
/** The wrapped pool that performs the remote connections and serves pages. */
|
|
138
139
|
readonly browserPool: BrowserPool;
|
|
139
|
-
/** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
|
|
140
|
-
private readonly pool;
|
|
141
|
-
private readonly registry;
|
|
142
|
-
private readonly slotPollIntervalMillis;
|
|
143
|
-
private readonly log;
|
|
144
|
-
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
|
|
145
|
-
private _capacityChange?;
|
|
146
140
|
constructor(options: RemoteBrowserPoolOptions);
|
|
147
141
|
/** Maximum number of remote browsers that may be open at the same time. */
|
|
148
142
|
get maxOpenBrowsers(): number;
|
|
@@ -160,11 +154,11 @@ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<P
|
|
|
160
154
|
/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
|
|
161
155
|
destroy(): Promise<void>;
|
|
162
156
|
/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
|
|
163
|
-
private
|
|
157
|
+
private waitForFreeSlot;
|
|
164
158
|
/**
|
|
165
159
|
* Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
|
|
166
160
|
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
|
|
167
161
|
* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
|
|
168
162
|
*/
|
|
169
|
-
private
|
|
163
|
+
private nextCapacityChange;
|
|
170
164
|
}
|
package/remote-browser-pool.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { BrowserPool } from './browser-pool.js';
|
|
3
|
+
import { BROWSER_CONTROLLER_EVENTS, BROWSER_POOL_EVENTS } from './events.js';
|
|
3
4
|
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
4
5
|
/**
|
|
5
6
|
* Owns the lifecycle of remote browser sessions for a single {@link RemoteBrowserPool}: endpoint
|
|
@@ -7,18 +8,18 @@ import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
|
7
8
|
* {@link RemoteConnection} so it can be injected into a plugin.
|
|
8
9
|
*/
|
|
9
10
|
class RemoteSessionRegistry {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
#sessions = new Map();
|
|
12
|
+
#nextToken = 0;
|
|
13
|
+
#endpoint;
|
|
14
|
+
#onRelease;
|
|
15
|
+
#log;
|
|
15
16
|
constructor(endpoint, onRelease, log) {
|
|
16
|
-
this
|
|
17
|
-
this
|
|
18
|
-
this
|
|
17
|
+
this.#endpoint = endpoint;
|
|
18
|
+
this.#onRelease = onRelease;
|
|
19
|
+
this.#log = log;
|
|
19
20
|
}
|
|
20
21
|
async resolve(options) {
|
|
21
|
-
const resolved = typeof this
|
|
22
|
+
const resolved = typeof this.#endpoint === 'function' ? await this.#endpoint(options) : this.#endpoint;
|
|
22
23
|
let result;
|
|
23
24
|
if (typeof resolved === 'string') {
|
|
24
25
|
if (!resolved)
|
|
@@ -31,30 +32,30 @@ class RemoteSessionRegistry {
|
|
|
31
32
|
else {
|
|
32
33
|
result = resolved;
|
|
33
34
|
}
|
|
34
|
-
const token = this
|
|
35
|
-
this
|
|
35
|
+
const token = this.#nextToken++;
|
|
36
|
+
this.#sessions.set(token, { url: result.url, context: result.context, released: false });
|
|
36
37
|
return { url: result.url, token };
|
|
37
38
|
}
|
|
38
39
|
async release(token) {
|
|
39
|
-
const session = this
|
|
40
|
+
const session = this.#sessions.get(token);
|
|
40
41
|
// Release at most once per session — guards a close()/teardown race (the `released` flag is set
|
|
41
42
|
// synchronously before the awaited onRelease, so releaseAll() can't double-fire an in-flight release).
|
|
42
43
|
if (!session || session.released)
|
|
43
44
|
return;
|
|
44
45
|
session.released = true;
|
|
45
46
|
try {
|
|
46
|
-
await this
|
|
47
|
+
await this.#onRelease?.({ endpoint: session.url, context: session.context });
|
|
47
48
|
}
|
|
48
49
|
catch (err) {
|
|
49
|
-
this
|
|
50
|
+
this.#log.warning('Remote browser release() failed.', { error: err?.message });
|
|
50
51
|
}
|
|
51
52
|
finally {
|
|
52
|
-
this
|
|
53
|
+
this.#sessions.delete(token);
|
|
53
54
|
}
|
|
54
55
|
}
|
|
55
56
|
/** Releases every session that is still open. Called on pool teardown so no remote session leaks. */
|
|
56
57
|
async releaseAll() {
|
|
57
|
-
await Promise.all([...this
|
|
58
|
+
await Promise.all([...this.#sessions.keys()].map(async (token) => this.release(token)));
|
|
58
59
|
}
|
|
59
60
|
}
|
|
60
61
|
/**
|
|
@@ -92,16 +93,16 @@ export class RemoteBrowserPool {
|
|
|
92
93
|
/** The wrapped pool that performs the remote connections and serves pages. */
|
|
93
94
|
browserPool;
|
|
94
95
|
/** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
|
|
95
|
-
pool;
|
|
96
|
-
registry;
|
|
97
|
-
slotPollIntervalMillis;
|
|
98
|
-
log;
|
|
96
|
+
#pool;
|
|
97
|
+
#registry;
|
|
98
|
+
#slotPollIntervalMillis;
|
|
99
|
+
#log;
|
|
99
100
|
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
|
|
100
|
-
|
|
101
|
+
#capacityChange;
|
|
101
102
|
constructor(options) {
|
|
102
103
|
const { browserPlugins, endpoint, release, maxOpenBrowsers, connection = {}, browserPoolOptions = {}, slotPollIntervalMillis = 500, } = options;
|
|
103
|
-
this
|
|
104
|
-
this
|
|
104
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
|
|
105
|
+
this.#slotPollIntervalMillis = slotPollIntervalMillis;
|
|
105
106
|
// A RemoteBrowserProvider carries its own endpoint, release, and maxOpenBrowsers.
|
|
106
107
|
const provider = endpoint instanceof RemoteBrowserProvider ? endpoint : undefined;
|
|
107
108
|
const resolvedEndpoint = provider
|
|
@@ -111,20 +112,20 @@ export class RemoteBrowserPool {
|
|
|
111
112
|
? ({ context }) => provider.release(context)
|
|
112
113
|
: release;
|
|
113
114
|
const resolvedMax = maxOpenBrowsers ?? provider?.maxOpenBrowsers;
|
|
114
|
-
this
|
|
115
|
+
this.#registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.#log);
|
|
115
116
|
// Wire every plugin for remote connection.
|
|
116
117
|
for (const plugin of browserPlugins) {
|
|
117
|
-
plugin.useRemoteConnection(this
|
|
118
|
+
plugin.useRemoteConnection(this.#registry, connection);
|
|
118
119
|
}
|
|
119
120
|
this.browserPool = new BrowserPool({ ...browserPoolOptions, browserPlugins });
|
|
120
|
-
this
|
|
121
|
+
this.#pool = this.browserPool;
|
|
121
122
|
// Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed
|
|
122
123
|
// kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.
|
|
123
|
-
this.browserPool.on(
|
|
124
|
-
controller.once(
|
|
125
|
-
const token = controller.launchContext.
|
|
124
|
+
this.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (controller) => {
|
|
125
|
+
controller.once(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, () => {
|
|
126
|
+
const token = controller.launchContext.remoteToken;
|
|
126
127
|
if (token !== undefined)
|
|
127
|
-
void this
|
|
128
|
+
void this.#registry.release(token);
|
|
128
129
|
});
|
|
129
130
|
});
|
|
130
131
|
if (resolvedMax !== undefined) {
|
|
@@ -143,28 +144,28 @@ export class RemoteBrowserPool {
|
|
|
143
144
|
* allows it (either a new browser slot is free, or an active browser still has page capacity).
|
|
144
145
|
*/
|
|
145
146
|
async newPage(options) {
|
|
146
|
-
await this.
|
|
147
|
-
return this
|
|
147
|
+
await this.waitForFreeSlot();
|
|
148
|
+
return this.#pool.newPage(options);
|
|
148
149
|
}
|
|
149
150
|
async closePage(page, options) {
|
|
150
|
-
return this
|
|
151
|
+
return this.#pool.closePage(page, options);
|
|
151
152
|
}
|
|
152
153
|
async extractPageState(page) {
|
|
153
|
-
return this
|
|
154
|
+
return this.#pool.extractPageState(page);
|
|
154
155
|
}
|
|
155
156
|
async injectPageState(page, state) {
|
|
156
|
-
return this
|
|
157
|
+
return this.#pool.injectPageState(page, state);
|
|
157
158
|
}
|
|
158
159
|
/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
|
|
159
160
|
async destroy() {
|
|
160
161
|
await this.browserPool.destroy();
|
|
161
162
|
// Backstop: release any sessions whose browser never emitted a close (e.g. dropped on teardown).
|
|
162
|
-
await this
|
|
163
|
+
await this.#registry.releaseAll();
|
|
163
164
|
}
|
|
164
165
|
/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
|
|
165
|
-
async
|
|
166
|
+
async waitForFreeSlot() {
|
|
166
167
|
while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {
|
|
167
|
-
await this.
|
|
168
|
+
await this.nextCapacityChange();
|
|
168
169
|
}
|
|
169
170
|
}
|
|
170
171
|
/**
|
|
@@ -172,20 +173,20 @@ export class RemoteBrowserPool {
|
|
|
172
173
|
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
|
|
173
174
|
* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
|
|
174
175
|
*/
|
|
175
|
-
|
|
176
|
-
this
|
|
176
|
+
nextCapacityChange() {
|
|
177
|
+
this.#capacityChange ??= new Promise((resolve) => {
|
|
177
178
|
const done = () => {
|
|
178
179
|
clearTimeout(timer);
|
|
179
|
-
this.browserPool.off(
|
|
180
|
-
this.browserPool.off(
|
|
181
|
-
this
|
|
180
|
+
this.browserPool.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
181
|
+
this.browserPool.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
182
|
+
this.#capacityChange = undefined;
|
|
182
183
|
resolve();
|
|
183
184
|
};
|
|
184
|
-
const timer = setTimeout(done, this
|
|
185
|
+
const timer = setTimeout(done, this.#slotPollIntervalMillis);
|
|
185
186
|
timer.unref?.();
|
|
186
|
-
this.browserPool.once(
|
|
187
|
-
this.browserPool.once(
|
|
187
|
+
this.browserPool.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
188
|
+
this.browserPool.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
188
189
|
});
|
|
189
|
-
return this
|
|
190
|
+
return this.#capacityChange;
|
|
190
191
|
}
|
|
191
192
|
}
|