@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/abstract-classes/browser-controller.d.ts +31 -19
- package/abstract-classes/browser-controller.js +62 -13
- package/abstract-classes/browser-plugin.d.ts +11 -13
- package/abstract-classes/browser-plugin.js +13 -9
- package/anonymize-proxy.d.ts +1 -0
- package/browser-pool.d.ts +30 -41
- package/browser-pool.js +211 -147
- package/events.d.ts +2 -3
- package/events.js +0 -1
- package/fingerprinting/hooks.js +2 -1
- package/fingerprinting/types.d.ts +3 -33
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.js +5 -5
- package/index.d.ts +1 -1
- package/launch-context.d.ts +3 -3
- package/launch-context.js +11 -8
- package/package.json +9 -8
- package/playwright/playwright-browser.d.ts +3 -5
- package/playwright/playwright-browser.js +17 -16
- package/playwright/playwright-controller.d.ts +1 -0
- package/playwright/playwright-controller.js +4 -3
- package/playwright/playwright-plugin.d.ts +5 -4
- package/playwright/playwright-plugin.js +16 -12
- package/puppeteer/puppeteer-controller.d.ts +1 -0
- package/puppeteer/puppeteer-controller.js +13 -11
- package/puppeteer/puppeteer-plugin.d.ts +7 -3
- package/puppeteer/puppeteer-plugin.js +9 -5
- package/remote-browser-pool.d.ts +21 -21
- package/remote-browser-pool.js +63 -48
- package/container-proxy-server.d.ts +0 -11
- package/container-proxy-server.js +0 -42
package/fingerprinting/hooks.js
CHANGED
|
@@ -35,7 +35,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
35
35
|
if (cacheKey)
|
|
36
36
|
fingerprintCache?.set(cacheKey, fingerprint);
|
|
37
37
|
}
|
|
38
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
39
40
|
if (useIncognitoPages) {
|
|
40
41
|
return;
|
|
41
42
|
}
|
|
@@ -1,42 +1,13 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export interface FingerprintGenerator {
|
|
3
|
-
getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
|
|
4
|
-
}
|
|
5
|
-
export interface GetFingerprintReturn {
|
|
6
|
-
fingerprint: Fingerprint;
|
|
7
|
-
}
|
|
1
|
+
import type { FingerprintGeneratorOptions as FingerprintOptionsOriginal } from 'fingerprint-generator';
|
|
8
2
|
export interface FingerprintGeneratorOptions extends Partial<FingerprintOptionsOriginal> {
|
|
9
3
|
}
|
|
10
|
-
declare const SUPPORTED_HTTP_VERSIONS: readonly ["1", "2"];
|
|
11
|
-
/**
|
|
12
|
-
* String specifying the HTTP version to use.
|
|
13
|
-
*/
|
|
14
|
-
type HttpVersion = (typeof SUPPORTED_HTTP_VERSIONS)[number];
|
|
15
4
|
export declare enum BrowserName {
|
|
16
5
|
chrome = "chrome",
|
|
17
6
|
firefox = "firefox",
|
|
18
7
|
safari = "safari",
|
|
19
8
|
edge = "edge"
|
|
20
9
|
}
|
|
21
|
-
export
|
|
22
|
-
/**
|
|
23
|
-
* String representing the browser name.
|
|
24
|
-
*/
|
|
25
|
-
name: BrowserName;
|
|
26
|
-
/**
|
|
27
|
-
* Minimum version of browser used.
|
|
28
|
-
*/
|
|
29
|
-
minVersion?: number;
|
|
30
|
-
/**
|
|
31
|
-
* Maximum version of browser used.
|
|
32
|
-
*/
|
|
33
|
-
maxVersion?: number;
|
|
34
|
-
/**
|
|
35
|
-
* HTTP version to be used for header generation (the headers differ depending on the version).
|
|
36
|
-
*/
|
|
37
|
-
httpVersion?: HttpVersion;
|
|
38
|
-
}
|
|
39
|
-
export declare const enum OperatingSystemsName {
|
|
10
|
+
export declare enum OperatingSystemsName {
|
|
40
11
|
linux = "linux",
|
|
41
12
|
macos = "macos",
|
|
42
13
|
windows = "windows",
|
|
@@ -49,7 +20,7 @@ export declare const enum OperatingSystemsName {
|
|
|
49
20
|
*/
|
|
50
21
|
ios = "ios"
|
|
51
22
|
}
|
|
52
|
-
export declare
|
|
23
|
+
export declare enum DeviceCategory {
|
|
53
24
|
/**
|
|
54
25
|
* Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
|
|
55
26
|
* > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
|
|
@@ -60,4 +31,3 @@ export declare const enum DeviceCategory {
|
|
|
60
31
|
*/
|
|
61
32
|
desktop = "desktop"
|
|
62
33
|
}
|
|
63
|
-
export {};
|
package/fingerprinting/types.js
CHANGED
package/fingerprinting/utils.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
|
-
import { BrowserName } from './types.js';
|
|
3
|
+
import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
|
|
4
4
|
export const getGeneratorDefaultOptions = (launchContext) => {
|
|
5
5
|
const { browserPlugin, launchOptions } = launchContext;
|
|
6
6
|
const options = {
|
|
7
|
-
devices: [
|
|
7
|
+
devices: [DeviceCategory.desktop],
|
|
8
8
|
locales: ['en-US'],
|
|
9
9
|
browsers: [getBrowserName(browserPlugin, launchOptions)],
|
|
10
10
|
operatingSystems: [getOperatingSystem()],
|
|
@@ -34,11 +34,11 @@ const getOperatingSystem = () => {
|
|
|
34
34
|
switch (platform) {
|
|
35
35
|
case 'win32':
|
|
36
36
|
// platform is win32 even for 64-bit
|
|
37
|
-
return
|
|
37
|
+
return OperatingSystemsName.windows;
|
|
38
38
|
case 'darwin':
|
|
39
|
-
return
|
|
39
|
+
return OperatingSystemsName.macos;
|
|
40
40
|
default:
|
|
41
41
|
// consider everything else a linux
|
|
42
|
-
return
|
|
42
|
+
return OperatingSystemsName.linux;
|
|
43
43
|
}
|
|
44
44
|
};
|
package/index.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export * from './browser-pool.js';
|
|
|
28
28
|
export * from './playwright/playwright-plugin.js';
|
|
29
29
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
30
30
|
export * from './events.js';
|
|
31
|
-
export type {
|
|
31
|
+
export type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
|
|
32
32
|
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
33
33
|
export type { BrowserControllerEvents, IBrowserController, IBrowserLaunchContext, } from './abstract-classes/browser-controller.js';
|
|
34
34
|
export { BrowserController } from './abstract-classes/browser-controller.js';
|
package/launch-context.d.ts
CHANGED
|
@@ -52,10 +52,12 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
52
52
|
* Whether this launch context represents a connection to a remote browser
|
|
53
53
|
* rather than a locally launched one.
|
|
54
54
|
* @default false
|
|
55
|
+
* @internal
|
|
55
56
|
*/
|
|
56
57
|
isRemote?: boolean;
|
|
57
58
|
}
|
|
58
59
|
export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
60
|
+
#private;
|
|
59
61
|
id?: string;
|
|
60
62
|
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
61
63
|
launchOptions: LibraryOptions;
|
|
@@ -64,15 +66,13 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
64
66
|
userDataDir: string;
|
|
65
67
|
readonly isRemote: boolean;
|
|
66
68
|
ignoreProxyCertificate?: boolean;
|
|
67
|
-
private _proxyUrl?;
|
|
68
|
-
private readonly _reservedFieldNames;
|
|
69
69
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
70
70
|
/**
|
|
71
71
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
72
72
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
73
73
|
* @internal
|
|
74
74
|
*/
|
|
75
|
-
|
|
75
|
+
remoteToken?: number;
|
|
76
76
|
[K: PropertyKey]: unknown;
|
|
77
77
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
78
78
|
/**
|
package/launch-context.js
CHANGED
|
@@ -7,15 +7,15 @@ export class LaunchContext {
|
|
|
7
7
|
userDataDir;
|
|
8
8
|
isRemote;
|
|
9
9
|
ignoreProxyCertificate;
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
#proxyUrl;
|
|
11
|
+
#reservedFieldNames;
|
|
12
12
|
fingerprint;
|
|
13
13
|
/**
|
|
14
14
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
15
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
16
|
* @internal
|
|
17
17
|
*/
|
|
18
|
-
|
|
18
|
+
remoteToken;
|
|
19
19
|
constructor(options) {
|
|
20
20
|
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
21
21
|
this.id = id;
|
|
@@ -26,7 +26,10 @@ export class LaunchContext {
|
|
|
26
26
|
this.userDataDir = userDataDir;
|
|
27
27
|
this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
|
|
28
28
|
this.isRemote = isRemote ?? false;
|
|
29
|
-
this
|
|
29
|
+
this.#proxyUrl = proxyUrl;
|
|
30
|
+
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
|
+
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* Extend the launch context with any extra fields.
|
|
@@ -37,7 +40,7 @@ export class LaunchContext {
|
|
|
37
40
|
*/
|
|
38
41
|
extend(fields) {
|
|
39
42
|
Object.entries(fields).forEach(([key, value]) => {
|
|
40
|
-
if (this.
|
|
43
|
+
if (this.#reservedFieldNames.includes(key)) {
|
|
41
44
|
throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
|
|
42
45
|
}
|
|
43
46
|
else {
|
|
@@ -51,7 +54,7 @@ export class LaunchContext {
|
|
|
51
54
|
*/
|
|
52
55
|
set proxyUrl(url) {
|
|
53
56
|
if (!url) {
|
|
54
|
-
this
|
|
57
|
+
this.#proxyUrl = undefined;
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
60
|
const urlInstance = new URL(url);
|
|
@@ -59,12 +62,12 @@ export class LaunchContext {
|
|
|
59
62
|
urlInstance.search = '';
|
|
60
63
|
urlInstance.hash = '';
|
|
61
64
|
// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
|
|
62
|
-
this
|
|
65
|
+
this.#proxyUrl = urlInstance.href.slice(0, -1);
|
|
63
66
|
}
|
|
64
67
|
/**
|
|
65
68
|
* Returns the proxy URL of the browser.
|
|
66
69
|
*/
|
|
67
70
|
get proxyUrl() {
|
|
68
|
-
return this
|
|
71
|
+
return this.#proxyUrl;
|
|
69
72
|
}
|
|
70
73
|
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-
|
|
3
|
+
"version": "4.0.0-rc.1",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -30,19 +30,20 @@
|
|
|
30
30
|
"copy": "tsx ../../scripts/copy.ts"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@apify/timeout": "^0.
|
|
34
|
-
"@crawlee/core": "4.0.0-
|
|
35
|
-
"@crawlee/types": "4.0.0-
|
|
33
|
+
"@apify/timeout": "^1.0.1",
|
|
34
|
+
"@crawlee/core": "4.0.0-rc.1",
|
|
35
|
+
"@crawlee/types": "4.0.0-rc.1",
|
|
36
|
+
"@crawlee/utils": "4.0.0-rc.1",
|
|
36
37
|
"fingerprint-generator": "^2.1.68",
|
|
37
38
|
"fingerprint-injector": "^2.1.68",
|
|
38
39
|
"lodash.merge": "^4.6.2",
|
|
39
40
|
"nanoid": "^5.1.5",
|
|
40
|
-
"ow": "^2.0.0",
|
|
41
41
|
"p-limit": "^6.2.0",
|
|
42
42
|
"proxy-chain": "^2.5.8",
|
|
43
43
|
"quick-lru": "^7.0.1",
|
|
44
44
|
"tiny-typed-emitter": "^2.1.0",
|
|
45
|
-
"tslib": "^2.8.1"
|
|
45
|
+
"tslib": "^2.8.1",
|
|
46
|
+
"zod": "^4.5.4"
|
|
46
47
|
},
|
|
47
48
|
"peerDependencies": {
|
|
48
49
|
"playwright": "*",
|
|
@@ -63,5 +64,5 @@
|
|
|
63
64
|
}
|
|
64
65
|
}
|
|
65
66
|
},
|
|
66
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
|
|
67
68
|
}
|
|
@@ -9,10 +9,8 @@ export interface BrowserOptions {
|
|
|
9
9
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
10
10
|
*/
|
|
11
11
|
export declare class PlaywrightBrowser extends EventEmitter {
|
|
12
|
-
private
|
|
13
|
-
|
|
14
|
-
private _isConnected;
|
|
15
|
-
private _browserType?;
|
|
12
|
+
#private;
|
|
13
|
+
/** @internal */
|
|
16
14
|
constructor(options: BrowserOptions);
|
|
17
15
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
16
|
close(): Promise<void>;
|
|
@@ -20,7 +18,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
20
18
|
isConnected(): boolean;
|
|
21
19
|
version(): string;
|
|
22
20
|
/** @internal */
|
|
23
|
-
|
|
21
|
+
setBrowserType(browserType: BrowserType): void;
|
|
24
22
|
browserType(): BrowserType;
|
|
25
23
|
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
|
26
24
|
newContext(): Promise<never>;
|
|
@@ -3,17 +3,18 @@ import { EventEmitter } from 'node:events';
|
|
|
3
3
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
4
4
|
*/
|
|
5
5
|
export class PlaywrightBrowser extends EventEmitter {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
#browserContext;
|
|
7
|
+
#version;
|
|
8
|
+
#isConnected = true;
|
|
9
|
+
#browserType;
|
|
10
|
+
/** @internal */
|
|
10
11
|
constructor(options) {
|
|
11
12
|
super();
|
|
12
13
|
const { browserContext, version } = options;
|
|
13
|
-
this
|
|
14
|
-
this
|
|
15
|
-
this.
|
|
16
|
-
this
|
|
14
|
+
this.#browserContext = browserContext;
|
|
15
|
+
this.#version = version;
|
|
16
|
+
this.#browserContext.once('close', () => {
|
|
17
|
+
this.#isConnected = false;
|
|
17
18
|
this.emit('disconnected');
|
|
18
19
|
});
|
|
19
20
|
}
|
|
@@ -21,26 +22,26 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
21
22
|
await this.close();
|
|
22
23
|
}
|
|
23
24
|
async close() {
|
|
24
|
-
await this.
|
|
25
|
+
await this.#browserContext.close();
|
|
25
26
|
}
|
|
26
27
|
contexts() {
|
|
27
|
-
return [this
|
|
28
|
+
return [this.#browserContext];
|
|
28
29
|
}
|
|
29
30
|
isConnected() {
|
|
30
|
-
return this
|
|
31
|
+
return this.#isConnected;
|
|
31
32
|
}
|
|
32
33
|
version() {
|
|
33
|
-
return this
|
|
34
|
+
return this.#version;
|
|
34
35
|
}
|
|
35
36
|
/** @internal */
|
|
36
|
-
|
|
37
|
-
this
|
|
37
|
+
setBrowserType(browserType) {
|
|
38
|
+
this.#browserType = browserType;
|
|
38
39
|
}
|
|
39
40
|
browserType() {
|
|
40
|
-
return this
|
|
41
|
+
return this.#browserType;
|
|
41
42
|
}
|
|
42
43
|
async newPage(...args) {
|
|
43
|
-
return this.
|
|
44
|
+
return this.#browserContext.newPage(...args);
|
|
44
45
|
}
|
|
45
46
|
async newContext() {
|
|
46
47
|
throw new Error('Function `newContext()` is not available in incognito mode');
|
|
@@ -4,6 +4,7 @@ import type { Browser, BrowserType, Page } from 'playwright';
|
|
|
4
4
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
5
5
|
import type { SafeParameters } from '../utils.js';
|
|
6
6
|
export declare class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
|
|
7
|
+
/** @internal */
|
|
7
8
|
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
8
9
|
protected _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page>;
|
|
9
10
|
protected _close(): Promise<void>;
|
|
@@ -2,6 +2,7 @@ import { tryCancel } from '@apify/timeout';
|
|
|
2
2
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
4
|
export class PlaywrightController extends BrowserController {
|
|
5
|
+
/** @internal */
|
|
5
6
|
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
6
7
|
if (!proxyUrl) {
|
|
7
8
|
return {};
|
|
@@ -46,9 +47,9 @@ export class PlaywrightController extends BrowserController {
|
|
|
46
47
|
}
|
|
47
48
|
try {
|
|
48
49
|
const page = await this.browser.newPage(contextOptions);
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
this.registerPageTeardown(page, close);
|
|
51
|
+
page.once('close', () => {
|
|
52
|
+
this.registerPageClosed(page);
|
|
52
53
|
});
|
|
53
54
|
tryCancel();
|
|
54
55
|
return page;
|
|
@@ -6,15 +6,16 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
|
|
|
6
6
|
import type { SafeParameters } from '../utils.js';
|
|
7
7
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
8
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
|
-
private
|
|
9
|
+
#private;
|
|
10
10
|
/**
|
|
11
11
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
12
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
|
+
*
|
|
14
|
+
* @internal
|
|
13
15
|
*/
|
|
14
16
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
15
17
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
16
|
-
private _throwOnFailedLaunch;
|
|
17
18
|
createController(): PlaywrightController;
|
|
18
|
-
protected
|
|
19
|
-
protected
|
|
19
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
20
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
20
21
|
}
|
|
@@ -5,10 +5,12 @@ import { getLocalProxyAddress } from '../proxy-server.js';
|
|
|
5
5
|
import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
|
|
6
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
7
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
8
|
-
|
|
8
|
+
#browserVersion;
|
|
9
9
|
/**
|
|
10
10
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
11
11
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
12
|
+
*
|
|
13
|
+
* @internal
|
|
12
14
|
*/
|
|
13
15
|
useRemoteConnection(connection, parameters = {}) {
|
|
14
16
|
super.useRemoteConnection(connection, parameters);
|
|
@@ -20,7 +22,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
20
22
|
}
|
|
21
23
|
async _launch(launchContext) {
|
|
22
24
|
if (this.remoteConnection) {
|
|
23
|
-
return this.
|
|
25
|
+
return this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
24
26
|
const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
|
|
25
27
|
if (this.remoteConnectionParameters?.protocol === 'playwright') {
|
|
26
28
|
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
@@ -53,7 +55,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
53
55
|
try {
|
|
54
56
|
if (useIncognitoPages) {
|
|
55
57
|
browser = await this.library.launch(launchOptions).catch((error) => {
|
|
56
|
-
return this
|
|
58
|
+
return this.#throwOnFailedLaunch(launchContext, error);
|
|
57
59
|
});
|
|
58
60
|
if (anonymizedProxyUrl) {
|
|
59
61
|
browser.on('disconnected', async () => {
|
|
@@ -65,7 +67,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
65
67
|
const browserContext = await this.library
|
|
66
68
|
.launchPersistentContext(userDataDir, launchOptions)
|
|
67
69
|
.catch((error) => {
|
|
68
|
-
return this
|
|
70
|
+
return this.#throwOnFailedLaunch(launchContext, error);
|
|
69
71
|
});
|
|
70
72
|
browserContext.once('close', () => {
|
|
71
73
|
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
|
|
@@ -80,18 +82,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
80
82
|
await close();
|
|
81
83
|
});
|
|
82
84
|
}
|
|
83
|
-
if (!this
|
|
85
|
+
if (!this.#browserVersion) {
|
|
84
86
|
// Launches unused browser just to get the browser version.
|
|
85
87
|
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
86
|
-
this
|
|
88
|
+
this.#browserVersion = inactiveBrowser.version();
|
|
87
89
|
inactiveBrowser.close().catch((error) => {
|
|
88
90
|
this.log.exception(error, 'Failed to close browser.');
|
|
89
91
|
});
|
|
90
92
|
}
|
|
91
|
-
|
|
93
|
+
const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
|
|
92
94
|
browserContext,
|
|
93
|
-
version: this
|
|
95
|
+
version: this.#browserVersion,
|
|
94
96
|
});
|
|
97
|
+
persistentBrowser.setBrowserType(this.library);
|
|
98
|
+
browser = persistentBrowser;
|
|
95
99
|
}
|
|
96
100
|
}
|
|
97
101
|
catch (error) {
|
|
@@ -100,13 +104,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
100
104
|
}
|
|
101
105
|
return browser;
|
|
102
106
|
}
|
|
103
|
-
|
|
104
|
-
this.
|
|
107
|
+
#throwOnFailedLaunch(launchContext, cause) {
|
|
108
|
+
this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
|
|
105
109
|
}
|
|
106
110
|
createController() {
|
|
107
111
|
return new PlaywrightController(this);
|
|
108
112
|
}
|
|
109
|
-
async
|
|
113
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
110
114
|
launchContext.launchOptions ??= {};
|
|
111
115
|
const { launchOptions, proxyUrl } = launchContext;
|
|
112
116
|
if (proxyUrl) {
|
|
@@ -118,7 +122,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
118
122
|
};
|
|
119
123
|
}
|
|
120
124
|
}
|
|
121
|
-
|
|
125
|
+
isChromiumBasedBrowser() {
|
|
122
126
|
const name = this.library.name();
|
|
123
127
|
return name === 'chromium';
|
|
124
128
|
}
|
|
@@ -9,6 +9,7 @@ export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOp
|
|
|
9
9
|
proxyPassword?: string;
|
|
10
10
|
}
|
|
11
11
|
export declare class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
12
|
+
/** @internal */
|
|
12
13
|
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
13
14
|
protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
|
|
14
15
|
protected _close(): Promise<void>;
|
|
@@ -3,6 +3,7 @@ import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
4
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
5
5
|
export class PuppeteerController extends BrowserController {
|
|
6
|
+
/** @internal */
|
|
6
7
|
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
7
8
|
if (!proxyUrl) {
|
|
8
9
|
return {};
|
|
@@ -51,17 +52,18 @@ export class PuppeteerController extends BrowserController {
|
|
|
51
52
|
tryCancel();
|
|
52
53
|
}
|
|
53
54
|
*/
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
catch (error) {
|
|
55
|
+
this.registerPageTeardown(page, async () => {
|
|
56
|
+
// The proxy server is not chained behind the context close: that can hang on
|
|
57
|
+
// the same target the page hung on, and the proxy runs in this process, so it
|
|
58
|
+
// has to be reclaimed either way.
|
|
59
|
+
const contextClosed = context.close().catch((error) => {
|
|
60
60
|
this.log.exception(error, 'Failed to close context.');
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
});
|
|
62
|
+
await close();
|
|
63
|
+
await contextClosed;
|
|
64
|
+
});
|
|
65
|
+
page.once('close', () => {
|
|
66
|
+
this.registerPageClosed(page);
|
|
65
67
|
});
|
|
66
68
|
return page;
|
|
67
69
|
}
|
|
@@ -73,7 +75,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
73
75
|
const page = await this.browser.newPage();
|
|
74
76
|
tryCancel();
|
|
75
77
|
page.once('close', () => {
|
|
76
|
-
this.
|
|
78
|
+
this.registerPageClosed(page);
|
|
77
79
|
});
|
|
78
80
|
return page;
|
|
79
81
|
}
|
|
@@ -8,10 +8,14 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
|
|
|
8
8
|
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
9
9
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
10
10
|
export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
|
|
13
|
+
*
|
|
14
|
+
* @internal
|
|
15
|
+
*/
|
|
12
16
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
13
17
|
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
|
14
18
|
createController(): PuppeteerController;
|
|
15
|
-
protected
|
|
16
|
-
protected
|
|
19
|
+
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
20
|
+
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
|
17
21
|
}
|
|
@@ -5,7 +5,11 @@ import { noop } from '../utils.js';
|
|
|
5
5
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
6
6
|
const PROXY_SERVER_ARG = '--proxy-server=';
|
|
7
7
|
export class PuppeteerPlugin extends BrowserPlugin {
|
|
8
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
|
|
10
|
+
*
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
9
13
|
useRemoteConnection(connection, parameters = {}) {
|
|
10
14
|
super.useRemoteConnection(connection, parameters);
|
|
11
15
|
if (!this.useIncognitoPages) {
|
|
@@ -27,7 +31,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
27
31
|
const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
|
|
28
32
|
let browser;
|
|
29
33
|
if (this.remoteConnection) {
|
|
30
|
-
browser = await this.
|
|
34
|
+
browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
31
35
|
const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
|
|
32
36
|
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
33
37
|
return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
|
|
@@ -73,7 +77,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
73
77
|
}
|
|
74
78
|
catch (error) {
|
|
75
79
|
await close();
|
|
76
|
-
this.
|
|
80
|
+
this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
|
|
77
81
|
}
|
|
78
82
|
}
|
|
79
83
|
}
|
|
@@ -177,7 +181,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
177
181
|
createController() {
|
|
178
182
|
return new PuppeteerController(this);
|
|
179
183
|
}
|
|
180
|
-
async
|
|
184
|
+
async addProxyToLaunchOptions(_launchContext) {
|
|
181
185
|
/*
|
|
182
186
|
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
183
187
|
launchContext.launchOptions ??= {};
|
|
@@ -204,7 +208,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
204
208
|
}
|
|
205
209
|
*/
|
|
206
210
|
}
|
|
207
|
-
|
|
211
|
+
isChromiumBasedBrowser(_launchContext) {
|
|
208
212
|
return true;
|
|
209
213
|
}
|
|
210
214
|
}
|