@crawlee/browser-pool 4.0.0-beta.12 → 4.0.0-beta.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/abstract-classes/browser-controller.d.ts +70 -13
- package/abstract-classes/browser-controller.js +21 -22
- package/abstract-classes/browser-plugin.d.ts +48 -15
- package/abstract-classes/browser-plugin.js +69 -11
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +100 -28
- package/browser-pool.js +193 -65
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +2 -3
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +2 -3
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +20 -5
- package/launch-context.js +21 -10
- package/package.json +8 -9
- package/playwright/playwright-browser.d.ts +2 -6
- package/playwright/playwright-browser.js +16 -17
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +20 -10
- package/playwright/playwright-plugin.d.ts +10 -7
- package/playwright/playwright-plugin.js +40 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +5 -3
- package/puppeteer/puppeteer-plugin.js +83 -46
- package/remote-browser-pool.d.ts +164 -0
- package/remote-browser-pool.js +192 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/fingerprinting/utils.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
|
-
import { BrowserName } from './types.js';
|
|
3
|
+
import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
|
|
4
4
|
export const getGeneratorDefaultOptions = (launchContext) => {
|
|
5
5
|
const { browserPlugin, launchOptions } = launchContext;
|
|
6
6
|
const options = {
|
|
7
|
-
devices: [
|
|
7
|
+
devices: [DeviceCategory.desktop],
|
|
8
8
|
locales: ['en-US'],
|
|
9
9
|
browsers: [getBrowserName(browserPlugin, launchOptions)],
|
|
10
10
|
operatingSystems: [getOperatingSystem()],
|
|
@@ -34,12 +34,11 @@ const getOperatingSystem = () => {
|
|
|
34
34
|
switch (platform) {
|
|
35
35
|
case 'win32':
|
|
36
36
|
// platform is win32 even for 64-bit
|
|
37
|
-
return
|
|
37
|
+
return OperatingSystemsName.windows;
|
|
38
38
|
case 'darwin':
|
|
39
|
-
return
|
|
39
|
+
return OperatingSystemsName.macos;
|
|
40
40
|
default:
|
|
41
41
|
// consider everything else a linux
|
|
42
|
-
return
|
|
42
|
+
return OperatingSystemsName.linux;
|
|
43
43
|
}
|
|
44
44
|
};
|
|
45
|
-
//# sourceMappingURL=utils.js.map
|
package/index.d.ts
CHANGED
|
@@ -28,13 +28,20 @@ export * from './browser-pool.js';
|
|
|
28
28
|
export * from './playwright/playwright-plugin.js';
|
|
29
29
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
30
30
|
export * from './events.js';
|
|
31
|
-
export {
|
|
32
|
-
export {
|
|
31
|
+
export type { BrowserSpecification, FingerprintGenerator, FingerprintGeneratorOptions, GetFingerprintReturn, } from './fingerprinting/types.js';
|
|
32
|
+
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
33
|
+
export type { BrowserControllerEvents, IBrowserController, IBrowserLaunchContext, } from './abstract-classes/browser-controller.js';
|
|
34
|
+
export { BrowserController } from './abstract-classes/browser-controller.js';
|
|
33
35
|
export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
|
|
34
36
|
export { PlaywrightController } from './playwright/playwright-controller.js';
|
|
35
37
|
export { PlaywrightBrowser } from './playwright/playwright-browser.js';
|
|
36
|
-
export { CommonPage, CommonLibrary,
|
|
37
|
-
export {
|
|
38
|
-
export {
|
|
39
|
-
export {
|
|
40
|
-
|
|
38
|
+
export type { CommonPage, CommonLibrary, BrowserPluginOptions, CreateLaunchContextOptions, } from './abstract-classes/browser-plugin.js';
|
|
39
|
+
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
|
|
40
|
+
export type { LaunchContextOptions } from './launch-context.js';
|
|
41
|
+
export { LaunchContext } from './launch-context.js';
|
|
42
|
+
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
43
|
+
export { RemoteBrowserPool } from './remote-browser-pool.js';
|
|
44
|
+
export type { RemoteBrowserPoolOptions, CrawlerRemoteBrowserOptions, RemoteBrowserEndpoint, ResolvedRemoteEndpoint, RemoteConnection, RemoteConnectionParameters, } from './remote-browser-pool.js';
|
|
45
|
+
export type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
|
|
46
|
+
export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy.js';
|
|
47
|
+
export type { IBrowserPool, NewPageOptions } from '@crawlee/types';
|
package/index.js
CHANGED
|
@@ -26,11 +26,13 @@ export * from './browser-pool.js';
|
|
|
26
26
|
export * from './playwright/playwright-plugin.js';
|
|
27
27
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
28
28
|
export * from './events.js';
|
|
29
|
-
export { BrowserName, DeviceCategory, OperatingSystemsName
|
|
29
|
+
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
30
30
|
export { BrowserController } from './abstract-classes/browser-controller.js';
|
|
31
31
|
export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
|
|
32
32
|
export { PlaywrightController } from './playwright/playwright-controller.js';
|
|
33
33
|
export { PlaywrightBrowser } from './playwright/playwright-browser.js';
|
|
34
|
-
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT
|
|
34
|
+
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
|
|
35
35
|
export { LaunchContext } from './launch-context.js';
|
|
36
|
-
|
|
36
|
+
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
37
|
+
export { RemoteBrowserPool } from './remote-browser-pool.js';
|
|
38
|
+
export { anonymizeProxySugar } from './anonymize-proxy.js';
|
package/launch-context.d.ts
CHANGED
|
@@ -43,19 +43,35 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
43
43
|
*/
|
|
44
44
|
userDataDir?: string;
|
|
45
45
|
proxyUrl?: string;
|
|
46
|
-
|
|
46
|
+
/**
|
|
47
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
48
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
49
|
+
*/
|
|
50
|
+
ignoreProxyCertificate?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Whether this launch context represents a connection to a remote browser
|
|
53
|
+
* rather than a locally launched one.
|
|
54
|
+
* @default false
|
|
55
|
+
*/
|
|
56
|
+
isRemote?: boolean;
|
|
47
57
|
}
|
|
48
58
|
export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
59
|
+
#private;
|
|
49
60
|
id?: string;
|
|
50
61
|
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
51
62
|
launchOptions: LibraryOptions;
|
|
52
63
|
useIncognitoPages: boolean;
|
|
53
64
|
browserPerProxy?: boolean;
|
|
54
65
|
userDataDir: string;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
private readonly _reservedFieldNames;
|
|
66
|
+
readonly isRemote: boolean;
|
|
67
|
+
ignoreProxyCertificate?: boolean;
|
|
58
68
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
69
|
+
/**
|
|
70
|
+
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
71
|
+
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
72
|
+
* @internal
|
|
73
|
+
*/
|
|
74
|
+
remoteToken?: number;
|
|
59
75
|
[K: PropertyKey]: unknown;
|
|
60
76
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
61
77
|
/**
|
|
@@ -76,4 +92,3 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
76
92
|
*/
|
|
77
93
|
get proxyUrl(): string | undefined;
|
|
78
94
|
}
|
|
79
|
-
//# sourceMappingURL=launch-context.d.ts.map
|
package/launch-context.js
CHANGED
|
@@ -5,20 +5,31 @@ export class LaunchContext {
|
|
|
5
5
|
useIncognitoPages;
|
|
6
6
|
browserPerProxy;
|
|
7
7
|
userDataDir;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
isRemote;
|
|
9
|
+
ignoreProxyCertificate;
|
|
10
|
+
#proxyUrl;
|
|
11
|
+
#reservedFieldNames;
|
|
11
12
|
fingerprint;
|
|
13
|
+
/**
|
|
14
|
+
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
|
+
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
remoteToken;
|
|
12
19
|
constructor(options) {
|
|
13
|
-
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '',
|
|
20
|
+
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
14
21
|
this.id = id;
|
|
15
22
|
this.browserPlugin = browserPlugin;
|
|
16
23
|
this.launchOptions = launchOptions;
|
|
17
24
|
this.browserPerProxy = browserPerProxy ?? false;
|
|
18
25
|
this.useIncognitoPages = useIncognitoPages ?? false;
|
|
19
26
|
this.userDataDir = userDataDir;
|
|
20
|
-
this.
|
|
21
|
-
this.
|
|
27
|
+
this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
|
|
28
|
+
this.isRemote = isRemote ?? false;
|
|
29
|
+
this.#proxyUrl = proxyUrl;
|
|
30
|
+
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
|
+
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
|
|
22
33
|
}
|
|
23
34
|
/**
|
|
24
35
|
* Extend the launch context with any extra fields.
|
|
@@ -29,7 +40,7 @@ export class LaunchContext {
|
|
|
29
40
|
*/
|
|
30
41
|
extend(fields) {
|
|
31
42
|
Object.entries(fields).forEach(([key, value]) => {
|
|
32
|
-
if (this.
|
|
43
|
+
if (this.#reservedFieldNames.includes(key)) {
|
|
33
44
|
throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
|
|
34
45
|
}
|
|
35
46
|
else {
|
|
@@ -43,6 +54,7 @@ export class LaunchContext {
|
|
|
43
54
|
*/
|
|
44
55
|
set proxyUrl(url) {
|
|
45
56
|
if (!url) {
|
|
57
|
+
this.#proxyUrl = undefined;
|
|
46
58
|
return;
|
|
47
59
|
}
|
|
48
60
|
const urlInstance = new URL(url);
|
|
@@ -50,13 +62,12 @@ export class LaunchContext {
|
|
|
50
62
|
urlInstance.search = '';
|
|
51
63
|
urlInstance.hash = '';
|
|
52
64
|
// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
|
|
53
|
-
this
|
|
65
|
+
this.#proxyUrl = urlInstance.href.slice(0, -1);
|
|
54
66
|
}
|
|
55
67
|
/**
|
|
56
68
|
* Returns the proxy URL of the browser.
|
|
57
69
|
*/
|
|
58
70
|
get proxyUrl() {
|
|
59
|
-
return this
|
|
71
|
+
return this.#proxyUrl;
|
|
60
72
|
}
|
|
61
73
|
}
|
|
62
|
-
//# sourceMappingURL=launch-context.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.121",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -24,18 +24,17 @@
|
|
|
24
24
|
"url": "https://github.com/apify/crawlee/issues"
|
|
25
25
|
},
|
|
26
26
|
"scripts": {
|
|
27
|
-
"build": "
|
|
27
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
28
28
|
"clean": "rimraf ./dist",
|
|
29
29
|
"compile": "tsc -p tsconfig.build.json",
|
|
30
30
|
"copy": "tsx ../../scripts/copy.ts"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@apify/
|
|
34
|
-
"@
|
|
35
|
-
"@crawlee/
|
|
36
|
-
"
|
|
37
|
-
"fingerprint-
|
|
38
|
-
"fingerprint-injector": "^2.1.66",
|
|
33
|
+
"@apify/timeout": "^0.4.4",
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.121",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.121",
|
|
36
|
+
"fingerprint-generator": "^2.1.68",
|
|
37
|
+
"fingerprint-injector": "^2.1.68",
|
|
39
38
|
"lodash.merge": "^4.6.2",
|
|
40
39
|
"nanoid": "^5.1.5",
|
|
41
40
|
"ow": "^2.0.0",
|
|
@@ -64,5 +63,5 @@
|
|
|
64
63
|
}
|
|
65
64
|
}
|
|
66
65
|
},
|
|
67
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
|
|
68
67
|
}
|
|
@@ -9,10 +9,7 @@ export interface BrowserOptions {
|
|
|
9
9
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
10
10
|
*/
|
|
11
11
|
export declare class PlaywrightBrowser extends EventEmitter {
|
|
12
|
-
private
|
|
13
|
-
private _version;
|
|
14
|
-
private _isConnected;
|
|
15
|
-
private _browserType?;
|
|
12
|
+
#private;
|
|
16
13
|
constructor(options: BrowserOptions);
|
|
17
14
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
15
|
close(): Promise<void>;
|
|
@@ -20,7 +17,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
20
17
|
isConnected(): boolean;
|
|
21
18
|
version(): string;
|
|
22
19
|
/** @internal */
|
|
23
|
-
|
|
20
|
+
setBrowserType(browserType: BrowserType): void;
|
|
24
21
|
browserType(): BrowserType;
|
|
25
22
|
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
|
26
23
|
newContext(): Promise<never>;
|
|
@@ -28,4 +25,3 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
28
25
|
startTracing(): Promise<never>;
|
|
29
26
|
stopTracing(): Promise<never>;
|
|
30
27
|
}
|
|
31
|
-
//# sourceMappingURL=playwright-browser.d.ts.map
|
|
@@ -3,17 +3,17 @@ import { EventEmitter } from 'node:events';
|
|
|
3
3
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
4
4
|
*/
|
|
5
5
|
export class PlaywrightBrowser extends EventEmitter {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
#browserContext;
|
|
7
|
+
#version;
|
|
8
|
+
#isConnected = true;
|
|
9
|
+
#browserType;
|
|
10
10
|
constructor(options) {
|
|
11
11
|
super();
|
|
12
12
|
const { browserContext, version } = options;
|
|
13
|
-
this
|
|
14
|
-
this
|
|
15
|
-
this.
|
|
16
|
-
this
|
|
13
|
+
this.#browserContext = browserContext;
|
|
14
|
+
this.#version = version;
|
|
15
|
+
this.#browserContext.once('close', () => {
|
|
16
|
+
this.#isConnected = false;
|
|
17
17
|
this.emit('disconnected');
|
|
18
18
|
});
|
|
19
19
|
}
|
|
@@ -21,26 +21,26 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
21
21
|
await this.close();
|
|
22
22
|
}
|
|
23
23
|
async close() {
|
|
24
|
-
await this.
|
|
24
|
+
await this.#browserContext.close();
|
|
25
25
|
}
|
|
26
26
|
contexts() {
|
|
27
|
-
return [this
|
|
27
|
+
return [this.#browserContext];
|
|
28
28
|
}
|
|
29
29
|
isConnected() {
|
|
30
|
-
return this
|
|
30
|
+
return this.#isConnected;
|
|
31
31
|
}
|
|
32
32
|
version() {
|
|
33
|
-
return this
|
|
33
|
+
return this.#version;
|
|
34
34
|
}
|
|
35
35
|
/** @internal */
|
|
36
|
-
|
|
37
|
-
this
|
|
36
|
+
setBrowserType(browserType) {
|
|
37
|
+
this.#browserType = browserType;
|
|
38
38
|
}
|
|
39
39
|
browserType() {
|
|
40
|
-
return this
|
|
40
|
+
return this.#browserType;
|
|
41
41
|
}
|
|
42
42
|
async newPage(...args) {
|
|
43
|
-
return this.
|
|
43
|
+
return this.#browserContext.newPage(...args);
|
|
44
44
|
}
|
|
45
45
|
async newContext() {
|
|
46
46
|
throw new Error('Function `newContext()` is not available in incognito mode');
|
|
@@ -55,4 +55,3 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
55
55
|
throw new Error('Function `stopTracing()` is not available in incognito mode');
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
|
-
//# sourceMappingURL=playwright-browser.js.map
|
|
@@ -11,7 +11,7 @@ export class PlaywrightController extends BrowserController {
|
|
|
11
11
|
const password = decodeURIComponent(url.password);
|
|
12
12
|
return {
|
|
13
13
|
proxy: {
|
|
14
|
-
server: url.
|
|
14
|
+
server: `${url.protocol}//${url.host}`,
|
|
15
15
|
username,
|
|
16
16
|
password,
|
|
17
17
|
bypass: pageOptions?.proxy?.bypass,
|
|
@@ -23,15 +23,26 @@ export class PlaywrightController extends BrowserController {
|
|
|
23
23
|
throw new Error('A new page can be created with provided context only when using incognito pages.');
|
|
24
24
|
}
|
|
25
25
|
let close = async () => { };
|
|
26
|
-
if (this.launchContext.useIncognitoPages
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
26
|
+
if (this.launchContext.useIncognitoPages) {
|
|
27
|
+
// Each page requires to have all the context options applied
|
|
28
|
+
contextOptions = {
|
|
29
|
+
...this.launchContext.launchOptions,
|
|
30
|
+
...contextOptions,
|
|
31
|
+
};
|
|
32
|
+
// Remote browsers handle their own proxy — don't inject local proxy settings into context
|
|
33
|
+
if (this.launchContext.isRemote) {
|
|
34
|
+
delete contextOptions?.proxy;
|
|
35
|
+
}
|
|
36
|
+
if (contextOptions?.proxy) {
|
|
37
|
+
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
|
|
38
|
+
if (anonymizedProxyUrl) {
|
|
39
|
+
contextOptions.proxy = {
|
|
40
|
+
server: anonymizedProxyUrl,
|
|
41
|
+
bypass: contextOptions.proxy.bypass,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
close = closeProxy;
|
|
33
45
|
}
|
|
34
|
-
close = closeProxy;
|
|
35
46
|
}
|
|
36
47
|
try {
|
|
37
48
|
const page = await this.browser.newPage(contextOptions);
|
|
@@ -63,4 +74,3 @@ export class PlaywrightController extends BrowserController {
|
|
|
63
74
|
return context.addCookies(cookies);
|
|
64
75
|
}
|
|
65
76
|
}
|
|
66
|
-
//# sourceMappingURL=playwright-controller.js.map
|
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
2
|
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
3
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
4
|
-
import type { createProxyServerForContainers } from '../container-proxy-server.js';
|
|
5
4
|
import type { LaunchContext } from '../launch-context.js';
|
|
5
|
+
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
6
6
|
import type { SafeParameters } from '../utils.js';
|
|
7
7
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
8
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
|
-
private
|
|
10
|
-
|
|
9
|
+
#private;
|
|
10
|
+
/**
|
|
11
|
+
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
|
+
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
|
+
*/
|
|
14
|
+
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
11
15
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
12
|
-
private
|
|
16
|
+
private throwOnFailedLaunch;
|
|
13
17
|
createController(): PlaywrightController;
|
|
14
|
-
protected
|
|
15
|
-
protected
|
|
18
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
19
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
16
20
|
}
|
|
17
|
-
//# sourceMappingURL=playwright-plugin.d.ts.map
|
|
@@ -1,14 +1,35 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
4
|
import { getLocalProxyAddress } from '../proxy-server.js';
|
|
6
5
|
import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
|
|
7
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
#browserVersion;
|
|
9
|
+
/**
|
|
10
|
+
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
11
|
+
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
12
|
+
*/
|
|
13
|
+
useRemoteConnection(connection, parameters = {}) {
|
|
14
|
+
super.useRemoteConnection(connection, parameters);
|
|
15
|
+
if (!this.useIncognitoPages) {
|
|
16
|
+
this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
|
|
17
|
+
'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
|
|
18
|
+
}
|
|
19
|
+
this.useIncognitoPages = true;
|
|
20
|
+
}
|
|
11
21
|
async _launch(launchContext) {
|
|
22
|
+
if (this.remoteConnection) {
|
|
23
|
+
return this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
24
|
+
const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
|
|
25
|
+
if (this.remoteConnectionParameters?.protocol === 'playwright') {
|
|
26
|
+
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
27
|
+
return this.library.connect(url, connectOptions);
|
|
28
|
+
}
|
|
29
|
+
this.log.info('Connecting to remote browser via connectOverCDP.');
|
|
30
|
+
return this.library.connectOverCDP(url, connectOptions);
|
|
31
|
+
});
|
|
32
|
+
}
|
|
12
33
|
const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
|
|
13
34
|
let browser;
|
|
14
35
|
// Required for the `proxy` context option to work.
|
|
@@ -20,7 +41,9 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
20
41
|
if (this.library.name() === 'webkit') {
|
|
21
42
|
launchOptions.args = launchOptions.args?.filter((arg) => arg !== '--no-sandbox');
|
|
22
43
|
}
|
|
23
|
-
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl
|
|
44
|
+
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
|
|
45
|
+
ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
|
|
46
|
+
});
|
|
24
47
|
if (anonymizedProxyUrl) {
|
|
25
48
|
launchOptions.proxy = {
|
|
26
49
|
server: anonymizedProxyUrl,
|
|
@@ -30,7 +53,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
30
53
|
try {
|
|
31
54
|
if (useIncognitoPages) {
|
|
32
55
|
browser = await this.library.launch(launchOptions).catch((error) => {
|
|
33
|
-
return this.
|
|
56
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
34
57
|
});
|
|
35
58
|
if (anonymizedProxyUrl) {
|
|
36
59
|
browser.on('disconnected', async () => {
|
|
@@ -42,7 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
42
65
|
const browserContext = await this.library
|
|
43
66
|
.launchPersistentContext(userDataDir, launchOptions)
|
|
44
67
|
.catch((error) => {
|
|
45
|
-
return this.
|
|
68
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
46
69
|
});
|
|
47
70
|
browserContext.once('close', () => {
|
|
48
71
|
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
|
|
@@ -57,18 +80,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
57
80
|
await close();
|
|
58
81
|
});
|
|
59
82
|
}
|
|
60
|
-
if (!this
|
|
83
|
+
if (!this.#browserVersion) {
|
|
61
84
|
// Launches unused browser just to get the browser version.
|
|
62
85
|
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
63
|
-
this
|
|
86
|
+
this.#browserVersion = inactiveBrowser.version();
|
|
64
87
|
inactiveBrowser.close().catch((error) => {
|
|
65
|
-
log.exception(error, 'Failed to close browser.');
|
|
88
|
+
this.log.exception(error, 'Failed to close browser.');
|
|
66
89
|
});
|
|
67
90
|
}
|
|
68
|
-
|
|
91
|
+
const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
|
|
69
92
|
browserContext,
|
|
70
|
-
version: this
|
|
93
|
+
version: this.#browserVersion,
|
|
71
94
|
});
|
|
95
|
+
persistentBrowser.setBrowserType(this.library);
|
|
96
|
+
browser = persistentBrowser;
|
|
72
97
|
}
|
|
73
98
|
}
|
|
74
99
|
catch (error) {
|
|
@@ -77,13 +102,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
77
102
|
}
|
|
78
103
|
return browser;
|
|
79
104
|
}
|
|
80
|
-
|
|
81
|
-
this.
|
|
105
|
+
throwOnFailedLaunch(launchContext, cause) {
|
|
106
|
+
this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
|
|
82
107
|
}
|
|
83
108
|
createController() {
|
|
84
109
|
return new PlaywrightController(this);
|
|
85
110
|
}
|
|
86
|
-
async
|
|
111
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
87
112
|
launchContext.launchOptions ??= {};
|
|
88
113
|
const { launchOptions, proxyUrl } = launchContext;
|
|
89
114
|
if (proxyUrl) {
|
|
@@ -95,9 +120,8 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
95
120
|
};
|
|
96
121
|
}
|
|
97
122
|
}
|
|
98
|
-
|
|
123
|
+
isChromiumBasedBrowser() {
|
|
99
124
|
const name = this.library.name();
|
|
100
125
|
return name === 'chromium';
|
|
101
126
|
}
|
|
102
127
|
}
|
|
103
|
-
//# sourceMappingURL=playwright-plugin.js.map
|
package/proxy-server.d.ts
CHANGED
package/proxy-server.js
CHANGED
|
@@ -16,4 +16,3 @@ export declare class PuppeteerController extends BrowserController<typeof Puppet
|
|
|
16
16
|
protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
|
|
17
17
|
protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
|
|
18
18
|
}
|
|
19
|
-
//# sourceMappingURL=puppeteer-controller.d.ts.map
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { tryCancel } from '@apify/timeout';
|
|
2
2
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
4
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
5
|
export class PuppeteerController extends BrowserController {
|
|
7
6
|
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
@@ -12,7 +11,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
12
11
|
const username = decodeURIComponent(url.username);
|
|
13
12
|
const password = decodeURIComponent(url.password);
|
|
14
13
|
return {
|
|
15
|
-
proxyServer: url.
|
|
14
|
+
proxyServer: `${url.protocol}//${url.host}`,
|
|
16
15
|
proxyUsername: username,
|
|
17
16
|
proxyPassword: password,
|
|
18
17
|
proxyBypassList: pageOptions?.proxyBypassList,
|
|
@@ -25,7 +24,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
25
24
|
}
|
|
26
25
|
let close = async () => { };
|
|
27
26
|
if (contextOptions.proxyServer) {
|
|
28
|
-
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
|
|
27
|
+
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
|
|
29
28
|
if (anonymizedProxyUrl) {
|
|
30
29
|
contextOptions.proxyServer = anonymizedProxyUrl;
|
|
31
30
|
delete contextOptions.proxyUsername;
|
|
@@ -58,7 +57,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
58
57
|
await context.close();
|
|
59
58
|
}
|
|
60
59
|
catch (error) {
|
|
61
|
-
log.exception(error, 'Failed to close context.');
|
|
60
|
+
this.log.exception(error, 'Failed to close context.');
|
|
62
61
|
}
|
|
63
62
|
finally {
|
|
64
63
|
await close();
|
|
@@ -84,7 +83,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
84
83
|
async _kill() {
|
|
85
84
|
const browserProcess = this.browser.process();
|
|
86
85
|
if (!browserProcess) {
|
|
87
|
-
log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
|
|
86
|
+
this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
|
|
88
87
|
return;
|
|
89
88
|
}
|
|
90
89
|
const timeout = setTimeout(() => {
|
|
@@ -98,14 +97,22 @@ export class PuppeteerController extends BrowserController {
|
|
|
98
97
|
clearTimeout(timeout);
|
|
99
98
|
}
|
|
100
99
|
catch (error) {
|
|
101
|
-
log.debug('Browser was already killed.', { error });
|
|
100
|
+
this.log.debug('Browser was already killed.', { error });
|
|
102
101
|
}
|
|
103
102
|
}
|
|
104
103
|
async _getCookies(page) {
|
|
105
|
-
|
|
104
|
+
const cookies = await page.browserContext().cookies();
|
|
105
|
+
// Puppeteer 25+ can report `sameSite: 'Default'`, which means the attribute is unspecified.
|
|
106
|
+
return cookies.map(({ sameSite, ...rest }) => ({
|
|
107
|
+
...rest,
|
|
108
|
+
sameSite: sameSite === 'Default' ? undefined : sameSite,
|
|
109
|
+
}));
|
|
106
110
|
}
|
|
107
111
|
async _setCookies(page, cookies) {
|
|
108
|
-
|
|
112
|
+
// BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill
|
|
113
|
+
// the page's current URL for us. Replicate that so callers who pass neither don't get rejected.
|
|
114
|
+
const pageUrl = page.url();
|
|
115
|
+
const normalized = cookies.map((cookie) => cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl });
|
|
116
|
+
return page.browserContext().setCookie(...normalized);
|
|
109
117
|
}
|
|
110
118
|
}
|
|
111
|
-
//# sourceMappingURL=puppeteer-controller.js.map
|
|
@@ -4,12 +4,14 @@ import type Puppeteer from 'puppeteer';
|
|
|
4
4
|
import type * as PuppeteerTypes from 'puppeteer';
|
|
5
5
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
6
6
|
import type { LaunchContext } from '../launch-context.js';
|
|
7
|
+
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
7
8
|
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
8
9
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
9
10
|
export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
11
|
+
/** Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito). */
|
|
12
|
+
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
10
13
|
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
|
11
14
|
createController(): PuppeteerController;
|
|
12
|
-
protected
|
|
13
|
-
protected
|
|
15
|
+
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
16
|
+
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
|
14
17
|
}
|
|
15
|
-
//# sourceMappingURL=puppeteer-plugin.d.ts.map
|