@crawlee/browser-pool 4.0.0-beta.20 → 4.0.0-beta.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/abstract-classes/browser-controller.d.ts +100 -29
- package/abstract-classes/browser-controller.js +70 -24
- package/abstract-classes/browser-plugin.d.ts +54 -23
- package/abstract-classes/browser-plugin.js +75 -13
- package/anonymize-proxy.d.ts +6 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +122 -49
- package/browser-pool.js +319 -147
- package/events.d.ts +2 -4
- package/events.js +0 -2
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +3 -34
- package/fingerprinting/types.js +0 -2
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +21 -5
- package/launch-context.js +21 -10
- package/package.json +9 -9
- package/playwright/playwright-browser.d.ts +3 -6
- package/playwright/playwright-browser.js +17 -17
- package/playwright/playwright-controller.d.ts +1 -1
- package/playwright/playwright-controller.js +10 -6
- package/playwright/playwright-plugin.d.ts +12 -7
- package/playwright/playwright-plugin.js +42 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +1 -1
- package/puppeteer/puppeteer-controller.js +29 -20
- package/puppeteer/puppeteer-plugin.d.ts +9 -3
- package/puppeteer/puppeteer-plugin.js +87 -46
- package/remote-browser-pool.d.ts +178 -0
- package/remote-browser-pool.js +206 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts +0 -12
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js +0 -43
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/events.d.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
|
-
BROWSER_CLOSED = "browserClosed",
|
|
5
4
|
PAGE_CREATED = "pageCreated",
|
|
6
5
|
PAGE_CLOSED = "pageClosed"
|
|
7
6
|
}
|
|
8
|
-
export declare
|
|
7
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
8
|
BROWSER_CLOSED = "browserClosed"
|
|
10
9
|
}
|
|
11
|
-
//# sourceMappingURL=events.d.ts.map
|
package/events.js
CHANGED
|
@@ -2,7 +2,6 @@ export var BROWSER_POOL_EVENTS;
|
|
|
2
2
|
(function (BROWSER_POOL_EVENTS) {
|
|
3
3
|
BROWSER_POOL_EVENTS["BROWSER_LAUNCHED"] = "browserLaunched";
|
|
4
4
|
BROWSER_POOL_EVENTS["BROWSER_RETIRED"] = "browserRetired";
|
|
5
|
-
BROWSER_POOL_EVENTS["BROWSER_CLOSED"] = "browserClosed";
|
|
6
5
|
BROWSER_POOL_EVENTS["PAGE_CREATED"] = "pageCreated";
|
|
7
6
|
BROWSER_POOL_EVENTS["PAGE_CLOSED"] = "pageClosed";
|
|
8
7
|
})(BROWSER_POOL_EVENTS || (BROWSER_POOL_EVENTS = {}));
|
|
@@ -10,4 +9,3 @@ export var BROWSER_CONTROLLER_EVENTS;
|
|
|
10
9
|
(function (BROWSER_CONTROLLER_EVENTS) {
|
|
11
10
|
BROWSER_CONTROLLER_EVENTS["BROWSER_CLOSED"] = "browserClosed";
|
|
12
11
|
})(BROWSER_CONTROLLER_EVENTS || (BROWSER_CONTROLLER_EVENTS = {}));
|
|
13
|
-
//# sourceMappingURL=events.js.map
|
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -1,29 +1,42 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
3
|
import { getGeneratorDefaultOptions } from './utils.js';
|
|
4
|
+
function applySessionHints(base, fingerprint) {
|
|
5
|
+
if (!fingerprint)
|
|
6
|
+
return base;
|
|
7
|
+
return {
|
|
8
|
+
...base,
|
|
9
|
+
...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
|
|
10
|
+
...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
|
|
11
|
+
...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
/**
|
|
5
15
|
* @internal
|
|
6
16
|
*/
|
|
7
17
|
export function createFingerprintPreLaunchHook(browserPool) {
|
|
8
18
|
const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
|
|
9
19
|
return (_pageId, launchContext) => {
|
|
20
|
+
// Remote browsers may have their own fingerprinting — skip local fingerprint injection
|
|
21
|
+
if (launchContext.isRemote)
|
|
22
|
+
return;
|
|
10
23
|
const { useIncognitoPages } = launchContext;
|
|
11
|
-
const
|
|
24
|
+
const session = launchContext.session;
|
|
25
|
+
const cacheKey = session?.id ?? launchContext.proxyUrl;
|
|
12
26
|
const { launchOptions } = launchContext;
|
|
13
|
-
// If no options are passed we try to pass best default options as possible to match browser and OS.
|
|
14
|
-
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
15
27
|
let fingerprint;
|
|
16
28
|
if (cacheKey && fingerprintCache?.has(cacheKey)) {
|
|
17
29
|
fingerprint = fingerprintCache.get(cacheKey);
|
|
18
30
|
}
|
|
19
|
-
else if (cacheKey) {
|
|
20
|
-
fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
|
|
21
|
-
fingerprintCache?.set(cacheKey, fingerprint);
|
|
22
|
-
}
|
|
23
31
|
else {
|
|
24
|
-
|
|
32
|
+
const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
33
|
+
const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
|
|
34
|
+
fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
|
|
35
|
+
if (cacheKey)
|
|
36
|
+
fingerprintCache?.set(cacheKey, fingerprint);
|
|
25
37
|
}
|
|
26
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
27
40
|
if (useIncognitoPages) {
|
|
28
41
|
return;
|
|
29
42
|
}
|
|
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
41
54
|
export function createPrePageCreateHook() {
|
|
42
55
|
return (_pageId, browserController, pageOptions) => {
|
|
43
56
|
const { launchContext, browserPlugin } = browserController;
|
|
57
|
+
if (launchContext.isRemote)
|
|
58
|
+
return;
|
|
44
59
|
const { fingerprint } = launchContext.fingerprint;
|
|
45
60
|
if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
|
|
46
61
|
pageOptions.userAgent ??= fingerprint.navigator.userAgent;
|
|
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
|
|
|
57
72
|
export function createPostPageCreateHook(fingerprintInjector) {
|
|
58
73
|
return async (page, browserController) => {
|
|
59
74
|
const { browserPlugin, launchContext } = browserController;
|
|
75
|
+
if (launchContext.isRemote)
|
|
76
|
+
return;
|
|
60
77
|
const fingerprint = launchContext.fingerprint;
|
|
61
78
|
// TODO this will require refactoring, we should use common API instead of branching based on plugin type,
|
|
62
79
|
// and there should be no public methods specific to some browser.
|
|
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
|
|
|
79
96
|
}
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
|
-
//# sourceMappingURL=hooks.js.map
|
|
@@ -1,42 +1,13 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export interface FingerprintGenerator {
|
|
3
|
-
getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
|
|
4
|
-
}
|
|
5
|
-
export interface GetFingerprintReturn {
|
|
6
|
-
fingerprint: Fingerprint;
|
|
7
|
-
}
|
|
1
|
+
import type { FingerprintGeneratorOptions as FingerprintOptionsOriginal } from 'fingerprint-generator';
|
|
8
2
|
export interface FingerprintGeneratorOptions extends Partial<FingerprintOptionsOriginal> {
|
|
9
3
|
}
|
|
10
|
-
declare const SUPPORTED_HTTP_VERSIONS: readonly ["1", "2"];
|
|
11
|
-
/**
|
|
12
|
-
* String specifying the HTTP version to use.
|
|
13
|
-
*/
|
|
14
|
-
type HttpVersion = (typeof SUPPORTED_HTTP_VERSIONS)[number];
|
|
15
4
|
export declare enum BrowserName {
|
|
16
5
|
chrome = "chrome",
|
|
17
6
|
firefox = "firefox",
|
|
18
7
|
safari = "safari",
|
|
19
8
|
edge = "edge"
|
|
20
9
|
}
|
|
21
|
-
export
|
|
22
|
-
/**
|
|
23
|
-
* String representing the browser name.
|
|
24
|
-
*/
|
|
25
|
-
name: BrowserName;
|
|
26
|
-
/**
|
|
27
|
-
* Minimum version of browser used.
|
|
28
|
-
*/
|
|
29
|
-
minVersion?: number;
|
|
30
|
-
/**
|
|
31
|
-
* Maximum version of browser used.
|
|
32
|
-
*/
|
|
33
|
-
maxVersion?: number;
|
|
34
|
-
/**
|
|
35
|
-
* HTTP version to be used for header generation (the headers differ depending on the version).
|
|
36
|
-
*/
|
|
37
|
-
httpVersion?: HttpVersion;
|
|
38
|
-
}
|
|
39
|
-
export declare const enum OperatingSystemsName {
|
|
10
|
+
export declare enum OperatingSystemsName {
|
|
40
11
|
linux = "linux",
|
|
41
12
|
macos = "macos",
|
|
42
13
|
windows = "windows",
|
|
@@ -49,7 +20,7 @@ export declare const enum OperatingSystemsName {
|
|
|
49
20
|
*/
|
|
50
21
|
ios = "ios"
|
|
51
22
|
}
|
|
52
|
-
export declare
|
|
23
|
+
export declare enum DeviceCategory {
|
|
53
24
|
/**
|
|
54
25
|
* Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
|
|
55
26
|
* > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
|
|
@@ -60,5 +31,3 @@ export declare const enum DeviceCategory {
|
|
|
60
31
|
*/
|
|
61
32
|
desktop = "desktop"
|
|
62
33
|
}
|
|
63
|
-
export {};
|
|
64
|
-
//# sourceMappingURL=types.d.ts.map
|
package/fingerprinting/types.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
const SUPPORTED_HTTP_VERSIONS = ['1', '2'];
|
|
2
1
|
export var BrowserName;
|
|
3
2
|
(function (BrowserName) {
|
|
4
3
|
BrowserName["chrome"] = "chrome";
|
|
@@ -32,4 +31,3 @@ export var DeviceCategory;
|
|
|
32
31
|
*/
|
|
33
32
|
DeviceCategory["desktop"] = "desktop";
|
|
34
33
|
})(DeviceCategory || (DeviceCategory = {}));
|
|
35
|
-
//# sourceMappingURL=types.js.map
|
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
import type { LaunchContext } from '../launch-context.js';
|
|
2
2
|
import type { FingerprintGeneratorOptions } from './types.js';
|
|
3
3
|
export declare const getGeneratorDefaultOptions: (launchContext: LaunchContext) => FingerprintGeneratorOptions;
|
|
4
|
-
//# sourceMappingURL=utils.d.ts.map
|
package/fingerprinting/utils.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
|
-
import { BrowserName } from './types.js';
|
|
3
|
+
import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
|
|
4
4
|
export const getGeneratorDefaultOptions = (launchContext) => {
|
|
5
5
|
const { browserPlugin, launchOptions } = launchContext;
|
|
6
6
|
const options = {
|
|
7
|
-
devices: [
|
|
7
|
+
devices: [DeviceCategory.desktop],
|
|
8
8
|
locales: ['en-US'],
|
|
9
9
|
browsers: [getBrowserName(browserPlugin, launchOptions)],
|
|
10
10
|
operatingSystems: [getOperatingSystem()],
|
|
@@ -34,12 +34,11 @@ const getOperatingSystem = () => {
|
|
|
34
34
|
switch (platform) {
|
|
35
35
|
case 'win32':
|
|
36
36
|
// platform is win32 even for 64-bit
|
|
37
|
-
return
|
|
37
|
+
return OperatingSystemsName.windows;
|
|
38
38
|
case 'darwin':
|
|
39
|
-
return
|
|
39
|
+
return OperatingSystemsName.macos;
|
|
40
40
|
default:
|
|
41
41
|
// consider everything else a linux
|
|
42
|
-
return
|
|
42
|
+
return OperatingSystemsName.linux;
|
|
43
43
|
}
|
|
44
44
|
};
|
|
45
|
-
//# sourceMappingURL=utils.js.map
|
package/index.d.ts
CHANGED
|
@@ -28,13 +28,20 @@ export * from './browser-pool.js';
|
|
|
28
28
|
export * from './playwright/playwright-plugin.js';
|
|
29
29
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
30
30
|
export * from './events.js';
|
|
31
|
-
export {
|
|
32
|
-
export {
|
|
31
|
+
export type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
|
|
32
|
+
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
33
|
+
export type { BrowserControllerEvents, IBrowserController, IBrowserLaunchContext, } from './abstract-classes/browser-controller.js';
|
|
34
|
+
export { BrowserController } from './abstract-classes/browser-controller.js';
|
|
33
35
|
export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
|
|
34
36
|
export { PlaywrightController } from './playwright/playwright-controller.js';
|
|
35
37
|
export { PlaywrightBrowser } from './playwright/playwright-browser.js';
|
|
36
|
-
export { CommonPage, CommonLibrary,
|
|
37
|
-
export {
|
|
38
|
-
export {
|
|
39
|
-
export {
|
|
40
|
-
|
|
38
|
+
export type { CommonPage, CommonLibrary, BrowserPluginOptions, CreateLaunchContextOptions, } from './abstract-classes/browser-plugin.js';
|
|
39
|
+
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
|
|
40
|
+
export type { LaunchContextOptions } from './launch-context.js';
|
|
41
|
+
export { LaunchContext } from './launch-context.js';
|
|
42
|
+
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
43
|
+
export { RemoteBrowserPool } from './remote-browser-pool.js';
|
|
44
|
+
export type { RemoteBrowserPoolOptions, CrawlerRemoteBrowserOptions, RemoteBrowserEndpoint, ResolvedRemoteEndpoint, RemoteConnection, RemoteConnectionParameters, } from './remote-browser-pool.js';
|
|
45
|
+
export type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
|
|
46
|
+
export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy.js';
|
|
47
|
+
export type { IBrowserPool, NewPageOptions } from '@crawlee/types';
|
package/index.js
CHANGED
|
@@ -26,11 +26,13 @@ export * from './browser-pool.js';
|
|
|
26
26
|
export * from './playwright/playwright-plugin.js';
|
|
27
27
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
28
28
|
export * from './events.js';
|
|
29
|
-
export { BrowserName, DeviceCategory, OperatingSystemsName
|
|
29
|
+
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
30
30
|
export { BrowserController } from './abstract-classes/browser-controller.js';
|
|
31
31
|
export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
|
|
32
32
|
export { PlaywrightController } from './playwright/playwright-controller.js';
|
|
33
33
|
export { PlaywrightBrowser } from './playwright/playwright-browser.js';
|
|
34
|
-
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT
|
|
34
|
+
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
|
|
35
35
|
export { LaunchContext } from './launch-context.js';
|
|
36
|
-
|
|
36
|
+
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
37
|
+
export { RemoteBrowserPool } from './remote-browser-pool.js';
|
|
38
|
+
export { anonymizeProxySugar } from './anonymize-proxy.js';
|
package/launch-context.d.ts
CHANGED
|
@@ -43,19 +43,36 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
43
43
|
*/
|
|
44
44
|
userDataDir?: string;
|
|
45
45
|
proxyUrl?: string;
|
|
46
|
-
|
|
46
|
+
/**
|
|
47
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
48
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
49
|
+
*/
|
|
50
|
+
ignoreProxyCertificate?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Whether this launch context represents a connection to a remote browser
|
|
53
|
+
* rather than a locally launched one.
|
|
54
|
+
* @default false
|
|
55
|
+
* @internal
|
|
56
|
+
*/
|
|
57
|
+
isRemote?: boolean;
|
|
47
58
|
}
|
|
48
59
|
export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
60
|
+
#private;
|
|
49
61
|
id?: string;
|
|
50
62
|
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
51
63
|
launchOptions: LibraryOptions;
|
|
52
64
|
useIncognitoPages: boolean;
|
|
53
65
|
browserPerProxy?: boolean;
|
|
54
66
|
userDataDir: string;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
private readonly _reservedFieldNames;
|
|
67
|
+
readonly isRemote: boolean;
|
|
68
|
+
ignoreProxyCertificate?: boolean;
|
|
58
69
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
70
|
+
/**
|
|
71
|
+
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
72
|
+
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
73
|
+
* @internal
|
|
74
|
+
*/
|
|
75
|
+
remoteToken?: number;
|
|
59
76
|
[K: PropertyKey]: unknown;
|
|
60
77
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
61
78
|
/**
|
|
@@ -76,4 +93,3 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
76
93
|
*/
|
|
77
94
|
get proxyUrl(): string | undefined;
|
|
78
95
|
}
|
|
79
|
-
//# sourceMappingURL=launch-context.d.ts.map
|
package/launch-context.js
CHANGED
|
@@ -5,20 +5,31 @@ export class LaunchContext {
|
|
|
5
5
|
useIncognitoPages;
|
|
6
6
|
browserPerProxy;
|
|
7
7
|
userDataDir;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
isRemote;
|
|
9
|
+
ignoreProxyCertificate;
|
|
10
|
+
#proxyUrl;
|
|
11
|
+
#reservedFieldNames;
|
|
11
12
|
fingerprint;
|
|
13
|
+
/**
|
|
14
|
+
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
|
+
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
remoteToken;
|
|
12
19
|
constructor(options) {
|
|
13
|
-
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '',
|
|
20
|
+
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
14
21
|
this.id = id;
|
|
15
22
|
this.browserPlugin = browserPlugin;
|
|
16
23
|
this.launchOptions = launchOptions;
|
|
17
24
|
this.browserPerProxy = browserPerProxy ?? false;
|
|
18
25
|
this.useIncognitoPages = useIncognitoPages ?? false;
|
|
19
26
|
this.userDataDir = userDataDir;
|
|
20
|
-
this.
|
|
21
|
-
this.
|
|
27
|
+
this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
|
|
28
|
+
this.isRemote = isRemote ?? false;
|
|
29
|
+
this.#proxyUrl = proxyUrl;
|
|
30
|
+
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
|
+
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
|
|
22
33
|
}
|
|
23
34
|
/**
|
|
24
35
|
* Extend the launch context with any extra fields.
|
|
@@ -29,7 +40,7 @@ export class LaunchContext {
|
|
|
29
40
|
*/
|
|
30
41
|
extend(fields) {
|
|
31
42
|
Object.entries(fields).forEach(([key, value]) => {
|
|
32
|
-
if (this.
|
|
43
|
+
if (this.#reservedFieldNames.includes(key)) {
|
|
33
44
|
throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
|
|
34
45
|
}
|
|
35
46
|
else {
|
|
@@ -43,6 +54,7 @@ export class LaunchContext {
|
|
|
43
54
|
*/
|
|
44
55
|
set proxyUrl(url) {
|
|
45
56
|
if (!url) {
|
|
57
|
+
this.#proxyUrl = undefined;
|
|
46
58
|
return;
|
|
47
59
|
}
|
|
48
60
|
const urlInstance = new URL(url);
|
|
@@ -50,13 +62,12 @@ export class LaunchContext {
|
|
|
50
62
|
urlInstance.search = '';
|
|
51
63
|
urlInstance.hash = '';
|
|
52
64
|
// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
|
|
53
|
-
this
|
|
65
|
+
this.#proxyUrl = urlInstance.href.slice(0, -1);
|
|
54
66
|
}
|
|
55
67
|
/**
|
|
56
68
|
* Returns the proxy URL of the browser.
|
|
57
69
|
*/
|
|
58
70
|
get proxyUrl() {
|
|
59
|
-
return this
|
|
71
|
+
return this.#proxyUrl;
|
|
60
72
|
}
|
|
61
73
|
}
|
|
62
|
-
//# sourceMappingURL=launch-context.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.201",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -24,26 +24,26 @@
|
|
|
24
24
|
"url": "https://github.com/apify/crawlee/issues"
|
|
25
25
|
},
|
|
26
26
|
"scripts": {
|
|
27
|
-
"build": "
|
|
27
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
28
28
|
"clean": "rimraf ./dist",
|
|
29
29
|
"compile": "tsc -p tsconfig.build.json",
|
|
30
30
|
"copy": "tsx ../../scripts/copy.ts"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@apify/
|
|
34
|
-
"@
|
|
35
|
-
"@crawlee/
|
|
36
|
-
"@crawlee/
|
|
33
|
+
"@apify/timeout": "^1.0.1",
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.201",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.201",
|
|
36
|
+
"@crawlee/utils": "4.0.0-beta.201",
|
|
37
37
|
"fingerprint-generator": "^2.1.68",
|
|
38
38
|
"fingerprint-injector": "^2.1.68",
|
|
39
39
|
"lodash.merge": "^4.6.2",
|
|
40
40
|
"nanoid": "^5.1.5",
|
|
41
|
-
"ow": "^2.0.0",
|
|
42
41
|
"p-limit": "^6.2.0",
|
|
43
42
|
"proxy-chain": "^2.5.8",
|
|
44
43
|
"quick-lru": "^7.0.1",
|
|
45
44
|
"tiny-typed-emitter": "^2.1.0",
|
|
46
|
-
"tslib": "^2.8.1"
|
|
45
|
+
"tslib": "^2.8.1",
|
|
46
|
+
"zod": "^4.5.4"
|
|
47
47
|
},
|
|
48
48
|
"peerDependencies": {
|
|
49
49
|
"playwright": "*",
|
|
@@ -64,5 +64,5 @@
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "5b29925251e92eb69252045f4980153844f58462"
|
|
68
68
|
}
|
|
@@ -9,10 +9,8 @@ export interface BrowserOptions {
|
|
|
9
9
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
10
10
|
*/
|
|
11
11
|
export declare class PlaywrightBrowser extends EventEmitter {
|
|
12
|
-
private
|
|
13
|
-
|
|
14
|
-
private _isConnected;
|
|
15
|
-
private _browserType?;
|
|
12
|
+
#private;
|
|
13
|
+
/** @internal */
|
|
16
14
|
constructor(options: BrowserOptions);
|
|
17
15
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
16
|
close(): Promise<void>;
|
|
@@ -20,7 +18,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
20
18
|
isConnected(): boolean;
|
|
21
19
|
version(): string;
|
|
22
20
|
/** @internal */
|
|
23
|
-
|
|
21
|
+
setBrowserType(browserType: BrowserType): void;
|
|
24
22
|
browserType(): BrowserType;
|
|
25
23
|
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
|
26
24
|
newContext(): Promise<never>;
|
|
@@ -28,4 +26,3 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
28
26
|
startTracing(): Promise<never>;
|
|
29
27
|
stopTracing(): Promise<never>;
|
|
30
28
|
}
|
|
31
|
-
//# sourceMappingURL=playwright-browser.d.ts.map
|
|
@@ -3,17 +3,18 @@ import { EventEmitter } from 'node:events';
|
|
|
3
3
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
4
4
|
*/
|
|
5
5
|
export class PlaywrightBrowser extends EventEmitter {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
#browserContext;
|
|
7
|
+
#version;
|
|
8
|
+
#isConnected = true;
|
|
9
|
+
#browserType;
|
|
10
|
+
/** @internal */
|
|
10
11
|
constructor(options) {
|
|
11
12
|
super();
|
|
12
13
|
const { browserContext, version } = options;
|
|
13
|
-
this
|
|
14
|
-
this
|
|
15
|
-
this.
|
|
16
|
-
this
|
|
14
|
+
this.#browserContext = browserContext;
|
|
15
|
+
this.#version = version;
|
|
16
|
+
this.#browserContext.once('close', () => {
|
|
17
|
+
this.#isConnected = false;
|
|
17
18
|
this.emit('disconnected');
|
|
18
19
|
});
|
|
19
20
|
}
|
|
@@ -21,26 +22,26 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
21
22
|
await this.close();
|
|
22
23
|
}
|
|
23
24
|
async close() {
|
|
24
|
-
await this.
|
|
25
|
+
await this.#browserContext.close();
|
|
25
26
|
}
|
|
26
27
|
contexts() {
|
|
27
|
-
return [this
|
|
28
|
+
return [this.#browserContext];
|
|
28
29
|
}
|
|
29
30
|
isConnected() {
|
|
30
|
-
return this
|
|
31
|
+
return this.#isConnected;
|
|
31
32
|
}
|
|
32
33
|
version() {
|
|
33
|
-
return this
|
|
34
|
+
return this.#version;
|
|
34
35
|
}
|
|
35
36
|
/** @internal */
|
|
36
|
-
|
|
37
|
-
this
|
|
37
|
+
setBrowserType(browserType) {
|
|
38
|
+
this.#browserType = browserType;
|
|
38
39
|
}
|
|
39
40
|
browserType() {
|
|
40
|
-
return this
|
|
41
|
+
return this.#browserType;
|
|
41
42
|
}
|
|
42
43
|
async newPage(...args) {
|
|
43
|
-
return this.
|
|
44
|
+
return this.#browserContext.newPage(...args);
|
|
44
45
|
}
|
|
45
46
|
async newContext() {
|
|
46
47
|
throw new Error('Function `newContext()` is not available in incognito mode');
|
|
@@ -55,4 +56,3 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
55
56
|
throw new Error('Function `stopTracing()` is not available in incognito mode');
|
|
56
57
|
}
|
|
57
58
|
}
|
|
58
|
-
//# sourceMappingURL=playwright-browser.js.map
|
|
@@ -4,6 +4,7 @@ import type { Browser, BrowserType, Page } from 'playwright';
|
|
|
4
4
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
5
5
|
import type { SafeParameters } from '../utils.js';
|
|
6
6
|
export declare class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
|
|
7
|
+
/** @internal */
|
|
7
8
|
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
8
9
|
protected _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page>;
|
|
9
10
|
protected _close(): Promise<void>;
|
|
@@ -11,4 +12,3 @@ export declare class PlaywrightController extends BrowserController<BrowserType,
|
|
|
11
12
|
protected _getCookies(page: Page): Promise<Cookie[]>;
|
|
12
13
|
protected _setCookies(page: Page, cookies: Cookie[]): Promise<void>;
|
|
13
14
|
}
|
|
14
|
-
//# sourceMappingURL=playwright-controller.d.ts.map
|
|
@@ -2,6 +2,7 @@ import { tryCancel } from '@apify/timeout';
|
|
|
2
2
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
4
|
export class PlaywrightController extends BrowserController {
|
|
5
|
+
/** @internal */
|
|
5
6
|
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
6
7
|
if (!proxyUrl) {
|
|
7
8
|
return {};
|
|
@@ -11,7 +12,7 @@ export class PlaywrightController extends BrowserController {
|
|
|
11
12
|
const password = decodeURIComponent(url.password);
|
|
12
13
|
return {
|
|
13
14
|
proxy: {
|
|
14
|
-
server: url.
|
|
15
|
+
server: `${url.protocol}//${url.host}`,
|
|
15
16
|
username,
|
|
16
17
|
password,
|
|
17
18
|
bypass: pageOptions?.proxy?.bypass,
|
|
@@ -29,8 +30,12 @@ export class PlaywrightController extends BrowserController {
|
|
|
29
30
|
...this.launchContext.launchOptions,
|
|
30
31
|
...contextOptions,
|
|
31
32
|
};
|
|
33
|
+
// Remote browsers handle their own proxy — don't inject local proxy settings into context
|
|
34
|
+
if (this.launchContext.isRemote) {
|
|
35
|
+
delete contextOptions?.proxy;
|
|
36
|
+
}
|
|
32
37
|
if (contextOptions?.proxy) {
|
|
33
|
-
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password);
|
|
38
|
+
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
|
|
34
39
|
if (anonymizedProxyUrl) {
|
|
35
40
|
contextOptions.proxy = {
|
|
36
41
|
server: anonymizedProxyUrl,
|
|
@@ -42,9 +47,9 @@ export class PlaywrightController extends BrowserController {
|
|
|
42
47
|
}
|
|
43
48
|
try {
|
|
44
49
|
const page = await this.browser.newPage(contextOptions);
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
this.registerPageTeardown(page, close);
|
|
51
|
+
page.once('close', () => {
|
|
52
|
+
this.registerPageClosed(page);
|
|
48
53
|
});
|
|
49
54
|
tryCancel();
|
|
50
55
|
return page;
|
|
@@ -70,4 +75,3 @@ export class PlaywrightController extends BrowserController {
|
|
|
70
75
|
return context.addCookies(cookies);
|
|
71
76
|
}
|
|
72
77
|
}
|
|
73
|
-
//# sourceMappingURL=playwright-controller.js.map
|
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
2
|
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
3
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
4
|
-
import type { createProxyServerForContainers } from '../container-proxy-server.js';
|
|
5
4
|
import type { LaunchContext } from '../launch-context.js';
|
|
5
|
+
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
6
6
|
import type { SafeParameters } from '../utils.js';
|
|
7
7
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
8
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
|
-
private
|
|
10
|
-
|
|
9
|
+
#private;
|
|
10
|
+
/**
|
|
11
|
+
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
|
+
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
|
+
*
|
|
14
|
+
* @internal
|
|
15
|
+
*/
|
|
16
|
+
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
11
17
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
12
|
-
private
|
|
18
|
+
private throwOnFailedLaunch;
|
|
13
19
|
createController(): PlaywrightController;
|
|
14
|
-
protected
|
|
15
|
-
protected
|
|
20
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
21
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
16
22
|
}
|
|
17
|
-
//# sourceMappingURL=playwright-plugin.d.ts.map
|