@crawlee/browser-pool 4.0.0-beta.25 → 4.0.0-beta.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/abstract-classes/browser-controller.d.ts +100 -29
- package/abstract-classes/browser-controller.js +70 -24
- package/abstract-classes/browser-plugin.d.ts +25 -22
- package/abstract-classes/browser-plugin.js +21 -13
- package/anonymize-proxy.d.ts +6 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +110 -51
- package/browser-pool.js +318 -147
- package/events.d.ts +2 -4
- package/events.js +0 -2
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +3 -34
- package/fingerprinting/types.js +0 -2
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +15 -7
- package/index.js +6 -3
- package/launch-context.d.ts +15 -5
- package/launch-context.js +15 -10
- package/package.json +15 -15
- package/playwright/playwright-browser.d.ts +3 -6
- package/playwright/playwright-browser.js +17 -17
- package/playwright/playwright-controller.d.ts +1 -1
- package/playwright/playwright-controller.js +10 -6
- package/playwright/playwright-plugin.d.ts +3 -7
- package/playwright/playwright-plugin.js +18 -17
- package/playwright/remote-playwright-plugin.d.ts +29 -0
- package/playwright/remote-playwright-plugin.js +38 -0
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +1 -1
- package/puppeteer/puppeteer-controller.js +29 -20
- package/puppeteer/puppeteer-plugin.d.ts +12 -4
- package/puppeteer/puppeteer-plugin.js +75 -48
- package/puppeteer/remote-puppeteer-plugin.d.ts +22 -0
- package/puppeteer/remote-puppeteer-plugin.js +31 -0
- package/remote-browser-plugin.d.ts +49 -0
- package/remote-browser-plugin.js +63 -0
- package/remote-browser-provider.d.ts +81 -0
- package/remote-browser-provider.js +65 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts +0 -12
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js +0 -43
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
4
|
import { getLocalProxyAddress } from '../proxy-server.js';
|
|
6
5
|
import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
|
|
7
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
9
|
-
|
|
10
|
-
_containerProxyServer;
|
|
8
|
+
#browserVersion;
|
|
11
9
|
async _launch(launchContext) {
|
|
12
10
|
const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
|
|
13
11
|
let browser;
|
|
@@ -20,7 +18,9 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
20
18
|
if (this.library.name() === 'webkit') {
|
|
21
19
|
launchOptions.args = launchOptions.args?.filter((arg) => arg !== '--no-sandbox');
|
|
22
20
|
}
|
|
23
|
-
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl
|
|
21
|
+
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
|
|
22
|
+
ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
|
|
23
|
+
});
|
|
24
24
|
if (anonymizedProxyUrl) {
|
|
25
25
|
launchOptions.proxy = {
|
|
26
26
|
server: anonymizedProxyUrl,
|
|
@@ -30,7 +30,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
30
30
|
try {
|
|
31
31
|
if (useIncognitoPages) {
|
|
32
32
|
browser = await this.library.launch(launchOptions).catch((error) => {
|
|
33
|
-
return this
|
|
33
|
+
return this.#throwOnFailedLaunch(launchContext, error);
|
|
34
34
|
});
|
|
35
35
|
if (anonymizedProxyUrl) {
|
|
36
36
|
browser.on('disconnected', async () => {
|
|
@@ -42,7 +42,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
42
42
|
const browserContext = await this.library
|
|
43
43
|
.launchPersistentContext(userDataDir, launchOptions)
|
|
44
44
|
.catch((error) => {
|
|
45
|
-
return this
|
|
45
|
+
return this.#throwOnFailedLaunch(launchContext, error);
|
|
46
46
|
});
|
|
47
47
|
browserContext.once('close', () => {
|
|
48
48
|
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
|
|
@@ -57,18 +57,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
57
57
|
await close();
|
|
58
58
|
});
|
|
59
59
|
}
|
|
60
|
-
if (!this
|
|
60
|
+
if (!this.#browserVersion) {
|
|
61
61
|
// Launches unused browser just to get the browser version.
|
|
62
62
|
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
63
|
-
this
|
|
63
|
+
this.#browserVersion = inactiveBrowser.version();
|
|
64
64
|
inactiveBrowser.close().catch((error) => {
|
|
65
|
-
log.exception(error, 'Failed to close browser.');
|
|
65
|
+
this.log.exception(error, 'Failed to close browser.');
|
|
66
66
|
});
|
|
67
67
|
}
|
|
68
|
-
|
|
68
|
+
const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
|
|
69
69
|
browserContext,
|
|
70
|
-
version: this
|
|
70
|
+
version: this.#browserVersion,
|
|
71
71
|
});
|
|
72
|
+
persistentBrowser.setBrowserType(this.library);
|
|
73
|
+
browser = persistentBrowser;
|
|
72
74
|
}
|
|
73
75
|
}
|
|
74
76
|
catch (error) {
|
|
@@ -77,27 +79,26 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
77
79
|
}
|
|
78
80
|
return browser;
|
|
79
81
|
}
|
|
80
|
-
|
|
81
|
-
this.
|
|
82
|
+
#throwOnFailedLaunch(launchContext, cause) {
|
|
83
|
+
this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
|
|
82
84
|
}
|
|
83
85
|
createController() {
|
|
84
86
|
return new PlaywrightController(this);
|
|
85
87
|
}
|
|
86
|
-
async
|
|
88
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
87
89
|
launchContext.launchOptions ??= {};
|
|
88
90
|
const { launchOptions, proxyUrl } = launchContext;
|
|
89
91
|
if (proxyUrl) {
|
|
90
92
|
const url = new URL(proxyUrl);
|
|
91
93
|
launchOptions.proxy = {
|
|
92
|
-
server: url.
|
|
94
|
+
server: `${url.protocol}//${url.host}`,
|
|
93
95
|
username: decodeURIComponent(url.username),
|
|
94
96
|
password: decodeURIComponent(url.password),
|
|
95
97
|
};
|
|
96
98
|
}
|
|
97
99
|
}
|
|
98
|
-
|
|
100
|
+
isChromiumBasedBrowser() {
|
|
99
101
|
const name = this.library.name();
|
|
100
102
|
return name === 'chromium';
|
|
101
103
|
}
|
|
102
104
|
}
|
|
103
|
-
//# sourceMappingURL=playwright-plugin.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
+
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
|
+
import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
|
|
4
|
+
import type { LaunchContext } from '../launch-context.js';
|
|
5
|
+
import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
|
|
6
|
+
import type { SafeParameters } from '../utils.js';
|
|
7
|
+
import { PlaywrightPlugin } from './playwright-plugin.js';
|
|
8
|
+
export interface RemotePlaywrightPluginOptions extends BrowserPluginOptions<SafeParameters<BrowserType['launch']>[0]>, RemoteBrowserOptions {
|
|
9
|
+
/**
|
|
10
|
+
* Which protocol to connect with. `'cdp'` uses `connectOverCDP()` (the default), `'playwright'` uses
|
|
11
|
+
* `connect()` (Playwright's own WebSocket protocol, for endpoints started with `launchServer()`).
|
|
12
|
+
*/
|
|
13
|
+
protocol?: 'cdp' | 'playwright';
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
17
|
+
* instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
18
|
+
*
|
|
19
|
+
* Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
|
|
20
|
+
* contexts), so `useIncognitoPages` is always `true`.
|
|
21
|
+
*/
|
|
22
|
+
export declare class RemotePlaywrightPlugin extends PlaywrightPlugin {
|
|
23
|
+
#private;
|
|
24
|
+
constructor(library: BrowserType, options: RemotePlaywrightPluginOptions);
|
|
25
|
+
createLaunchContext(options?: CreateLaunchContextOptions<BrowserType>): LaunchContext<BrowserType>;
|
|
26
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
27
|
+
launch(launchContext?: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
28
|
+
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { connectRemoteBrowser } from '../remote-browser-plugin.js';
|
|
2
|
+
import { PlaywrightPlugin } from './playwright-plugin.js';
|
|
3
|
+
/**
|
|
4
|
+
* A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
5
|
+
* instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
6
|
+
*
|
|
7
|
+
* Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
|
|
8
|
+
* contexts), so `useIncognitoPages` is always `true`.
|
|
9
|
+
*/
|
|
10
|
+
export class RemotePlaywrightPlugin extends PlaywrightPlugin {
|
|
11
|
+
#remote;
|
|
12
|
+
constructor(library, options) {
|
|
13
|
+
super(library, { ...options, useIncognitoPages: true });
|
|
14
|
+
this.#remote = options;
|
|
15
|
+
if (options.useIncognitoPages === false) {
|
|
16
|
+
this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
|
|
17
|
+
'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
createLaunchContext(options = {}) {
|
|
21
|
+
return super.createLaunchContext({ ...options, isRemote: true });
|
|
22
|
+
}
|
|
23
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
24
|
+
async launch(launchContext = this.createLaunchContext()) {
|
|
25
|
+
return this._launch(launchContext);
|
|
26
|
+
}
|
|
27
|
+
async _launch(launchContext) {
|
|
28
|
+
const { protocol, connectOptions = {} } = this.#remote;
|
|
29
|
+
return connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
|
|
30
|
+
if (protocol === 'playwright') {
|
|
31
|
+
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
32
|
+
return this.library.connect(url, connectOptions);
|
|
33
|
+
}
|
|
34
|
+
this.log.info('Connecting to remote browser via connectOverCDP.');
|
|
35
|
+
return this.library.connectOverCDP(url, connectOptions);
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
package/proxy-server.d.ts
CHANGED
package/proxy-server.js
CHANGED
|
@@ -9,6 +9,7 @@ export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOp
|
|
|
9
9
|
proxyPassword?: string;
|
|
10
10
|
}
|
|
11
11
|
export declare class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
12
|
+
/** @internal */
|
|
12
13
|
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
13
14
|
protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
|
|
14
15
|
protected _close(): Promise<void>;
|
|
@@ -16,4 +17,3 @@ export declare class PuppeteerController extends BrowserController<typeof Puppet
|
|
|
16
17
|
protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
|
|
17
18
|
protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
|
|
18
19
|
}
|
|
19
|
-
//# sourceMappingURL=puppeteer-controller.d.ts.map
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { tryCancel } from '@apify/timeout';
|
|
2
2
|
import { BrowserController } from '../abstract-classes/browser-controller.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
4
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
5
|
export class PuppeteerController extends BrowserController {
|
|
6
|
+
/** @internal */
|
|
7
7
|
normalizeProxyOptions(proxyUrl, pageOptions) {
|
|
8
8
|
if (!proxyUrl) {
|
|
9
9
|
return {};
|
|
@@ -12,7 +12,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
12
12
|
const username = decodeURIComponent(url.username);
|
|
13
13
|
const password = decodeURIComponent(url.password);
|
|
14
14
|
return {
|
|
15
|
-
proxyServer: url.
|
|
15
|
+
proxyServer: `${url.protocol}//${url.host}`,
|
|
16
16
|
proxyUsername: username,
|
|
17
17
|
proxyPassword: password,
|
|
18
18
|
proxyBypassList: pageOptions?.proxyBypassList,
|
|
@@ -25,7 +25,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
25
25
|
}
|
|
26
26
|
let close = async () => { };
|
|
27
27
|
if (contextOptions.proxyServer) {
|
|
28
|
-
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
|
|
28
|
+
const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
|
|
29
29
|
if (anonymizedProxyUrl) {
|
|
30
30
|
contextOptions.proxyServer = anonymizedProxyUrl;
|
|
31
31
|
delete contextOptions.proxyUsername;
|
|
@@ -52,17 +52,18 @@ export class PuppeteerController extends BrowserController {
|
|
|
52
52
|
tryCancel();
|
|
53
53
|
}
|
|
54
54
|
*/
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
55
|
+
this.registerPageTeardown(page, async () => {
|
|
56
|
+
// The proxy server is not chained behind the context close: that can hang on
|
|
57
|
+
// the same target the page hung on, and the proxy runs in this process, so it
|
|
58
|
+
// has to be reclaimed either way.
|
|
59
|
+
const contextClosed = context.close().catch((error) => {
|
|
60
|
+
this.log.exception(error, 'Failed to close context.');
|
|
61
|
+
});
|
|
62
|
+
await close();
|
|
63
|
+
await contextClosed;
|
|
64
|
+
});
|
|
65
|
+
page.once('close', () => {
|
|
66
|
+
this.registerPageClosed(page);
|
|
66
67
|
});
|
|
67
68
|
return page;
|
|
68
69
|
}
|
|
@@ -74,7 +75,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
74
75
|
const page = await this.browser.newPage();
|
|
75
76
|
tryCancel();
|
|
76
77
|
page.once('close', () => {
|
|
77
|
-
this.
|
|
78
|
+
this.registerPageClosed(page);
|
|
78
79
|
});
|
|
79
80
|
return page;
|
|
80
81
|
}
|
|
@@ -84,7 +85,7 @@ export class PuppeteerController extends BrowserController {
|
|
|
84
85
|
async _kill() {
|
|
85
86
|
const browserProcess = this.browser.process();
|
|
86
87
|
if (!browserProcess) {
|
|
87
|
-
log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
|
|
88
|
+
this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
|
|
88
89
|
return;
|
|
89
90
|
}
|
|
90
91
|
const timeout = setTimeout(() => {
|
|
@@ -98,14 +99,22 @@ export class PuppeteerController extends BrowserController {
|
|
|
98
99
|
clearTimeout(timeout);
|
|
99
100
|
}
|
|
100
101
|
catch (error) {
|
|
101
|
-
log.debug('Browser was already killed.', { error });
|
|
102
|
+
this.log.debug('Browser was already killed.', { error });
|
|
102
103
|
}
|
|
103
104
|
}
|
|
104
105
|
async _getCookies(page) {
|
|
105
|
-
|
|
106
|
+
const cookies = await page.browserContext().cookies();
|
|
107
|
+
// Puppeteer 25+ can report `sameSite: 'Default'`, which means the attribute is unspecified.
|
|
108
|
+
return cookies.map(({ sameSite, ...rest }) => ({
|
|
109
|
+
...rest,
|
|
110
|
+
sameSite: sameSite === 'Default' ? undefined : sameSite,
|
|
111
|
+
}));
|
|
106
112
|
}
|
|
107
113
|
async _setCookies(page, cookies) {
|
|
108
|
-
|
|
114
|
+
// BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill
|
|
115
|
+
// the page's current URL for us. Replicate that so callers who pass neither don't get rejected.
|
|
116
|
+
const pageUrl = page.url();
|
|
117
|
+
const normalized = cookies.map((cookie) => cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl });
|
|
118
|
+
return page.browserContext().setCookie(...normalized);
|
|
109
119
|
}
|
|
110
120
|
}
|
|
111
|
-
//# sourceMappingURL=puppeteer-controller.js.map
|
|
@@ -6,10 +6,18 @@ import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
|
6
6
|
import type { LaunchContext } from '../launch-context.js';
|
|
7
7
|
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
8
8
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
9
|
+
export type PuppeteerLaunchContext = LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>;
|
|
9
10
|
export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
10
|
-
protected _launch(launchContext:
|
|
11
|
+
protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
12
|
+
/** @internal */
|
|
13
|
+
protected isOldPuppeteerVersion(): Promise<boolean>;
|
|
14
|
+
/**
|
|
15
|
+
* Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
|
|
16
|
+
* and {@link RemotePuppeteerPlugin}.
|
|
17
|
+
* @internal
|
|
18
|
+
*/
|
|
19
|
+
protected wrapBrowser(browser: PuppeteerTypes.Browser, launchContext: PuppeteerLaunchContext, oldPuppeteerVersion: boolean): PuppeteerTypes.Browser;
|
|
11
20
|
createController(): PuppeteerController;
|
|
12
|
-
protected
|
|
13
|
-
protected
|
|
21
|
+
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
22
|
+
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
|
14
23
|
}
|
|
15
|
-
//# sourceMappingURL=puppeteer-plugin.d.ts.map
|
|
@@ -1,23 +1,16 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
3
3
|
import { anonymizeProxySugar } from '../anonymize-proxy.js';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
4
|
import { noop } from '../utils.js';
|
|
6
5
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
7
6
|
const PROXY_SERVER_ARG = '--proxy-server=';
|
|
8
7
|
export class PuppeteerPlugin extends BrowserPlugin {
|
|
9
8
|
async _launch(launchContext) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
const version = +parsed.version.split('.')[0];
|
|
15
|
-
oldPuppeteerVersion = version < 22;
|
|
16
|
-
}
|
|
17
|
-
catch {
|
|
18
|
-
// ignore
|
|
9
|
+
const oldPuppeteerVersion = await this.isOldPuppeteerVersion();
|
|
10
|
+
const { launchOptions, userDataDir, experimentalContainers, proxyUrl } = launchContext;
|
|
11
|
+
if (experimentalContainers) {
|
|
12
|
+
throw new Error('Experimental containers are only available with Playwright');
|
|
19
13
|
}
|
|
20
|
-
const { launchOptions, userDataDir, useIncognitoPages, proxyUrl } = launchContext;
|
|
21
14
|
launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
|
|
22
15
|
if (launchOptions.headless === false) {
|
|
23
16
|
if (Array.isArray(launchOptions.args)) {
|
|
@@ -30,45 +23,71 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
30
23
|
if (launchOptions.headless === true && oldPuppeteerVersion) {
|
|
31
24
|
launchOptions.headless = 'new';
|
|
32
25
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
else {
|
|
42
|
-
launchOptions.args = [proxyArg];
|
|
43
|
-
}
|
|
26
|
+
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
|
|
27
|
+
ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
|
|
28
|
+
});
|
|
29
|
+
if (proxyUrl) {
|
|
30
|
+
const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
|
|
31
|
+
if (Array.isArray(launchOptions.args)) {
|
|
32
|
+
launchOptions.args.push(proxyArg);
|
|
44
33
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if (anonymizedProxyUrl) {
|
|
48
|
-
browser.on('disconnected', async () => {
|
|
49
|
-
await close();
|
|
50
|
-
});
|
|
51
|
-
}
|
|
34
|
+
else {
|
|
35
|
+
launchOptions.args = [proxyArg];
|
|
52
36
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
37
|
+
}
|
|
38
|
+
let browser;
|
|
39
|
+
try {
|
|
40
|
+
browser = await this.library.launch(launchOptions);
|
|
41
|
+
if (anonymizedProxyUrl) {
|
|
42
|
+
browser.on('disconnected', async () => {
|
|
43
|
+
await close();
|
|
44
|
+
});
|
|
56
45
|
}
|
|
57
46
|
}
|
|
58
|
-
|
|
47
|
+
catch (error) {
|
|
48
|
+
await close();
|
|
49
|
+
this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
|
|
50
|
+
}
|
|
51
|
+
return this.wrapBrowser(browser, launchContext, oldPuppeteerVersion);
|
|
52
|
+
}
|
|
53
|
+
/** @internal */
|
|
54
|
+
async isOldPuppeteerVersion() {
|
|
55
|
+
try {
|
|
56
|
+
const jsonPath = require.resolve('puppeteer/package.json');
|
|
57
|
+
const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
|
|
58
|
+
return +parsed.version.split('.')[0] < 22;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
|
|
66
|
+
* and {@link RemotePuppeteerPlugin}.
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
wrapBrowser(browser, launchContext, oldPuppeteerVersion) {
|
|
70
|
+
const { useIncognitoPages, proxyUrl, ignoreProxyCertificate, isRemote } = launchContext;
|
|
71
|
+
const targetCreatedHandler = async (target) => {
|
|
59
72
|
try {
|
|
60
73
|
const page = await target.page();
|
|
61
74
|
if (page) {
|
|
62
75
|
page.on('error', (error) => {
|
|
63
|
-
log.exception(error, 'Page crashed.');
|
|
76
|
+
this.log.exception(error, 'Page crashed.');
|
|
64
77
|
page.close().catch(noop);
|
|
65
78
|
});
|
|
66
79
|
}
|
|
67
80
|
}
|
|
68
81
|
catch (error) {
|
|
69
|
-
log.exception(error, 'Failed to retrieve page from target.');
|
|
82
|
+
this.log.exception(error, 'Failed to retrieve page from target.');
|
|
70
83
|
}
|
|
71
|
-
}
|
|
84
|
+
};
|
|
85
|
+
browser.on('targetcreated', targetCreatedHandler);
|
|
86
|
+
if (isRemote) {
|
|
87
|
+
browser.once('disconnected', () => {
|
|
88
|
+
browser.off('targetcreated', targetCreatedHandler);
|
|
89
|
+
});
|
|
90
|
+
}
|
|
72
91
|
const boundMethods = [
|
|
73
92
|
'newPage',
|
|
74
93
|
'close',
|
|
@@ -78,6 +97,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
78
97
|
'version',
|
|
79
98
|
'on',
|
|
80
99
|
'process',
|
|
100
|
+
'pages',
|
|
81
101
|
].reduce((map, method) => {
|
|
82
102
|
map[method] = browser[method]?.bind(browser);
|
|
83
103
|
return map;
|
|
@@ -89,22 +109,30 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
89
109
|
return async (...args) => {
|
|
90
110
|
let page;
|
|
91
111
|
if (useIncognitoPages) {
|
|
92
|
-
|
|
112
|
+
// Proxy is managed by the remote service.
|
|
113
|
+
const effectiveProxyUrl = isRemote ? undefined : proxyUrl;
|
|
114
|
+
const [anonymizedProxyUrl, close] = effectiveProxyUrl
|
|
115
|
+
? await anonymizeProxySugar(effectiveProxyUrl, undefined, undefined, {
|
|
116
|
+
ignoreProxyCertificate,
|
|
117
|
+
})
|
|
118
|
+
: [undefined, noop];
|
|
119
|
+
const proxyServer = anonymizedProxyUrl ?? effectiveProxyUrl;
|
|
120
|
+
const contextOptions = proxyServer ? { proxyServer } : {};
|
|
121
|
+
const context = (await browser[method](contextOptions));
|
|
93
122
|
try {
|
|
94
|
-
const context = (await browser[method]({
|
|
95
|
-
proxyServer: anonymizedProxyUrl ?? proxyUrl,
|
|
96
|
-
}));
|
|
97
123
|
page = await context.newPage(...args);
|
|
98
|
-
if (anonymizedProxyUrl) {
|
|
99
|
-
page.on('close', async () => {
|
|
100
|
-
await close();
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
124
|
}
|
|
104
125
|
catch (error) {
|
|
126
|
+
await context.close().catch(noop);
|
|
105
127
|
await close();
|
|
106
128
|
throw error;
|
|
107
129
|
}
|
|
130
|
+
page.once('close', async () => {
|
|
131
|
+
if (anonymizedProxyUrl) {
|
|
132
|
+
await close();
|
|
133
|
+
}
|
|
134
|
+
await context.close().catch(noop);
|
|
135
|
+
});
|
|
108
136
|
}
|
|
109
137
|
else {
|
|
110
138
|
page = await boundMethods.newPage(...args);
|
|
@@ -139,7 +167,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
139
167
|
createController() {
|
|
140
168
|
return new PuppeteerController(this);
|
|
141
169
|
}
|
|
142
|
-
async
|
|
170
|
+
async addProxyToLaunchOptions(_launchContext) {
|
|
143
171
|
/*
|
|
144
172
|
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
145
173
|
launchContext.launchOptions ??= {};
|
|
@@ -166,8 +194,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
166
194
|
}
|
|
167
195
|
*/
|
|
168
196
|
}
|
|
169
|
-
|
|
197
|
+
isChromiumBasedBrowser(_launchContext) {
|
|
170
198
|
return true;
|
|
171
199
|
}
|
|
172
200
|
}
|
|
173
|
-
//# sourceMappingURL=puppeteer-plugin.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
+
import type Puppeteer from 'puppeteer';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type * as PuppeteerTypes from 'puppeteer';
|
|
5
|
+
import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
|
|
6
|
+
import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
|
|
7
|
+
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
8
|
+
import { type PuppeteerLaunchContext, PuppeteerPlugin } from './puppeteer-plugin.js';
|
|
9
|
+
export interface RemotePuppeteerPluginOptions extends BrowserPluginOptions<PuppeteerTypes.LaunchOptions>, RemoteBrowserOptions {
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
13
|
+
* over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
14
|
+
*/
|
|
15
|
+
export declare class RemotePuppeteerPlugin extends PuppeteerPlugin {
|
|
16
|
+
#private;
|
|
17
|
+
constructor(library: typeof Puppeteer, options: RemotePuppeteerPluginOptions);
|
|
18
|
+
createLaunchContext(options?: CreateLaunchContextOptions<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): PuppeteerLaunchContext;
|
|
19
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
20
|
+
launch(launchContext?: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
21
|
+
protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { connectRemoteBrowser } from '../remote-browser-plugin.js';
|
|
2
|
+
import { PuppeteerPlugin } from './puppeteer-plugin.js';
|
|
3
|
+
/**
|
|
4
|
+
* A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
5
|
+
* over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
6
|
+
*/
|
|
7
|
+
export class RemotePuppeteerPlugin extends PuppeteerPlugin {
|
|
8
|
+
#remote;
|
|
9
|
+
constructor(library, options) {
|
|
10
|
+
super(library, options);
|
|
11
|
+
this.#remote = options;
|
|
12
|
+
if (!this.useIncognitoPages) {
|
|
13
|
+
this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
|
|
14
|
+
'browser instance (useIncognitoPages defaults to false).');
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
createLaunchContext(options = {}) {
|
|
18
|
+
return super.createLaunchContext({ ...options, isRemote: true });
|
|
19
|
+
}
|
|
20
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
21
|
+
async launch(launchContext = this.createLaunchContext()) {
|
|
22
|
+
return this._launch(launchContext);
|
|
23
|
+
}
|
|
24
|
+
async _launch(launchContext) {
|
|
25
|
+
const browser = await connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
|
|
26
|
+
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
27
|
+
return this.library.connect({ ...this.#remote.connectOptions, browserWSEndpoint: url });
|
|
28
|
+
});
|
|
29
|
+
return this.wrapBrowser(browser, launchContext, await this.isOldPuppeteerVersion());
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
3
|
+
/**
|
|
4
|
+
* The result of resolving a remote browser endpoint: the URL to connect to plus an optional opaque
|
|
5
|
+
* `context` object that is handed back to `release`.
|
|
6
|
+
*/
|
|
7
|
+
export interface ResolvedRemoteEndpoint {
|
|
8
|
+
/** The WebSocket / CDP URL to connect to. */
|
|
9
|
+
url: string;
|
|
10
|
+
/** Opaque per-session data (e.g. a session ID) passed back to `release`. */
|
|
11
|
+
context?: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A remote browser endpoint: either a static URL string, or a function called once per browser launch
|
|
15
|
+
* that returns the URL (optionally with a `context` for `release`). The function receives the proxy URL
|
|
16
|
+
* Crawlee resolved for the browser, so it can be forwarded to services that accept an external proxy.
|
|
17
|
+
*/
|
|
18
|
+
export type RemoteBrowserEndpoint = string | ((options?: {
|
|
19
|
+
proxyUrl?: string;
|
|
20
|
+
}) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);
|
|
21
|
+
/** Options shared by all remote browser plugins. */
|
|
22
|
+
export interface RemoteBrowserOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The remote browser endpoint: a static URL, a function returning one per launch, or a
|
|
25
|
+
* {@link RemoteBrowserProvider} instance encapsulating a session create/release lifecycle.
|
|
26
|
+
*/
|
|
27
|
+
endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;
|
|
28
|
+
/**
|
|
29
|
+
* Cleanup callback invoked when the browser disconnects or the connection fails right after the endpoint
|
|
30
|
+
* resolved. Receives the `context` returned by a function endpoint. Errors are caught and logged. Ignored
|
|
31
|
+
* when `endpoint` is a {@link RemoteBrowserProvider} (its own `release()` is used instead).
|
|
32
|
+
*/
|
|
33
|
+
release?: (info: {
|
|
34
|
+
endpoint: string;
|
|
35
|
+
context?: Record<string, unknown>;
|
|
36
|
+
}) => unknown;
|
|
37
|
+
/** Extra options forwarded to the library `connect()` call (endpoint excluded). */
|
|
38
|
+
connectOptions?: Record<string, unknown>;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Resolves `endpoint` for one launch, runs the library-specific `connect`, and ties the session's `release` to
|
|
42
|
+
* the browser's `disconnected` event. On connect failure the session is released and the error is wrapped in a
|
|
43
|
+
* {@link BrowserLaunchError}.
|
|
44
|
+
*
|
|
45
|
+
* @internal
|
|
46
|
+
*/
|
|
47
|
+
export declare function connectRemoteBrowser<Browser extends {
|
|
48
|
+
on(event: 'disconnected', listener: () => void): unknown;
|
|
49
|
+
}>(options: RemoteBrowserOptions, proxyUrl: string | undefined, log: CrawleeLogger, connect: (url: string) => Promise<Browser>): Promise<Browser>;
|