@crawlee/browser-pool 4.0.0-beta.25 → 4.0.0-beta.250

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +15 -15
  2. package/abstract-classes/browser-controller.d.ts +100 -29
  3. package/abstract-classes/browser-controller.js +70 -24
  4. package/abstract-classes/browser-plugin.d.ts +25 -22
  5. package/abstract-classes/browser-plugin.js +21 -13
  6. package/anonymize-proxy.d.ts +6 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +110 -51
  9. package/browser-pool.js +318 -147
  10. package/events.d.ts +2 -4
  11. package/events.js +0 -2
  12. package/fingerprinting/hooks.d.ts +0 -1
  13. package/fingerprinting/hooks.js +26 -10
  14. package/fingerprinting/types.d.ts +3 -34
  15. package/fingerprinting/types.js +0 -2
  16. package/fingerprinting/utils.d.ts +0 -1
  17. package/fingerprinting/utils.js +5 -6
  18. package/index.d.ts +15 -7
  19. package/index.js +6 -3
  20. package/launch-context.d.ts +15 -5
  21. package/launch-context.js +15 -10
  22. package/package.json +15 -15
  23. package/playwright/playwright-browser.d.ts +3 -6
  24. package/playwright/playwright-browser.js +17 -17
  25. package/playwright/playwright-controller.d.ts +1 -1
  26. package/playwright/playwright-controller.js +10 -6
  27. package/playwright/playwright-plugin.d.ts +3 -7
  28. package/playwright/playwright-plugin.js +18 -17
  29. package/playwright/remote-playwright-plugin.d.ts +29 -0
  30. package/playwright/remote-playwright-plugin.js +38 -0
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +1 -1
  34. package/puppeteer/puppeteer-controller.js +29 -20
  35. package/puppeteer/puppeteer-plugin.d.ts +12 -4
  36. package/puppeteer/puppeteer-plugin.js +75 -48
  37. package/puppeteer/remote-puppeteer-plugin.d.ts +22 -0
  38. package/puppeteer/remote-puppeteer-plugin.js +31 -0
  39. package/remote-browser-plugin.d.ts +49 -0
  40. package/remote-browser-plugin.js +63 -0
  41. package/remote-browser-provider.d.ts +81 -0
  42. package/remote-browser-provider.js +65 -0
  43. package/utils.d.ts +7 -1
  44. package/utils.js +19 -1
  45. package/abstract-classes/browser-controller.d.ts.map +0 -1
  46. package/abstract-classes/browser-controller.js.map +0 -1
  47. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  48. package/abstract-classes/browser-plugin.js.map +0 -1
  49. package/anonymize-proxy.d.ts.map +0 -1
  50. package/anonymize-proxy.js.map +0 -1
  51. package/browser-pool.d.ts.map +0 -1
  52. package/browser-pool.js.map +0 -1
  53. package/container-proxy-server.d.ts +0 -12
  54. package/container-proxy-server.d.ts.map +0 -1
  55. package/container-proxy-server.js +0 -43
  56. package/container-proxy-server.js.map +0 -1
  57. package/events.d.ts.map +0 -1
  58. package/events.js.map +0 -1
  59. package/fingerprinting/hooks.d.ts.map +0 -1
  60. package/fingerprinting/hooks.js.map +0 -1
  61. package/fingerprinting/types.d.ts.map +0 -1
  62. package/fingerprinting/types.js.map +0 -1
  63. package/fingerprinting/utils.d.ts.map +0 -1
  64. package/fingerprinting/utils.js.map +0 -1
  65. package/index.d.ts.map +0 -1
  66. package/index.js.map +0 -1
  67. package/launch-context.d.ts.map +0 -1
  68. package/launch-context.js.map +0 -1
  69. package/logger.d.ts +0 -3
  70. package/logger.d.ts.map +0 -1
  71. package/logger.js +0 -5
  72. package/logger.js.map +0 -1
  73. package/playwright/playwright-browser.d.ts.map +0 -1
  74. package/playwright/playwright-browser.js.map +0 -1
  75. package/playwright/playwright-controller.d.ts.map +0 -1
  76. package/playwright/playwright-controller.js.map +0 -1
  77. package/playwright/playwright-plugin.d.ts.map +0 -1
  78. package/playwright/playwright-plugin.js.map +0 -1
  79. package/proxy-server.d.ts.map +0 -1
  80. package/proxy-server.js.map +0 -1
  81. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  82. package/puppeteer/puppeteer-controller.js.map +0 -1
  83. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  84. package/puppeteer/puppeteer-plugin.js.map +0 -1
  85. package/utils.d.ts.map +0 -1
  86. package/utils.js.map +0 -1
@@ -1,13 +1,11 @@
1
1
  import fs from 'node:fs';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { getLocalProxyAddress } from '../proxy-server.js';
6
5
  import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
7
6
  import { PlaywrightController } from './playwright-controller.js';
8
7
  export class PlaywrightPlugin extends BrowserPlugin {
9
- _browserVersion;
10
- _containerProxyServer;
8
+ #browserVersion;
11
9
  async _launch(launchContext) {
12
10
  const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
13
11
  let browser;
@@ -20,7 +18,9 @@ export class PlaywrightPlugin extends BrowserPlugin {
20
18
  if (this.library.name() === 'webkit') {
21
19
  launchOptions.args = launchOptions.args?.filter((arg) => arg !== '--no-sandbox');
22
20
  }
23
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
21
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
22
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
23
+ });
24
24
  if (anonymizedProxyUrl) {
25
25
  launchOptions.proxy = {
26
26
  server: anonymizedProxyUrl,
@@ -30,7 +30,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
30
30
  try {
31
31
  if (useIncognitoPages) {
32
32
  browser = await this.library.launch(launchOptions).catch((error) => {
33
- return this._throwOnFailedLaunch(launchContext, error);
33
+ return this.#throwOnFailedLaunch(launchContext, error);
34
34
  });
35
35
  if (anonymizedProxyUrl) {
36
36
  browser.on('disconnected', async () => {
@@ -42,7 +42,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
42
42
  const browserContext = await this.library
43
43
  .launchPersistentContext(userDataDir, launchOptions)
44
44
  .catch((error) => {
45
- return this._throwOnFailedLaunch(launchContext, error);
45
+ return this.#throwOnFailedLaunch(launchContext, error);
46
46
  });
47
47
  browserContext.once('close', () => {
48
48
  if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -57,18 +57,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
57
57
  await close();
58
58
  });
59
59
  }
60
- if (!this._browserVersion) {
60
+ if (!this.#browserVersion) {
61
61
  // Launches unused browser just to get the browser version.
62
62
  const inactiveBrowser = await this.library.launch(launchOptions);
63
- this._browserVersion = inactiveBrowser.version();
63
+ this.#browserVersion = inactiveBrowser.version();
64
64
  inactiveBrowser.close().catch((error) => {
65
- log.exception(error, 'Failed to close browser.');
65
+ this.log.exception(error, 'Failed to close browser.');
66
66
  });
67
67
  }
68
- browser = new PlaywrightBrowserWithPersistentContext({
68
+ const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
69
69
  browserContext,
70
- version: this._browserVersion,
70
+ version: this.#browserVersion,
71
71
  });
72
+ persistentBrowser.setBrowserType(this.library);
73
+ browser = persistentBrowser;
72
74
  }
73
75
  }
74
76
  catch (error) {
@@ -77,27 +79,26 @@ export class PlaywrightPlugin extends BrowserPlugin {
77
79
  }
78
80
  return browser;
79
81
  }
80
- _throwOnFailedLaunch(launchContext, cause) {
81
- this._throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
82
+ #throwOnFailedLaunch(launchContext, cause) {
83
+ this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
82
84
  }
83
85
  createController() {
84
86
  return new PlaywrightController(this);
85
87
  }
86
- async _addProxyToLaunchOptions(launchContext) {
88
+ async addProxyToLaunchOptions(launchContext) {
87
89
  launchContext.launchOptions ??= {};
88
90
  const { launchOptions, proxyUrl } = launchContext;
89
91
  if (proxyUrl) {
90
92
  const url = new URL(proxyUrl);
91
93
  launchOptions.proxy = {
92
- server: url.origin,
94
+ server: `${url.protocol}//${url.host}`,
93
95
  username: decodeURIComponent(url.username),
94
96
  password: decodeURIComponent(url.password),
95
97
  };
96
98
  }
97
99
  }
98
- _isChromiumBasedBrowser() {
100
+ isChromiumBasedBrowser() {
99
101
  const name = this.library.name();
100
102
  return name === 'chromium';
101
103
  }
102
104
  }
103
- //# sourceMappingURL=playwright-plugin.js.map
@@ -0,0 +1,29 @@
1
+ // @ts-ignore optional peer dependency or compatibility with es2022
2
+ import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
3
+ import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
4
+ import type { LaunchContext } from '../launch-context.js';
5
+ import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
6
+ import type { SafeParameters } from '../utils.js';
7
+ import { PlaywrightPlugin } from './playwright-plugin.js';
8
+ export interface RemotePlaywrightPluginOptions extends BrowserPluginOptions<SafeParameters<BrowserType['launch']>[0]>, RemoteBrowserOptions {
9
+ /**
10
+ * Which protocol to connect with. `'cdp'` uses `connectOverCDP()` (the default), `'playwright'` uses
11
+ * `connect()` (Playwright's own WebSocket protocol, for endpoints started with `launchServer()`).
12
+ */
13
+ protocol?: 'cdp' | 'playwright';
14
+ }
15
+ /**
16
+ * A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
17
+ * instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
18
+ *
19
+ * Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
20
+ * contexts), so `useIncognitoPages` is always `true`.
21
+ */
22
+ export declare class RemotePlaywrightPlugin extends PlaywrightPlugin {
23
+ #private;
24
+ constructor(library: BrowserType, options: RemotePlaywrightPluginOptions);
25
+ createLaunchContext(options?: CreateLaunchContextOptions<BrowserType>): LaunchContext<BrowserType>;
26
+ /** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
27
+ launch(launchContext?: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
28
+ protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
29
+ }
@@ -0,0 +1,38 @@
1
+ import { connectRemoteBrowser } from '../remote-browser-plugin.js';
2
+ import { PlaywrightPlugin } from './playwright-plugin.js';
3
+ /**
4
+ * A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
5
+ * instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
6
+ *
7
+ * Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
8
+ * contexts), so `useIncognitoPages` is always `true`.
9
+ */
10
+ export class RemotePlaywrightPlugin extends PlaywrightPlugin {
11
+ #remote;
12
+ constructor(library, options) {
13
+ super(library, { ...options, useIncognitoPages: true });
14
+ this.#remote = options;
15
+ if (options.useIncognitoPages === false) {
16
+ this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
17
+ 'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
18
+ }
19
+ }
20
+ createLaunchContext(options = {}) {
21
+ return super.createLaunchContext({ ...options, isRemote: true });
22
+ }
23
+ /** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
24
+ async launch(launchContext = this.createLaunchContext()) {
25
+ return this._launch(launchContext);
26
+ }
27
+ async _launch(launchContext) {
28
+ const { protocol, connectOptions = {} } = this.#remote;
29
+ return connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
30
+ if (protocol === 'playwright') {
31
+ this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
32
+ return this.library.connect(url, connectOptions);
33
+ }
34
+ this.log.info('Connecting to remote browser via connectOverCDP.');
35
+ return this.library.connectOverCDP(url, connectOptions);
36
+ });
37
+ }
38
+ }
package/proxy-server.d.ts CHANGED
@@ -1,2 +1 @@
1
1
  export declare function getLocalProxyAddress(): Promise<string>;
2
- //# sourceMappingURL=proxy-server.d.ts.map
package/proxy-server.js CHANGED
@@ -10,4 +10,3 @@ export async function getLocalProxyAddress() {
10
10
  await listenPromise;
11
11
  return `http://127.0.0.1:${server.port}`;
12
12
  }
13
- //# sourceMappingURL=proxy-server.js.map
@@ -9,6 +9,7 @@ export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOp
9
9
  proxyPassword?: string;
10
10
  }
11
11
  export declare class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
12
+ /** @internal */
12
13
  normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
13
14
  protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
14
15
  protected _close(): Promise<void>;
@@ -16,4 +17,3 @@ export declare class PuppeteerController extends BrowserController<typeof Puppet
16
17
  protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
17
18
  protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
18
19
  }
19
- //# sourceMappingURL=puppeteer-controller.d.ts.map
@@ -1,9 +1,9 @@
1
1
  import { tryCancel } from '@apify/timeout';
2
2
  import { BrowserController } from '../abstract-classes/browser-controller.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
5
  export class PuppeteerController extends BrowserController {
6
+ /** @internal */
7
7
  normalizeProxyOptions(proxyUrl, pageOptions) {
8
8
  if (!proxyUrl) {
9
9
  return {};
@@ -12,7 +12,7 @@ export class PuppeteerController extends BrowserController {
12
12
  const username = decodeURIComponent(url.username);
13
13
  const password = decodeURIComponent(url.password);
14
14
  return {
15
- proxyServer: url.origin,
15
+ proxyServer: `${url.protocol}//${url.host}`,
16
16
  proxyUsername: username,
17
17
  proxyPassword: password,
18
18
  proxyBypassList: pageOptions?.proxyBypassList,
@@ -25,7 +25,7 @@ export class PuppeteerController extends BrowserController {
25
25
  }
26
26
  let close = async () => { };
27
27
  if (contextOptions.proxyServer) {
28
- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
28
+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
29
29
  if (anonymizedProxyUrl) {
30
30
  contextOptions.proxyServer = anonymizedProxyUrl;
31
31
  delete contextOptions.proxyUsername;
@@ -52,17 +52,18 @@ export class PuppeteerController extends BrowserController {
52
52
  tryCancel();
53
53
  }
54
54
  */
55
- page.once('close', async () => {
56
- this.activePages--;
57
- try {
58
- await context.close();
59
- }
60
- catch (error) {
61
- log.exception(error, 'Failed to close context.');
62
- }
63
- finally {
64
- await close();
65
- }
55
+ this.registerPageTeardown(page, async () => {
56
+ // The proxy server is not chained behind the context close: that can hang on
57
+ // the same target the page hung on, and the proxy runs in this process, so it
58
+ // has to be reclaimed either way.
59
+ const contextClosed = context.close().catch((error) => {
60
+ this.log.exception(error, 'Failed to close context.');
61
+ });
62
+ await close();
63
+ await contextClosed;
64
+ });
65
+ page.once('close', () => {
66
+ this.registerPageClosed(page);
66
67
  });
67
68
  return page;
68
69
  }
@@ -74,7 +75,7 @@ export class PuppeteerController extends BrowserController {
74
75
  const page = await this.browser.newPage();
75
76
  tryCancel();
76
77
  page.once('close', () => {
77
- this.activePages--;
78
+ this.registerPageClosed(page);
78
79
  });
79
80
  return page;
80
81
  }
@@ -84,7 +85,7 @@ export class PuppeteerController extends BrowserController {
84
85
  async _kill() {
85
86
  const browserProcess = this.browser.process();
86
87
  if (!browserProcess) {
87
- log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
88
+ this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
88
89
  return;
89
90
  }
90
91
  const timeout = setTimeout(() => {
@@ -98,14 +99,22 @@ export class PuppeteerController extends BrowserController {
98
99
  clearTimeout(timeout);
99
100
  }
100
101
  catch (error) {
101
- log.debug('Browser was already killed.', { error });
102
+ this.log.debug('Browser was already killed.', { error });
102
103
  }
103
104
  }
104
105
  async _getCookies(page) {
105
- return page.cookies();
106
+ const cookies = await page.browserContext().cookies();
107
+ // Puppeteer 25+ can report `sameSite: 'Default'`, which means the attribute is unspecified.
108
+ return cookies.map(({ sameSite, ...rest }) => ({
109
+ ...rest,
110
+ sameSite: sameSite === 'Default' ? undefined : sameSite,
111
+ }));
106
112
  }
107
113
  async _setCookies(page, cookies) {
108
- return page.setCookie(...cookies);
114
+ // BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill
115
+ // the page's current URL for us. Replicate that so callers who pass neither don't get rejected.
116
+ const pageUrl = page.url();
117
+ const normalized = cookies.map((cookie) => cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl });
118
+ return page.browserContext().setCookie(...normalized);
109
119
  }
110
120
  }
111
- //# sourceMappingURL=puppeteer-controller.js.map
@@ -6,10 +6,18 @@ import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
6
6
  import type { LaunchContext } from '../launch-context.js';
7
7
  import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
8
8
  import { PuppeteerController } from './puppeteer-controller.js';
9
+ export type PuppeteerLaunchContext = LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>;
9
10
  export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
10
- protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
11
+ protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
12
+ /** @internal */
13
+ protected isOldPuppeteerVersion(): Promise<boolean>;
14
+ /**
15
+ * Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
16
+ * and {@link RemotePuppeteerPlugin}.
17
+ * @internal
18
+ */
19
+ protected wrapBrowser(browser: PuppeteerTypes.Browser, launchContext: PuppeteerLaunchContext, oldPuppeteerVersion: boolean): PuppeteerTypes.Browser;
11
20
  createController(): PuppeteerController;
12
- protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
13
- protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
21
+ protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
22
+ protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
14
23
  }
15
- //# sourceMappingURL=puppeteer-plugin.d.ts.map
@@ -1,23 +1,16 @@
1
1
  import { readFile } from 'node:fs/promises';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { noop } from '../utils.js';
6
5
  import { PuppeteerController } from './puppeteer-controller.js';
7
6
  const PROXY_SERVER_ARG = '--proxy-server=';
8
7
  export class PuppeteerPlugin extends BrowserPlugin {
9
8
  async _launch(launchContext) {
10
- let oldPuppeteerVersion = false;
11
- try {
12
- const jsonPath = require.resolve('puppeteer/package.json');
13
- const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
14
- const version = +parsed.version.split('.')[0];
15
- oldPuppeteerVersion = version < 22;
16
- }
17
- catch {
18
- // ignore
9
+ const oldPuppeteerVersion = await this.isOldPuppeteerVersion();
10
+ const { launchOptions, userDataDir, experimentalContainers, proxyUrl } = launchContext;
11
+ if (experimentalContainers) {
12
+ throw new Error('Experimental containers are only available with Playwright');
19
13
  }
20
- const { launchOptions, userDataDir, useIncognitoPages, proxyUrl } = launchContext;
21
14
  launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
22
15
  if (launchOptions.headless === false) {
23
16
  if (Array.isArray(launchOptions.args)) {
@@ -30,45 +23,71 @@ export class PuppeteerPlugin extends BrowserPlugin {
30
23
  if (launchOptions.headless === true && oldPuppeteerVersion) {
31
24
  launchOptions.headless = 'new';
32
25
  }
33
- let browser;
34
- {
35
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
36
- if (proxyUrl) {
37
- const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
38
- if (Array.isArray(launchOptions.args)) {
39
- launchOptions.args.push(proxyArg);
40
- }
41
- else {
42
- launchOptions.args = [proxyArg];
43
- }
26
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
27
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
28
+ });
29
+ if (proxyUrl) {
30
+ const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
31
+ if (Array.isArray(launchOptions.args)) {
32
+ launchOptions.args.push(proxyArg);
44
33
  }
45
- try {
46
- browser = await this.library.launch(launchOptions);
47
- if (anonymizedProxyUrl) {
48
- browser.on('disconnected', async () => {
49
- await close();
50
- });
51
- }
34
+ else {
35
+ launchOptions.args = [proxyArg];
52
36
  }
53
- catch (error) {
54
- await close();
55
- this._throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
37
+ }
38
+ let browser;
39
+ try {
40
+ browser = await this.library.launch(launchOptions);
41
+ if (anonymizedProxyUrl) {
42
+ browser.on('disconnected', async () => {
43
+ await close();
44
+ });
56
45
  }
57
46
  }
58
- browser.on('targetcreated', async (target) => {
47
+ catch (error) {
48
+ await close();
49
+ this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
50
+ }
51
+ return this.wrapBrowser(browser, launchContext, oldPuppeteerVersion);
52
+ }
53
+ /** @internal */
54
+ async isOldPuppeteerVersion() {
55
+ try {
56
+ const jsonPath = require.resolve('puppeteer/package.json');
57
+ const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
58
+ return +parsed.version.split('.')[0] < 22;
59
+ }
60
+ catch {
61
+ return false;
62
+ }
63
+ }
64
+ /**
65
+ * Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
66
+ * and {@link RemotePuppeteerPlugin}.
67
+ * @internal
68
+ */
69
+ wrapBrowser(browser, launchContext, oldPuppeteerVersion) {
70
+ const { useIncognitoPages, proxyUrl, ignoreProxyCertificate, isRemote } = launchContext;
71
+ const targetCreatedHandler = async (target) => {
59
72
  try {
60
73
  const page = await target.page();
61
74
  if (page) {
62
75
  page.on('error', (error) => {
63
- log.exception(error, 'Page crashed.');
76
+ this.log.exception(error, 'Page crashed.');
64
77
  page.close().catch(noop);
65
78
  });
66
79
  }
67
80
  }
68
81
  catch (error) {
69
- log.exception(error, 'Failed to retrieve page from target.');
82
+ this.log.exception(error, 'Failed to retrieve page from target.');
70
83
  }
71
- });
84
+ };
85
+ browser.on('targetcreated', targetCreatedHandler);
86
+ if (isRemote) {
87
+ browser.once('disconnected', () => {
88
+ browser.off('targetcreated', targetCreatedHandler);
89
+ });
90
+ }
72
91
  const boundMethods = [
73
92
  'newPage',
74
93
  'close',
@@ -78,6 +97,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
78
97
  'version',
79
98
  'on',
80
99
  'process',
100
+ 'pages',
81
101
  ].reduce((map, method) => {
82
102
  map[method] = browser[method]?.bind(browser);
83
103
  return map;
@@ -89,22 +109,30 @@ export class PuppeteerPlugin extends BrowserPlugin {
89
109
  return async (...args) => {
90
110
  let page;
91
111
  if (useIncognitoPages) {
92
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
112
+ // Proxy is managed by the remote service.
113
+ const effectiveProxyUrl = isRemote ? undefined : proxyUrl;
114
+ const [anonymizedProxyUrl, close] = effectiveProxyUrl
115
+ ? await anonymizeProxySugar(effectiveProxyUrl, undefined, undefined, {
116
+ ignoreProxyCertificate,
117
+ })
118
+ : [undefined, noop];
119
+ const proxyServer = anonymizedProxyUrl ?? effectiveProxyUrl;
120
+ const contextOptions = proxyServer ? { proxyServer } : {};
121
+ const context = (await browser[method](contextOptions));
93
122
  try {
94
- const context = (await browser[method]({
95
- proxyServer: anonymizedProxyUrl ?? proxyUrl,
96
- }));
97
123
  page = await context.newPage(...args);
98
- if (anonymizedProxyUrl) {
99
- page.on('close', async () => {
100
- await close();
101
- });
102
- }
103
124
  }
104
125
  catch (error) {
126
+ await context.close().catch(noop);
105
127
  await close();
106
128
  throw error;
107
129
  }
130
+ page.once('close', async () => {
131
+ if (anonymizedProxyUrl) {
132
+ await close();
133
+ }
134
+ await context.close().catch(noop);
135
+ });
108
136
  }
109
137
  else {
110
138
  page = await boundMethods.newPage(...args);
@@ -139,7 +167,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
139
167
  createController() {
140
168
  return new PuppeteerController(this);
141
169
  }
142
- async _addProxyToLaunchOptions(_launchContext) {
170
+ async addProxyToLaunchOptions(_launchContext) {
143
171
  /*
144
172
  // DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
145
173
  launchContext.launchOptions ??= {};
@@ -166,8 +194,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
166
194
  }
167
195
  */
168
196
  }
169
- _isChromiumBasedBrowser(_launchContext) {
197
+ isChromiumBasedBrowser(_launchContext) {
170
198
  return true;
171
199
  }
172
200
  }
173
- //# sourceMappingURL=puppeteer-plugin.js.map
@@ -0,0 +1,22 @@
1
+ // @ts-ignore optional peer dependency or compatibility with es2022
2
+ import type Puppeteer from 'puppeteer';
3
+ // @ts-ignore optional peer dependency or compatibility with es2022
4
+ import type * as PuppeteerTypes from 'puppeteer';
5
+ import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
6
+ import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
7
+ import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
8
+ import { type PuppeteerLaunchContext, PuppeteerPlugin } from './puppeteer-plugin.js';
9
+ export interface RemotePuppeteerPluginOptions extends BrowserPluginOptions<PuppeteerTypes.LaunchOptions>, RemoteBrowserOptions {
10
+ }
11
+ /**
12
+ * A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
13
+ * over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
14
+ */
15
+ export declare class RemotePuppeteerPlugin extends PuppeteerPlugin {
16
+ #private;
17
+ constructor(library: typeof Puppeteer, options: RemotePuppeteerPluginOptions);
18
+ createLaunchContext(options?: CreateLaunchContextOptions<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): PuppeteerLaunchContext;
19
+ /** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
20
+ launch(launchContext?: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
21
+ protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
22
+ }
@@ -0,0 +1,31 @@
1
+ import { connectRemoteBrowser } from '../remote-browser-plugin.js';
2
+ import { PuppeteerPlugin } from './puppeteer-plugin.js';
3
+ /**
4
+ * A {@link PuppeteerPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
5
+ * over CDP instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
6
+ */
7
+ export class RemotePuppeteerPlugin extends PuppeteerPlugin {
8
+ #remote;
9
+ constructor(library, options) {
10
+ super(library, options);
11
+ this.#remote = options;
12
+ if (!this.useIncognitoPages) {
13
+ this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
14
+ 'browser instance (useIncognitoPages defaults to false).');
15
+ }
16
+ }
17
+ createLaunchContext(options = {}) {
18
+ return super.createLaunchContext({ ...options, isRemote: true });
19
+ }
20
+ /** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
21
+ async launch(launchContext = this.createLaunchContext()) {
22
+ return this._launch(launchContext);
23
+ }
24
+ async _launch(launchContext) {
25
+ const browser = await connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
26
+ this.log.info('Connecting to remote browser via connect (CDP).');
27
+ return this.library.connect({ ...this.#remote.connectOptions, browserWSEndpoint: url });
28
+ });
29
+ return this.wrapBrowser(browser, launchContext, await this.isOldPuppeteerVersion());
30
+ }
31
+ }
@@ -0,0 +1,49 @@
1
+ import type { CrawleeLogger } from '@crawlee/core';
2
+ import { RemoteBrowserProvider } from './remote-browser-provider.js';
3
+ /**
4
+ * The result of resolving a remote browser endpoint: the URL to connect to plus an optional opaque
5
+ * `context` object that is handed back to `release`.
6
+ */
7
+ export interface ResolvedRemoteEndpoint {
8
+ /** The WebSocket / CDP URL to connect to. */
9
+ url: string;
10
+ /** Opaque per-session data (e.g. a session ID) passed back to `release`. */
11
+ context?: Record<string, unknown>;
12
+ }
13
+ /**
14
+ * A remote browser endpoint: either a static URL string, or a function called once per browser launch
15
+ * that returns the URL (optionally with a `context` for `release`). The function receives the proxy URL
16
+ * Crawlee resolved for the browser, so it can be forwarded to services that accept an external proxy.
17
+ */
18
+ export type RemoteBrowserEndpoint = string | ((options?: {
19
+ proxyUrl?: string;
20
+ }) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);
21
+ /** Options shared by all remote browser plugins. */
22
+ export interface RemoteBrowserOptions {
23
+ /**
24
+ * The remote browser endpoint: a static URL, a function returning one per launch, or a
25
+ * {@link RemoteBrowserProvider} instance encapsulating a session create/release lifecycle.
26
+ */
27
+ endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;
28
+ /**
29
+ * Cleanup callback invoked when the browser disconnects or the connection fails right after the endpoint
30
+ * resolved. Receives the `context` returned by a function endpoint. Errors are caught and logged. Ignored
31
+ * when `endpoint` is a {@link RemoteBrowserProvider} (its own `release()` is used instead).
32
+ */
33
+ release?: (info: {
34
+ endpoint: string;
35
+ context?: Record<string, unknown>;
36
+ }) => unknown;
37
+ /** Extra options forwarded to the library `connect()` call (endpoint excluded). */
38
+ connectOptions?: Record<string, unknown>;
39
+ }
40
+ /**
41
+ * Resolves `endpoint` for one launch, runs the library-specific `connect`, and ties the session's `release` to
42
+ * the browser's `disconnected` event. On connect failure the session is released and the error is wrapped in a
43
+ * {@link BrowserLaunchError}.
44
+ *
45
+ * @internal
46
+ */
47
+ export declare function connectRemoteBrowser<Browser extends {
48
+ on(event: 'disconnected', listener: () => void): unknown;
49
+ }>(options: RemoteBrowserOptions, proxyUrl: string | undefined, log: CrawleeLogger, connect: (url: string) => Promise<Browser>): Promise<Browser>;