@crawlee/browser-pool 4.0.0-beta.21 → 4.0.0-beta.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +14 -14
  2. package/abstract-classes/browser-controller.d.ts +100 -29
  3. package/abstract-classes/browser-controller.js +70 -24
  4. package/abstract-classes/browser-plugin.d.ts +54 -23
  5. package/abstract-classes/browser-plugin.js +75 -13
  6. package/anonymize-proxy.d.ts +6 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +123 -49
  9. package/browser-pool.js +320 -147
  10. package/events.d.ts +2 -4
  11. package/events.js +0 -2
  12. package/fingerprinting/hooks.d.ts +0 -1
  13. package/fingerprinting/hooks.js +26 -10
  14. package/fingerprinting/types.d.ts +3 -34
  15. package/fingerprinting/types.js +0 -2
  16. package/fingerprinting/utils.d.ts +0 -1
  17. package/fingerprinting/utils.js +5 -6
  18. package/index.d.ts +14 -7
  19. package/index.js +5 -3
  20. package/launch-context.d.ts +21 -5
  21. package/launch-context.js +21 -10
  22. package/package.json +9 -9
  23. package/playwright/playwright-browser.d.ts +3 -6
  24. package/playwright/playwright-browser.js +17 -17
  25. package/playwright/playwright-controller.d.ts +1 -1
  26. package/playwright/playwright-controller.js +10 -6
  27. package/playwright/playwright-plugin.d.ts +12 -7
  28. package/playwright/playwright-plugin.js +42 -16
  29. package/proxy-server.d.ts +0 -1
  30. package/proxy-server.js +0 -1
  31. package/puppeteer/puppeteer-controller.d.ts +1 -1
  32. package/puppeteer/puppeteer-controller.js +29 -20
  33. package/puppeteer/puppeteer-plugin.d.ts +9 -3
  34. package/puppeteer/puppeteer-plugin.js +87 -46
  35. package/remote-browser-pool.d.ts +178 -0
  36. package/remote-browser-pool.js +206 -0
  37. package/remote-browser-provider.d.ts +83 -0
  38. package/remote-browser-provider.js +67 -0
  39. package/utils.d.ts +7 -1
  40. package/utils.js +19 -1
  41. package/abstract-classes/browser-controller.d.ts.map +0 -1
  42. package/abstract-classes/browser-controller.js.map +0 -1
  43. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  44. package/abstract-classes/browser-plugin.js.map +0 -1
  45. package/anonymize-proxy.d.ts.map +0 -1
  46. package/anonymize-proxy.js.map +0 -1
  47. package/browser-pool.d.ts.map +0 -1
  48. package/browser-pool.js.map +0 -1
  49. package/container-proxy-server.d.ts +0 -12
  50. package/container-proxy-server.d.ts.map +0 -1
  51. package/container-proxy-server.js +0 -43
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/utils.d.ts.map +0 -1
  82. package/utils.js.map +0 -1
@@ -1,14 +1,37 @@
1
1
  import fs from 'node:fs';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { getLocalProxyAddress } from '../proxy-server.js';
6
5
  import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
7
6
  import { PlaywrightController } from './playwright-controller.js';
8
7
  export class PlaywrightPlugin extends BrowserPlugin {
9
- _browserVersion;
10
- _containerProxyServer;
8
+ #browserVersion;
9
+ /**
10
+ * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
11
+ * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
12
+ *
13
+ * @internal
14
+ */
15
+ useRemoteConnection(connection, parameters = {}) {
16
+ super.useRemoteConnection(connection, parameters);
17
+ if (!this.useIncognitoPages) {
18
+ this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
19
+ 'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
20
+ }
21
+ this.useIncognitoPages = true;
22
+ }
11
23
  async _launch(launchContext) {
24
+ if (this.remoteConnection) {
25
+ return this.connectToRemoteBrowser(launchContext, async (url) => {
26
+ const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
27
+ if (this.remoteConnectionParameters?.protocol === 'playwright') {
28
+ this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
29
+ return this.library.connect(url, connectOptions);
30
+ }
31
+ this.log.info('Connecting to remote browser via connectOverCDP.');
32
+ return this.library.connectOverCDP(url, connectOptions);
33
+ });
34
+ }
12
35
  const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
13
36
  let browser;
14
37
  // Required for the `proxy` context option to work.
@@ -20,7 +43,9 @@ export class PlaywrightPlugin extends BrowserPlugin {
20
43
  if (this.library.name() === 'webkit') {
21
44
  launchOptions.args = launchOptions.args?.filter((arg) => arg !== '--no-sandbox');
22
45
  }
23
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
46
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
47
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
48
+ });
24
49
  if (anonymizedProxyUrl) {
25
50
  launchOptions.proxy = {
26
51
  server: anonymizedProxyUrl,
@@ -30,7 +55,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
30
55
  try {
31
56
  if (useIncognitoPages) {
32
57
  browser = await this.library.launch(launchOptions).catch((error) => {
33
- return this._throwOnFailedLaunch(launchContext, error);
58
+ return this.throwOnFailedLaunch(launchContext, error);
34
59
  });
35
60
  if (anonymizedProxyUrl) {
36
61
  browser.on('disconnected', async () => {
@@ -42,7 +67,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
42
67
  const browserContext = await this.library
43
68
  .launchPersistentContext(userDataDir, launchOptions)
44
69
  .catch((error) => {
45
- return this._throwOnFailedLaunch(launchContext, error);
70
+ return this.throwOnFailedLaunch(launchContext, error);
46
71
  });
47
72
  browserContext.once('close', () => {
48
73
  if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -57,18 +82,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
57
82
  await close();
58
83
  });
59
84
  }
60
- if (!this._browserVersion) {
85
+ if (!this.#browserVersion) {
61
86
  // Launches unused browser just to get the browser version.
62
87
  const inactiveBrowser = await this.library.launch(launchOptions);
63
- this._browserVersion = inactiveBrowser.version();
88
+ this.#browserVersion = inactiveBrowser.version();
64
89
  inactiveBrowser.close().catch((error) => {
65
- log.exception(error, 'Failed to close browser.');
90
+ this.log.exception(error, 'Failed to close browser.');
66
91
  });
67
92
  }
68
- browser = new PlaywrightBrowserWithPersistentContext({
93
+ const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
69
94
  browserContext,
70
- version: this._browserVersion,
95
+ version: this.#browserVersion,
71
96
  });
97
+ persistentBrowser.setBrowserType(this.library);
98
+ browser = persistentBrowser;
72
99
  }
73
100
  }
74
101
  catch (error) {
@@ -77,13 +104,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
77
104
  }
78
105
  return browser;
79
106
  }
80
- _throwOnFailedLaunch(launchContext, cause) {
81
- this._throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
107
+ throwOnFailedLaunch(launchContext, cause) {
108
+ this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
82
109
  }
83
110
  createController() {
84
111
  return new PlaywrightController(this);
85
112
  }
86
- async _addProxyToLaunchOptions(launchContext) {
113
+ async addProxyToLaunchOptions(launchContext) {
87
114
  launchContext.launchOptions ??= {};
88
115
  const { launchOptions, proxyUrl } = launchContext;
89
116
  if (proxyUrl) {
@@ -95,9 +122,8 @@ export class PlaywrightPlugin extends BrowserPlugin {
95
122
  };
96
123
  }
97
124
  }
98
- _isChromiumBasedBrowser() {
125
+ isChromiumBasedBrowser() {
99
126
  const name = this.library.name();
100
127
  return name === 'chromium';
101
128
  }
102
129
  }
103
- //# sourceMappingURL=playwright-plugin.js.map
package/proxy-server.d.ts CHANGED
@@ -1,2 +1 @@
1
1
  export declare function getLocalProxyAddress(): Promise<string>;
2
- //# sourceMappingURL=proxy-server.d.ts.map
package/proxy-server.js CHANGED
@@ -10,4 +10,3 @@ export async function getLocalProxyAddress() {
10
10
  await listenPromise;
11
11
  return `http://127.0.0.1:${server.port}`;
12
12
  }
13
- //# sourceMappingURL=proxy-server.js.map
@@ -9,6 +9,7 @@ export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOp
9
9
  proxyPassword?: string;
10
10
  }
11
11
  export declare class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
12
+ /** @internal */
12
13
  normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
13
14
  protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
14
15
  protected _close(): Promise<void>;
@@ -16,4 +17,3 @@ export declare class PuppeteerController extends BrowserController<typeof Puppet
16
17
  protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
17
18
  protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
18
19
  }
19
- //# sourceMappingURL=puppeteer-controller.d.ts.map
@@ -1,9 +1,9 @@
1
1
  import { tryCancel } from '@apify/timeout';
2
2
  import { BrowserController } from '../abstract-classes/browser-controller.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
5
  export class PuppeteerController extends BrowserController {
6
+ /** @internal */
7
7
  normalizeProxyOptions(proxyUrl, pageOptions) {
8
8
  if (!proxyUrl) {
9
9
  return {};
@@ -12,7 +12,7 @@ export class PuppeteerController extends BrowserController {
12
12
  const username = decodeURIComponent(url.username);
13
13
  const password = decodeURIComponent(url.password);
14
14
  return {
15
- proxyServer: url.origin,
15
+ proxyServer: `${url.protocol}//${url.host}`,
16
16
  proxyUsername: username,
17
17
  proxyPassword: password,
18
18
  proxyBypassList: pageOptions?.proxyBypassList,
@@ -25,7 +25,7 @@ export class PuppeteerController extends BrowserController {
25
25
  }
26
26
  let close = async () => { };
27
27
  if (contextOptions.proxyServer) {
28
- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
28
+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
29
29
  if (anonymizedProxyUrl) {
30
30
  contextOptions.proxyServer = anonymizedProxyUrl;
31
31
  delete contextOptions.proxyUsername;
@@ -52,17 +52,18 @@ export class PuppeteerController extends BrowserController {
52
52
  tryCancel();
53
53
  }
54
54
  */
55
- page.once('close', async () => {
56
- this.activePages--;
57
- try {
58
- await context.close();
59
- }
60
- catch (error) {
61
- log.exception(error, 'Failed to close context.');
62
- }
63
- finally {
64
- await close();
65
- }
55
+ this.registerPageTeardown(page, async () => {
56
+ // The proxy server is not chained behind the context close: that can hang on
57
+ // the same target the page hung on, and the proxy runs in this process, so it
58
+ // has to be reclaimed either way.
59
+ const contextClosed = context.close().catch((error) => {
60
+ this.log.exception(error, 'Failed to close context.');
61
+ });
62
+ await close();
63
+ await contextClosed;
64
+ });
65
+ page.once('close', () => {
66
+ this.registerPageClosed(page);
66
67
  });
67
68
  return page;
68
69
  }
@@ -74,7 +75,7 @@ export class PuppeteerController extends BrowserController {
74
75
  const page = await this.browser.newPage();
75
76
  tryCancel();
76
77
  page.once('close', () => {
77
- this.activePages--;
78
+ this.registerPageClosed(page);
78
79
  });
79
80
  return page;
80
81
  }
@@ -84,7 +85,7 @@ export class PuppeteerController extends BrowserController {
84
85
  async _kill() {
85
86
  const browserProcess = this.browser.process();
86
87
  if (!browserProcess) {
87
- log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
88
+ this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
88
89
  return;
89
90
  }
90
91
  const timeout = setTimeout(() => {
@@ -98,14 +99,22 @@ export class PuppeteerController extends BrowserController {
98
99
  clearTimeout(timeout);
99
100
  }
100
101
  catch (error) {
101
- log.debug('Browser was already killed.', { error });
102
+ this.log.debug('Browser was already killed.', { error });
102
103
  }
103
104
  }
104
105
  async _getCookies(page) {
105
- return page.cookies();
106
+ const cookies = await page.browserContext().cookies();
107
+ // Puppeteer 25+ can report `sameSite: 'Default'`, which means the attribute is unspecified.
108
+ return cookies.map(({ sameSite, ...rest }) => ({
109
+ ...rest,
110
+ sameSite: sameSite === 'Default' ? undefined : sameSite,
111
+ }));
106
112
  }
107
113
  async _setCookies(page, cookies) {
108
- return page.setCookie(...cookies);
114
+ // BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill
115
+ // the page's current URL for us. Replicate that so callers who pass neither don't get rejected.
116
+ const pageUrl = page.url();
117
+ const normalized = cookies.map((cookie) => cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl });
118
+ return page.browserContext().setCookie(...normalized);
109
119
  }
110
120
  }
111
- //# sourceMappingURL=puppeteer-controller.js.map
@@ -4,12 +4,18 @@ import type Puppeteer from 'puppeteer';
4
4
  import type * as PuppeteerTypes from 'puppeteer';
5
5
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
6
6
  import type { LaunchContext } from '../launch-context.js';
7
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
7
8
  import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
8
9
  import { PuppeteerController } from './puppeteer-controller.js';
9
10
  export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
11
+ /**
12
+ * Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
13
+ *
14
+ * @internal
15
+ */
16
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
10
17
  protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
11
18
  createController(): PuppeteerController;
12
- protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
13
- protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
19
+ protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
20
+ protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
14
21
  }
15
- //# sourceMappingURL=puppeteer-plugin.d.ts.map
@@ -1,11 +1,22 @@
1
1
  import { readFile } from 'node:fs/promises';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { noop } from '../utils.js';
6
5
  import { PuppeteerController } from './puppeteer-controller.js';
7
6
  const PROXY_SERVER_ARG = '--proxy-server=';
8
7
  export class PuppeteerPlugin extends BrowserPlugin {
8
+ /**
9
+ * Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
10
+ *
11
+ * @internal
12
+ */
13
+ useRemoteConnection(connection, parameters = {}) {
14
+ super.useRemoteConnection(connection, parameters);
15
+ if (!this.useIncognitoPages) {
16
+ this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
17
+ 'browser instance (useIncognitoPages defaults to false).');
18
+ }
19
+ }
9
20
  async _launch(launchContext) {
10
21
  let oldPuppeteerVersion = false;
11
22
  try {
@@ -17,58 +28,80 @@ export class PuppeteerPlugin extends BrowserPlugin {
17
28
  catch {
18
29
  // ignore
19
30
  }
20
- const { launchOptions, userDataDir, useIncognitoPages, proxyUrl } = launchContext;
21
- launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
22
- if (launchOptions.headless === false) {
23
- if (Array.isArray(launchOptions.args)) {
24
- launchOptions.args.push('--disable-site-isolation-trials');
25
- }
26
- else {
27
- launchOptions.args = ['--disable-site-isolation-trials'];
28
- }
29
- }
30
- if (launchOptions.headless === true && oldPuppeteerVersion) {
31
- launchOptions.headless = 'new';
32
- }
31
+ const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
33
32
  let browser;
34
- {
35
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
36
- if (proxyUrl) {
37
- const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
33
+ if (this.remoteConnection) {
34
+ browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
35
+ const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
36
+ this.log.info('Connecting to remote browser via connect (CDP).');
37
+ return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
38
+ });
39
+ }
40
+ else {
41
+ const { launchOptions, userDataDir, experimentalContainers } = launchContext;
42
+ if (experimentalContainers) {
43
+ throw new Error('Experimental containers are only available with Playwright');
44
+ }
45
+ launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
46
+ if (launchOptions.headless === false) {
38
47
  if (Array.isArray(launchOptions.args)) {
39
- launchOptions.args.push(proxyArg);
48
+ launchOptions.args.push('--disable-site-isolation-trials');
40
49
  }
41
50
  else {
42
- launchOptions.args = [proxyArg];
51
+ launchOptions.args = ['--disable-site-isolation-trials'];
43
52
  }
44
53
  }
45
- try {
46
- browser = await this.library.launch(launchOptions);
47
- if (anonymizedProxyUrl) {
48
- browser.on('disconnected', async () => {
49
- await close();
50
- });
51
- }
54
+ if (launchOptions.headless === true && oldPuppeteerVersion) {
55
+ launchOptions.headless = 'new';
52
56
  }
53
- catch (error) {
54
- await close();
55
- this._throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
57
+ {
58
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
59
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
60
+ });
61
+ if (proxyUrl) {
62
+ const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
63
+ if (Array.isArray(launchOptions.args)) {
64
+ launchOptions.args.push(proxyArg);
65
+ }
66
+ else {
67
+ launchOptions.args = [proxyArg];
68
+ }
69
+ }
70
+ try {
71
+ browser = await this.library.launch(launchOptions);
72
+ if (anonymizedProxyUrl) {
73
+ browser.on('disconnected', async () => {
74
+ await close();
75
+ });
76
+ }
77
+ }
78
+ catch (error) {
79
+ await close();
80
+ this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
81
+ }
56
82
  }
57
83
  }
58
- browser.on('targetcreated', async (target) => {
84
+ const targetCreatedHandler = async (target) => {
59
85
  try {
60
86
  const page = await target.page();
61
87
  if (page) {
62
88
  page.on('error', (error) => {
63
- log.exception(error, 'Page crashed.');
89
+ this.log.exception(error, 'Page crashed.');
64
90
  page.close().catch(noop);
65
91
  });
66
92
  }
67
93
  }
68
94
  catch (error) {
69
- log.exception(error, 'Failed to retrieve page from target.');
95
+ this.log.exception(error, 'Failed to retrieve page from target.');
70
96
  }
71
- });
97
+ };
98
+ browser.on('targetcreated', targetCreatedHandler);
99
+ // Clean up the listener when a remote browser disconnects to prevent leaks
100
+ if (this.remoteConnection) {
101
+ browser.once('disconnected', () => {
102
+ browser.off('targetcreated', targetCreatedHandler);
103
+ });
104
+ }
72
105
  const boundMethods = [
73
106
  'newPage',
74
107
  'close',
@@ -78,6 +111,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
78
111
  'version',
79
112
  'on',
80
113
  'process',
114
+ 'pages',
81
115
  ].reduce((map, method) => {
82
116
  map[method] = browser[method]?.bind(browser);
83
117
  return map;
@@ -89,22 +123,30 @@ export class PuppeteerPlugin extends BrowserPlugin {
89
123
  return async (...args) => {
90
124
  let page;
91
125
  if (useIncognitoPages) {
92
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
126
+ // Skip proxy setup for remote connections — proxy is managed by the remote service.
127
+ const effectiveProxyUrl = this.remoteConnection ? undefined : proxyUrl;
128
+ const [anonymizedProxyUrl, close] = effectiveProxyUrl
129
+ ? await anonymizeProxySugar(effectiveProxyUrl, undefined, undefined, {
130
+ ignoreProxyCertificate,
131
+ })
132
+ : [undefined, noop];
133
+ const proxyServer = anonymizedProxyUrl ?? effectiveProxyUrl;
134
+ const contextOptions = proxyServer ? { proxyServer } : {};
135
+ const context = (await browser[method](contextOptions));
93
136
  try {
94
- const context = (await browser[method]({
95
- proxyServer: anonymizedProxyUrl ?? proxyUrl,
96
- }));
97
137
  page = await context.newPage(...args);
98
- if (anonymizedProxyUrl) {
99
- page.on('close', async () => {
100
- await close();
101
- });
102
- }
103
138
  }
104
139
  catch (error) {
140
+ await context.close().catch(noop);
105
141
  await close();
106
142
  throw error;
107
143
  }
144
+ page.once('close', async () => {
145
+ if (anonymizedProxyUrl) {
146
+ await close();
147
+ }
148
+ await context.close().catch(noop);
149
+ });
108
150
  }
109
151
  else {
110
152
  page = await boundMethods.newPage(...args);
@@ -139,7 +181,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
139
181
  createController() {
140
182
  return new PuppeteerController(this);
141
183
  }
142
- async _addProxyToLaunchOptions(_launchContext) {
184
+ async addProxyToLaunchOptions(_launchContext) {
143
185
  /*
144
186
  // DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
145
187
  launchContext.launchOptions ??= {};
@@ -166,8 +208,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
166
208
  }
167
209
  */
168
210
  }
169
- _isChromiumBasedBrowser(_launchContext) {
211
+ isChromiumBasedBrowser(_launchContext) {
170
212
  return true;
171
213
  }
172
214
  }
173
- //# sourceMappingURL=puppeteer-plugin.js.map
@@ -0,0 +1,178 @@
1
+ import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
2
+ import type { BrowserPlugin } from './abstract-classes/browser-plugin.js';
3
+ import { BrowserPool } from './browser-pool.js';
4
+ import type { BrowserPoolHooks, BrowserPoolOptions } from './browser-pool.js';
5
+ import { RemoteBrowserProvider } from './remote-browser-provider.js';
6
+ /**
7
+ * The result of resolving a remote browser endpoint: the URL to connect to plus an optional opaque
8
+ * `context` object that is handed back to `release`.
9
+ */
10
+ export interface ResolvedRemoteEndpoint {
11
+ /** The browser endpoint URL to connect to. */
12
+ url: string;
13
+ /** Opaque metadata passed back to `release()` — e.g. session IDs, API tokens. */
14
+ context?: Record<string, unknown>;
15
+ }
16
+ /**
17
+ * A remote browser endpoint: either a static URL string, or a function called once per browser launch
18
+ * that returns a URL (optionally with a `context` for `release`).
19
+ *
20
+ * The function receives the `proxyUrl` resolved by Crawlee's proxy configuration for the launch, so it
21
+ * can forward it to the remote service's proxy API.
22
+ */
23
+ export type RemoteBrowserEndpoint = string | ((options?: {
24
+ proxyUrl?: string;
25
+ }) => string | ResolvedRemoteEndpoint | Promise<string | ResolvedRemoteEndpoint>);
26
+ /**
27
+ * The bridge a {@link RemoteBrowserPool} injects into a {@link BrowserPlugin} so the plugin can
28
+ * connect to a remote browser without owning any remote-session policy.
29
+ *
30
+ * The plugin only knows how to make the library-specific `connect()` call; everything else — resolving
31
+ * the endpoint, calling the user's `release()`, and guaranteeing release fires at most once — lives in
32
+ * the pool. The plugin calls {@link RemoteConnection.resolve|resolve} before connecting, stores the
33
+ * returned `token` on its launch context, and the controller later calls
34
+ * {@link RemoteConnection.release|release} with that token when the browser closes.
35
+ */
36
+ export interface RemoteConnection {
37
+ /** Resolves the endpoint for a single browser launch. The `token` identifies the session for release. */
38
+ resolve(options?: {
39
+ proxyUrl?: string;
40
+ }): Promise<{
41
+ url: string;
42
+ token: number;
43
+ }>;
44
+ /** Releases the remote session for `token`. Idempotent — safe to call from both `close()` and `kill()`. */
45
+ release(token: number): Promise<void>;
46
+ }
47
+ /**
48
+ * Per-plugin remote connection parameters, passed to {@link BrowserPlugin.useRemoteConnection}.
49
+ * The endpoint is supplied per-launch via {@link RemoteConnection}; these are the static connect()
50
+ * parameters (protocol, headers, timeouts, …).
51
+ */
52
+ export interface RemoteConnectionParameters {
53
+ /**
54
+ * Playwright only: which protocol to connect with. `'cdp'` uses `connectOverCDP()` (the default),
55
+ // @ts-ignore optional peer dependency or compatibility with es2022
56
+ * `'playwright'` uses `connect()` (Playwright's own WebSocket protocol). Ignored by Puppeteer.
57
+ */
58
+ protocol?: 'cdp' | 'playwright';
59
+ /** Extra options forwarded to the library `connect()` / `connectOverCDP()` call (endpoint excluded). */
60
+ connectOptions?: Record<string, unknown>;
61
+ }
62
+ export interface RemoteBrowserPoolOptions {
63
+ /**
64
+ * The browser plugin(s) used to connect to the remote service — e.g. `new PlaywrightPlugin(playwright.chromium)`
65
+ * or `new PuppeteerPlugin(puppeteer)`. The pool configures them for remote connection; do not set a local
66
+ * `launchOptions` on them.
67
+ */
68
+ browserPlugins: BrowserPlugin[];
69
+ /**
70
+ * The remote browser endpoint: a static URL, a function returning one per launch, or a
71
+ * {@link RemoteBrowserProvider} instance encapsulating a session create/release lifecycle.
72
+ */
73
+ endpoint: RemoteBrowserEndpoint | RemoteBrowserProvider<any>;
74
+ /**
75
+ * Cleanup callback invoked when a browser closes, crashes, or the pool is destroyed. Receives the
76
+ * `context` returned by a function endpoint. Errors are caught and logged. Ignored when `endpoint`
77
+ * is a {@link RemoteBrowserProvider} (its own `release()` is used instead).
78
+ */
79
+ release?: (info: {
80
+ endpoint: string;
81
+ context?: Record<string, unknown>;
82
+ }) => unknown;
83
+ /**
84
+ * Maximum number of remote browsers open at once. When reached, {@link RemoteBrowserPool.newPage|newPage}
85
+ * waits for a browser to close before connecting a new one. Set it to your service's concurrent-session limit
86
+ * to avoid `429` errors. Defaults to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or
87
+ * `Infinity`.
88
+ */
89
+ maxOpenBrowsers?: number;
90
+ /** Static connect() parameters (Playwright protocol selection, headers, timeouts, …). */
91
+ connection?: RemoteConnectionParameters;
92
+ /** Extra {@link BrowserPool} options (lifecycle hooks, page limits, fingerprinting, …). */
93
+ browserPoolOptions?: Omit<BrowserPoolOptions, 'browserPlugins'> & BrowserPoolHooks<any, any, any>;
94
+ /**
95
+ * Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it.
96
+ * @default 500
97
+ * @internal
98
+ */
99
+ slotPollIntervalMillis?: number;
100
+ }
101
+ /**
102
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the
103
+ * {@link RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its
104
+ * browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` — tuning the wrapping
105
+ * pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.
106
+ */
107
+ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
108
+ /**
109
+ * An {@link IBrowserPool} implementation for remote browser services.
110
+ *
111
+ * Unlike configuring a remote browser through a crawler's `launchContext`, this pool is the single owner
112
+ * of all remote-session concerns:
113
+ * - **endpoint resolution** — static URL, per-launch function, or {@link RemoteBrowserProvider};
114
+ * - **release lifecycle** — `release()` fires exactly once per session on close/crash/teardown (no leaks,
115
+ * no double-release);
116
+ * - **concurrency** — {@link RemoteBrowserPoolOptions.maxOpenBrowsers|maxOpenBrowsers} is enforced inside
117
+ * {@link RemoteBrowserPool.newPage|newPage}, which waits for a free slot rather than overshooting.
118
+ *
119
+ * The wrapped {@link BrowserPool} and its plugin only perform the library-specific `connect()` call.
120
+ *
121
+ * Pass an instance as the crawler's `browserPool` option:
122
+ *
123
+ * ```typescript
124
+ * import { PlaywrightPlugin, RemoteBrowserPool } from '@crawlee/browser-pool';
125
+ * import { PlaywrightCrawler } from 'crawlee';
126
+ // @ts-ignore optional peer dependency or compatibility with es2022
127
+ * import playwright from 'playwright';
128
+ *
129
+ * const browserPool = new RemoteBrowserPool({
130
+ * browserPlugins: [new PlaywrightPlugin(playwright.chromium)],
131
+ * endpoint: 'wss://production-sfo.browserless.io?token=xxx',
132
+ * maxOpenBrowsers: 2,
133
+ * });
134
+ *
135
+ * const crawler = new PlaywrightCrawler({ browserPool });
136
+ * ```
137
+ *
138
+ * @category Browser management
139
+ */
140
+ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
141
+ #private;
142
+ /**
143
+ * The wrapped pool that performs the remote connections and serves pages.
144
+ *
145
+ * @internal
146
+ */
147
+ readonly browserPool: BrowserPool;
148
+ constructor(options: RemoteBrowserPoolOptions);
149
+ /** Maximum number of remote browsers that may be open at the same time. */
150
+ get maxOpenBrowsers(): number;
151
+ set maxOpenBrowsers(value: number);
152
+ /**
153
+ * Opens a new page, waiting first until {@link RemoteBrowserPoolOptions.maxOpenBrowsers|maxOpenBrowsers}
154
+ * allows it (either a new browser slot is free, or an active browser still has page capacity).
155
+ */
156
+ newPage(options?: NewPageOptions): Promise<Page>;
157
+ closePage(page: Page, options?: {
158
+ error?: Error;
159
+ }): Promise<void>;
160
+ extractPageState(page: Page): Promise<PageState>;
161
+ injectPageState(page: Page, state: PageState): Promise<void>;
162
+ /**
163
+ * Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
164
+ * releases its session (see the constructor), so this needs no separate release pass.
165
+ */
166
+ releaseAllBrowsers(): Promise<void>;
167
+ [Symbol.asyncDispose](): Promise<void>;
168
+ /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
169
+ destroy(): Promise<void>;
170
+ /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
171
+ private waitForFreeSlot;
172
+ /**
173
+ * Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
174
+ * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
175
+ * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
176
+ */
177
+ private nextCapacityChange;
178
+ }