@crawlee/browser-pool 4.0.0-beta.1 → 4.0.0-beta.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +17 -13
  2. package/abstract-classes/browser-controller.d.ts +69 -10
  3. package/abstract-classes/browser-controller.js +8 -11
  4. package/abstract-classes/browser-plugin.d.ts +44 -11
  5. package/abstract-classes/browser-plugin.js +65 -7
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +88 -14
  9. package/browser-pool.js +151 -28
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +0 -1
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +24 -9
  16. package/fingerprinting/types.d.ts +0 -1
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +0 -1
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +19 -3
  23. package/launch-context.js +12 -4
  24. package/package.json +8 -9
  25. package/playwright/playwright-browser.d.ts +0 -1
  26. package/playwright/playwright-browser.js +0 -1
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +20 -10
  29. package/playwright/playwright-plugin.d.ts +6 -3
  30. package/playwright/playwright-plugin.js +27 -5
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +3 -1
  36. package/puppeteer/puppeteer-plugin.js +81 -44
  37. package/remote-browser-pool.d.ts +170 -0
  38. package/remote-browser-pool.js +191 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/tsconfig.build.tsbuildinfo +0 -1
  82. package/utils.d.ts.map +0 -1
  83. package/utils.js.map +0 -1
package/index.d.ts CHANGED
@@ -28,13 +28,20 @@ export * from './browser-pool.js';
28
28
  export * from './playwright/playwright-plugin.js';
29
29
  export * from './puppeteer/puppeteer-plugin.js';
30
30
  export * from './events.js';
31
- export { BrowserName, DeviceCategory, OperatingSystemsName, } from './fingerprinting/types.js';
32
- export { BrowserController, BrowserControllerEvents } from './abstract-classes/browser-controller.js';
31
+ export type { BrowserSpecification, FingerprintGenerator, FingerprintGeneratorOptions, GetFingerprintReturn, } from './fingerprinting/types.js';
32
+ export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
33
+ export type { BrowserControllerEvents, IBrowserController, IBrowserLaunchContext, } from './abstract-classes/browser-controller.js';
34
+ export { BrowserController } from './abstract-classes/browser-controller.js';
33
35
  export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
34
36
  export { PlaywrightController } from './playwright/playwright-controller.js';
35
37
  export { PlaywrightBrowser } from './playwright/playwright-browser.js';
36
- export { CommonPage, CommonLibrary, BrowserPlugin, BrowserPluginOptions, CreateLaunchContextOptions, BrowserLaunchError, DEFAULT_USER_AGENT, } from './abstract-classes/browser-plugin.js';
37
- export { LaunchContext, LaunchContextOptions } from './launch-context.js';
38
- export { BrowserSpecification, FingerprintGenerator, FingerprintGeneratorOptions, GetFingerprintReturn, } from './fingerprinting/types.js';
39
- export { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
40
- //# sourceMappingURL=index.d.ts.map
38
+ export type { CommonPage, CommonLibrary, BrowserPluginOptions, CreateLaunchContextOptions, } from './abstract-classes/browser-plugin.js';
39
+ export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
40
+ export type { LaunchContextOptions } from './launch-context.js';
41
+ export { LaunchContext } from './launch-context.js';
42
+ export { RemoteBrowserProvider } from './remote-browser-provider.js';
43
+ export { RemoteBrowserPool } from './remote-browser-pool.js';
44
+ export type { RemoteBrowserPoolOptions, CrawlerRemoteBrowserOptions, RemoteBrowserEndpoint, ResolvedRemoteEndpoint, RemoteConnection, RemoteConnectionParameters, } from './remote-browser-pool.js';
45
+ export type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
46
+ export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy.js';
47
+ export type { IBrowserPool, NewPageOptions } from '@crawlee/types';
package/index.js CHANGED
@@ -26,11 +26,13 @@ export * from './browser-pool.js';
26
26
  export * from './playwright/playwright-plugin.js';
27
27
  export * from './puppeteer/puppeteer-plugin.js';
28
28
  export * from './events.js';
29
- export { BrowserName, DeviceCategory, OperatingSystemsName, } from './fingerprinting/types.js';
29
+ export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
30
30
  export { BrowserController } from './abstract-classes/browser-controller.js';
31
31
  export { PuppeteerController } from './puppeteer/puppeteer-controller.js';
32
32
  export { PlaywrightController } from './playwright/playwright-controller.js';
33
33
  export { PlaywrightBrowser } from './playwright/playwright-browser.js';
34
- export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT, } from './abstract-classes/browser-plugin.js';
34
+ export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
35
35
  export { LaunchContext } from './launch-context.js';
36
- //# sourceMappingURL=index.js.map
36
+ export { RemoteBrowserProvider } from './remote-browser-provider.js';
37
+ export { RemoteBrowserPool } from './remote-browser-pool.js';
38
+ export { anonymizeProxySugar } from './anonymize-proxy.js';
@@ -43,7 +43,17 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
43
43
  */
44
44
  userDataDir?: string;
45
45
  proxyUrl?: string;
46
- proxyTier?: number;
46
+ /**
47
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
48
+ * This is useful when using HTTPS proxies with self-signed certificates.
49
+ */
50
+ ignoreProxyCertificate?: boolean;
51
+ /**
52
+ * Whether this launch context represents a connection to a remote browser
53
+ * rather than a locally launched one.
54
+ * @default false
55
+ */
56
+ isRemote?: boolean;
47
57
  }
48
58
  export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
49
59
  id?: string;
@@ -52,10 +62,17 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
52
62
  useIncognitoPages: boolean;
53
63
  browserPerProxy?: boolean;
54
64
  userDataDir: string;
55
- proxyTier?: number;
65
+ readonly isRemote: boolean;
66
+ ignoreProxyCertificate?: boolean;
56
67
  private _proxyUrl?;
57
68
  private readonly _reservedFieldNames;
58
69
  fingerprint?: BrowserFingerprintWithHeaders;
70
+ /**
71
+ * Token identifying the remote browser session this context connected to, set by the plugin and read by
72
+ * the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
73
+ * @internal
74
+ */
75
+ _remoteToken?: number;
59
76
  [K: PropertyKey]: unknown;
60
77
  constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
61
78
  /**
@@ -76,4 +93,3 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
76
93
  */
77
94
  get proxyUrl(): string | undefined;
78
95
  }
79
- //# sourceMappingURL=launch-context.d.ts.map
package/launch-context.js CHANGED
@@ -5,19 +5,27 @@ export class LaunchContext {
5
5
  useIncognitoPages;
6
6
  browserPerProxy;
7
7
  userDataDir;
8
- proxyTier;
8
+ isRemote;
9
+ ignoreProxyCertificate;
9
10
  _proxyUrl;
10
11
  _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];
11
12
  fingerprint;
13
+ /**
14
+ * Token identifying the remote browser session this context connected to, set by the plugin and read by
15
+ * the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
16
+ * @internal
17
+ */
18
+ _remoteToken;
12
19
  constructor(options) {
13
- const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', proxyTier, } = options;
20
+ const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
14
21
  this.id = id;
15
22
  this.browserPlugin = browserPlugin;
16
23
  this.launchOptions = launchOptions;
17
24
  this.browserPerProxy = browserPerProxy ?? false;
18
25
  this.useIncognitoPages = useIncognitoPages ?? false;
19
26
  this.userDataDir = userDataDir;
20
- this.proxyTier = proxyTier;
27
+ this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
28
+ this.isRemote = isRemote ?? false;
21
29
  this._proxyUrl = proxyUrl;
22
30
  }
23
31
  /**
@@ -43,6 +51,7 @@ export class LaunchContext {
43
51
  */
44
52
  set proxyUrl(url) {
45
53
  if (!url) {
54
+ this._proxyUrl = undefined;
46
55
  return;
47
56
  }
48
57
  const urlInstance = new URL(url);
@@ -59,4 +68,3 @@ export class LaunchContext {
59
68
  return this._proxyUrl;
60
69
  }
61
70
  }
62
- //# sourceMappingURL=launch-context.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser-pool",
3
- "version": "4.0.0-beta.1",
3
+ "version": "4.0.0-beta.100",
4
4
  "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -24,18 +24,17 @@
24
24
  "url": "https://github.com/apify/crawlee/issues"
25
25
  },
26
26
  "scripts": {
27
- "build": "yarn clean && yarn compile && yarn copy",
27
+ "build": "pnpm clean && pnpm compile && pnpm copy",
28
28
  "clean": "rimraf ./dist",
29
29
  "compile": "tsc -p tsconfig.build.json",
30
30
  "copy": "tsx ../../scripts/copy.ts"
31
31
  },
32
32
  "dependencies": {
33
- "@apify/log": "^2.5.18",
34
- "@apify/timeout": "^0.3.2",
35
- "@crawlee/core": "4.0.0-beta.1",
36
- "@crawlee/types": "4.0.0-beta.1",
37
- "fingerprint-generator": "^2.1.66",
38
- "fingerprint-injector": "^2.1.66",
33
+ "@apify/timeout": "^0.4.4",
34
+ "@crawlee/core": "4.0.0-beta.100",
35
+ "@crawlee/types": "4.0.0-beta.100",
36
+ "fingerprint-generator": "^2.1.68",
37
+ "fingerprint-injector": "^2.1.68",
39
38
  "lodash.merge": "^4.6.2",
40
39
  "nanoid": "^5.1.5",
41
40
  "ow": "^2.0.0",
@@ -64,5 +63,5 @@
64
63
  }
65
64
  }
66
65
  },
67
- "gitHead": "4375a5af786811feda9dc872bbbb3406b2f13974"
66
+ "gitHead": "109b4c2097f7b9a26bfe761b3c9969dd626f8e38"
68
67
  }
@@ -28,4 +28,3 @@ export declare class PlaywrightBrowser extends EventEmitter {
28
28
  startTracing(): Promise<never>;
29
29
  stopTracing(): Promise<never>;
30
30
  }
31
- //# sourceMappingURL=playwright-browser.d.ts.map
@@ -55,4 +55,3 @@ export class PlaywrightBrowser extends EventEmitter {
55
55
  throw new Error('Function `stopTracing()` is not available in incognito mode');
56
56
  }
57
57
  }
58
- //# sourceMappingURL=playwright-browser.js.map
@@ -11,4 +11,3 @@ export declare class PlaywrightController extends BrowserController<BrowserType,
11
11
  protected _getCookies(page: Page): Promise<Cookie[]>;
12
12
  protected _setCookies(page: Page, cookies: Cookie[]): Promise<void>;
13
13
  }
14
- //# sourceMappingURL=playwright-controller.d.ts.map
@@ -11,7 +11,7 @@ export class PlaywrightController extends BrowserController {
11
11
  const password = decodeURIComponent(url.password);
12
12
  return {
13
13
  proxy: {
14
- server: url.origin,
14
+ server: `${url.protocol}//${url.host}`,
15
15
  username,
16
16
  password,
17
17
  bypass: pageOptions?.proxy?.bypass,
@@ -23,15 +23,26 @@ export class PlaywrightController extends BrowserController {
23
23
  throw new Error('A new page can be created with provided context only when using incognito pages.');
24
24
  }
25
25
  let close = async () => { };
26
- if (this.launchContext.useIncognitoPages && contextOptions?.proxy) {
27
- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password);
28
- if (anonymizedProxyUrl) {
29
- contextOptions.proxy = {
30
- server: anonymizedProxyUrl,
31
- bypass: contextOptions.proxy.bypass,
32
- };
26
+ if (this.launchContext.useIncognitoPages) {
27
+ // Each page requires to have all the context options applied
28
+ contextOptions = {
29
+ ...this.launchContext.launchOptions,
30
+ ...contextOptions,
31
+ };
32
+ // Remote browsers handle their own proxy — don't inject local proxy settings into context
33
+ if (this.launchContext.isRemote) {
34
+ delete contextOptions?.proxy;
35
+ }
36
+ if (contextOptions?.proxy) {
37
+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxy.server, contextOptions.proxy.username, contextOptions.proxy.password, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
38
+ if (anonymizedProxyUrl) {
39
+ contextOptions.proxy = {
40
+ server: anonymizedProxyUrl,
41
+ bypass: contextOptions.proxy.bypass,
42
+ };
43
+ }
44
+ close = closeProxy;
33
45
  }
34
- close = closeProxy;
35
46
  }
36
47
  try {
37
48
  const page = await this.browser.newPage(contextOptions);
@@ -63,4 +74,3 @@ export class PlaywrightController extends BrowserController {
63
74
  return context.addCookies(cookies);
64
75
  }
65
76
  }
66
- //# sourceMappingURL=playwright-controller.js.map
@@ -1,17 +1,20 @@
1
1
  // @ts-ignore optional peer dependency or compatibility with es2022
2
2
  import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
3
3
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
4
- import type { createProxyServerForContainers } from '../container-proxy-server.js';
5
4
  import type { LaunchContext } from '../launch-context.js';
5
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
6
6
  import type { SafeParameters } from '../utils.js';
7
7
  import { PlaywrightController } from './playwright-controller.js';
8
8
  export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
9
9
  private _browserVersion?;
10
- _containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;
10
+ /**
11
+ * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
12
+ * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
13
+ */
14
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
11
15
  protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
12
16
  private _throwOnFailedLaunch;
13
17
  createController(): PlaywrightController;
14
18
  protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
15
19
  protected _isChromiumBasedBrowser(): boolean;
16
20
  }
17
- //# sourceMappingURL=playwright-plugin.d.ts.map
@@ -1,14 +1,35 @@
1
1
  import fs from 'node:fs';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { getLocalProxyAddress } from '../proxy-server.js';
6
5
  import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
7
6
  import { PlaywrightController } from './playwright-controller.js';
8
7
  export class PlaywrightPlugin extends BrowserPlugin {
9
8
  _browserVersion;
10
- _containerProxyServer;
9
+ /**
10
+ * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
11
+ * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
12
+ */
13
+ useRemoteConnection(connection, parameters = {}) {
14
+ super.useRemoteConnection(connection, parameters);
15
+ if (!this.useIncognitoPages) {
16
+ this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
17
+ 'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
18
+ }
19
+ this.useIncognitoPages = true;
20
+ }
11
21
  async _launch(launchContext) {
22
+ if (this.remoteConnection) {
23
+ return this._connectToRemoteBrowser(launchContext, async (url) => {
24
+ const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
25
+ if (this.remoteConnectionParameters?.protocol === 'playwright') {
26
+ this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
27
+ return this.library.connect(url, connectOptions);
28
+ }
29
+ this.log.info('Connecting to remote browser via connectOverCDP.');
30
+ return this.library.connectOverCDP(url, connectOptions);
31
+ });
32
+ }
12
33
  const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
13
34
  let browser;
14
35
  // Required for the `proxy` context option to work.
@@ -20,7 +41,9 @@ export class PlaywrightPlugin extends BrowserPlugin {
20
41
  if (this.library.name() === 'webkit') {
21
42
  launchOptions.args = launchOptions.args?.filter((arg) => arg !== '--no-sandbox');
22
43
  }
23
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
44
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
45
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
46
+ });
24
47
  if (anonymizedProxyUrl) {
25
48
  launchOptions.proxy = {
26
49
  server: anonymizedProxyUrl,
@@ -62,7 +85,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
62
85
  const inactiveBrowser = await this.library.launch(launchOptions);
63
86
  this._browserVersion = inactiveBrowser.version();
64
87
  inactiveBrowser.close().catch((error) => {
65
- log.exception(error, 'Failed to close browser.');
88
+ this.log.exception(error, 'Failed to close browser.');
66
89
  });
67
90
  }
68
91
  browser = new PlaywrightBrowserWithPersistentContext({
@@ -100,4 +123,3 @@ export class PlaywrightPlugin extends BrowserPlugin {
100
123
  return name === 'chromium';
101
124
  }
102
125
  }
103
- //# sourceMappingURL=playwright-plugin.js.map
package/proxy-server.d.ts CHANGED
@@ -1,2 +1 @@
1
1
  export declare function getLocalProxyAddress(): Promise<string>;
2
- //# sourceMappingURL=proxy-server.d.ts.map
package/proxy-server.js CHANGED
@@ -10,4 +10,3 @@ export async function getLocalProxyAddress() {
10
10
  await listenPromise;
11
11
  return `http://127.0.0.1:${server.port}`;
12
12
  }
13
- //# sourceMappingURL=proxy-server.js.map
@@ -16,4 +16,3 @@ export declare class PuppeteerController extends BrowserController<typeof Puppet
16
16
  protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
17
17
  protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
18
18
  }
19
- //# sourceMappingURL=puppeteer-controller.d.ts.map
@@ -1,7 +1,6 @@
1
1
  import { tryCancel } from '@apify/timeout';
2
2
  import { BrowserController } from '../abstract-classes/browser-controller.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
5
  export class PuppeteerController extends BrowserController {
7
6
  normalizeProxyOptions(proxyUrl, pageOptions) {
@@ -12,7 +11,7 @@ export class PuppeteerController extends BrowserController {
12
11
  const username = decodeURIComponent(url.username);
13
12
  const password = decodeURIComponent(url.password);
14
13
  return {
15
- proxyServer: url.origin,
14
+ proxyServer: `${url.protocol}//${url.host}`,
16
15
  proxyUsername: username,
17
16
  proxyPassword: password,
18
17
  proxyBypassList: pageOptions?.proxyBypassList,
@@ -25,7 +24,7 @@ export class PuppeteerController extends BrowserController {
25
24
  }
26
25
  let close = async () => { };
27
26
  if (contextOptions.proxyServer) {
28
- const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword);
27
+ const [anonymizedProxyUrl, closeProxy] = await anonymizeProxySugar(contextOptions.proxyServer, contextOptions.proxyUsername, contextOptions.proxyPassword, { ignoreProxyCertificate: this.launchContext.ignoreProxyCertificate });
29
28
  if (anonymizedProxyUrl) {
30
29
  contextOptions.proxyServer = anonymizedProxyUrl;
31
30
  delete contextOptions.proxyUsername;
@@ -58,7 +57,7 @@ export class PuppeteerController extends BrowserController {
58
57
  await context.close();
59
58
  }
60
59
  catch (error) {
61
- log.exception(error, 'Failed to close context.');
60
+ this.log.exception(error, 'Failed to close context.');
62
61
  }
63
62
  finally {
64
63
  await close();
@@ -84,7 +83,7 @@ export class PuppeteerController extends BrowserController {
84
83
  async _kill() {
85
84
  const browserProcess = this.browser.process();
86
85
  if (!browserProcess) {
87
- log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
86
+ this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
88
87
  return;
89
88
  }
90
89
  const timeout = setTimeout(() => {
@@ -98,14 +97,22 @@ export class PuppeteerController extends BrowserController {
98
97
  clearTimeout(timeout);
99
98
  }
100
99
  catch (error) {
101
- log.debug('Browser was already killed.', { error });
100
+ this.log.debug('Browser was already killed.', { error });
102
101
  }
103
102
  }
104
103
  async _getCookies(page) {
105
- return page.cookies();
104
+ const cookies = await page.browserContext().cookies();
105
+ // Puppeteer 25+ can report `sameSite: 'Default'`, which means the attribute is unspecified.
106
+ return cookies.map(({ sameSite, ...rest }) => ({
107
+ ...rest,
108
+ sameSite: sameSite === 'Default' ? undefined : sameSite,
109
+ }));
106
110
  }
107
111
  async _setCookies(page, cookies) {
108
- return page.setCookie(...cookies);
112
+ // BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill
113
+ // the page's current URL for us. Replicate that so callers who pass neither don't get rejected.
114
+ const pageUrl = page.url();
115
+ const normalized = cookies.map((cookie) => cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl });
116
+ return page.browserContext().setCookie(...normalized);
109
117
  }
110
118
  }
111
- //# sourceMappingURL=puppeteer-controller.js.map
@@ -4,12 +4,14 @@ import type Puppeteer from 'puppeteer';
4
4
  import type * as PuppeteerTypes from 'puppeteer';
5
5
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
6
6
  import type { LaunchContext } from '../launch-context.js';
7
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
7
8
  import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
8
9
  import { PuppeteerController } from './puppeteer-controller.js';
9
10
  export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
11
+ /** Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito). */
12
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
10
13
  protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
11
14
  createController(): PuppeteerController;
12
15
  protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
13
16
  protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
14
17
  }
15
- //# sourceMappingURL=puppeteer-plugin.d.ts.map
@@ -1,11 +1,18 @@
1
1
  import { readFile } from 'node:fs/promises';
2
2
  import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
3
3
  import { anonymizeProxySugar } from '../anonymize-proxy.js';
4
- import { log } from '../logger.js';
5
4
  import { noop } from '../utils.js';
6
5
  import { PuppeteerController } from './puppeteer-controller.js';
7
6
  const PROXY_SERVER_ARG = '--proxy-server=';
8
7
  export class PuppeteerPlugin extends BrowserPlugin {
8
+ /** Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito). */
9
+ useRemoteConnection(connection, parameters = {}) {
10
+ super.useRemoteConnection(connection, parameters);
11
+ if (!this.useIncognitoPages) {
12
+ this.log.info('Remote Puppeteer connection — pages will share cookies and storage on the remote ' +
13
+ 'browser instance (useIncognitoPages defaults to false).');
14
+ }
15
+ }
9
16
  async _launch(launchContext) {
10
17
  let oldPuppeteerVersion = false;
11
18
  try {
@@ -17,58 +24,80 @@ export class PuppeteerPlugin extends BrowserPlugin {
17
24
  catch {
18
25
  // ignore
19
26
  }
20
- const { launchOptions, userDataDir, useIncognitoPages, proxyUrl } = launchContext;
21
- launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
22
- if (launchOptions.headless === false) {
23
- if (Array.isArray(launchOptions.args)) {
24
- launchOptions.args.push('--disable-site-isolation-trials');
25
- }
26
- else {
27
- launchOptions.args = ['--disable-site-isolation-trials'];
28
- }
29
- }
30
- if (launchOptions.headless === true && oldPuppeteerVersion) {
31
- launchOptions.headless = 'new';
32
- }
27
+ const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
33
28
  let browser;
34
- {
35
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
36
- if (proxyUrl) {
37
- const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
29
+ if (this.remoteConnection) {
30
+ browser = await this._connectToRemoteBrowser(launchContext, async (url) => {
31
+ const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
32
+ this.log.info('Connecting to remote browser via connect (CDP).');
33
+ return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
34
+ });
35
+ }
36
+ else {
37
+ const { launchOptions, userDataDir, experimentalContainers } = launchContext;
38
+ if (experimentalContainers) {
39
+ throw new Error('Experimental containers are only available with Playwright');
40
+ }
41
+ launchOptions.userDataDir = launchOptions.userDataDir ?? userDataDir;
42
+ if (launchOptions.headless === false) {
38
43
  if (Array.isArray(launchOptions.args)) {
39
- launchOptions.args.push(proxyArg);
44
+ launchOptions.args.push('--disable-site-isolation-trials');
40
45
  }
41
46
  else {
42
- launchOptions.args = [proxyArg];
47
+ launchOptions.args = ['--disable-site-isolation-trials'];
43
48
  }
44
49
  }
45
- try {
46
- browser = await this.library.launch(launchOptions);
47
- if (anonymizedProxyUrl) {
48
- browser.on('disconnected', async () => {
49
- await close();
50
- });
51
- }
50
+ if (launchOptions.headless === true && oldPuppeteerVersion) {
51
+ launchOptions.headless = 'new';
52
52
  }
53
- catch (error) {
54
- await close();
55
- this._throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
53
+ {
54
+ const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl, undefined, undefined, {
55
+ ignoreProxyCertificate: launchContext.ignoreProxyCertificate,
56
+ });
57
+ if (proxyUrl) {
58
+ const proxyArg = `${PROXY_SERVER_ARG}${anonymizedProxyUrl ?? proxyUrl}`;
59
+ if (Array.isArray(launchOptions.args)) {
60
+ launchOptions.args.push(proxyArg);
61
+ }
62
+ else {
63
+ launchOptions.args = [proxyArg];
64
+ }
65
+ }
66
+ try {
67
+ browser = await this.library.launch(launchOptions);
68
+ if (anonymizedProxyUrl) {
69
+ browser.on('disconnected', async () => {
70
+ await close();
71
+ });
72
+ }
73
+ }
74
+ catch (error) {
75
+ await close();
76
+ this._throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
77
+ }
56
78
  }
57
79
  }
58
- browser.on('targetcreated', async (target) => {
80
+ const targetCreatedHandler = async (target) => {
59
81
  try {
60
82
  const page = await target.page();
61
83
  if (page) {
62
84
  page.on('error', (error) => {
63
- log.exception(error, 'Page crashed.');
85
+ this.log.exception(error, 'Page crashed.');
64
86
  page.close().catch(noop);
65
87
  });
66
88
  }
67
89
  }
68
90
  catch (error) {
69
- log.exception(error, 'Failed to retrieve page from target.');
91
+ this.log.exception(error, 'Failed to retrieve page from target.');
70
92
  }
71
- });
93
+ };
94
+ browser.on('targetcreated', targetCreatedHandler);
95
+ // Clean up the listener when a remote browser disconnects to prevent leaks
96
+ if (this.remoteConnection) {
97
+ browser.once('disconnected', () => {
98
+ browser.off('targetcreated', targetCreatedHandler);
99
+ });
100
+ }
72
101
  const boundMethods = [
73
102
  'newPage',
74
103
  'close',
@@ -78,6 +107,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
78
107
  'version',
79
108
  'on',
80
109
  'process',
110
+ 'pages',
81
111
  ].reduce((map, method) => {
82
112
  map[method] = browser[method]?.bind(browser);
83
113
  return map;
@@ -89,22 +119,30 @@ export class PuppeteerPlugin extends BrowserPlugin {
89
119
  return async (...args) => {
90
120
  let page;
91
121
  if (useIncognitoPages) {
92
- const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
122
+ // Skip proxy setup for remote connections — proxy is managed by the remote service.
123
+ const effectiveProxyUrl = this.remoteConnection ? undefined : proxyUrl;
124
+ const [anonymizedProxyUrl, close] = effectiveProxyUrl
125
+ ? await anonymizeProxySugar(effectiveProxyUrl, undefined, undefined, {
126
+ ignoreProxyCertificate,
127
+ })
128
+ : [undefined, noop];
129
+ const proxyServer = anonymizedProxyUrl ?? effectiveProxyUrl;
130
+ const contextOptions = proxyServer ? { proxyServer } : {};
131
+ const context = (await browser[method](contextOptions));
93
132
  try {
94
- const context = (await browser[method]({
95
- proxyServer: anonymizedProxyUrl ?? proxyUrl,
96
- }));
97
133
  page = await context.newPage(...args);
98
- if (anonymizedProxyUrl) {
99
- page.on('close', async () => {
100
- await close();
101
- });
102
- }
103
134
  }
104
135
  catch (error) {
136
+ await context.close().catch(noop);
105
137
  await close();
106
138
  throw error;
107
139
  }
140
+ page.once('close', async () => {
141
+ if (anonymizedProxyUrl) {
142
+ await close();
143
+ }
144
+ await context.close().catch(noop);
145
+ });
108
146
  }
109
147
  else {
110
148
  page = await boundMethods.newPage(...args);
@@ -170,4 +208,3 @@ export class PuppeteerPlugin extends BrowserPlugin {
170
208
  return true;
171
209
  }
172
210
  }
173
- //# sourceMappingURL=puppeteer-plugin.js.map