@crawlee/browser-pool 4.0.0-beta.8 → 4.0.0-beta.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +17 -13
  2. package/abstract-classes/browser-controller.d.ts +69 -10
  3. package/abstract-classes/browser-controller.js +8 -11
  4. package/abstract-classes/browser-plugin.d.ts +44 -9
  5. package/abstract-classes/browser-plugin.js +65 -7
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +88 -14
  9. package/browser-pool.js +151 -28
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +0 -1
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +24 -9
  16. package/fingerprinting/types.d.ts +0 -1
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +0 -1
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +19 -3
  23. package/launch-context.js +12 -4
  24. package/package.json +7 -8
  25. package/playwright/playwright-browser.d.ts +0 -1
  26. package/playwright/playwright-browser.js +0 -1
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +20 -10
  29. package/playwright/playwright-plugin.d.ts +6 -3
  30. package/playwright/playwright-plugin.js +27 -5
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +3 -1
  36. package/puppeteer/puppeteer-plugin.js +81 -44
  37. package/remote-browser-pool.d.ts +172 -0
  38. package/remote-browser-pool.js +191 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/tsconfig.build.tsbuildinfo +0 -1
  82. package/utils.d.ts.map +0 -1
  83. package/utils.js.map +0 -1
package/README.md CHANGED
@@ -1,19 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
14
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
15
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
16
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
+ </p>
15
+
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
17
21
  </p>
18
22
 
19
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
24
28
 
25
29
  > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
26
30
 
27
- > Crawlee for Python is open for early adopters. 🐍 [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
31
+ > Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
28
32
 
29
33
  ## Installation
30
34
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
85
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
86
90
 
87
91
  ```bash
88
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
89
93
  ```
90
94
 
91
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
94
98
  {
95
99
  "overrides": {
96
100
  "apify": {
97
- "@crawlee/core": "3.12.3-beta.13",
98
- "@crawlee/types": "3.12.3-beta.13",
99
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
100
104
  }
101
105
  }
102
106
  }
@@ -1,9 +1,73 @@
1
+ import { type CrawleeLogger } from '@crawlee/core';
1
2
  import type { Cookie, Dictionary } from '@crawlee/types';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
4
5
  import type { LaunchContext } from '../launch-context.js';
5
6
  import type { UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
8
+ /**
9
+ * The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
10
+ * Other fields are only available on the concrete `LaunchContext` class.
11
+ */
12
+ export interface IBrowserLaunchContext {
13
+ /**
14
+ * The proxy URL the browser was launched with, if any.
15
+ */
16
+ proxyUrl?: string;
17
+ /**
18
+ * The fingerprint applied to the browser, if fingerprinting is enabled.
19
+ * Typed as `unknown` here; cast to the concrete `LaunchContext` if you
20
+ * need the structured shape.
21
+ */
22
+ fingerprint?: unknown;
23
+ /**
24
+ * `true` if each page in this browser uses its own context.
25
+ */
26
+ useIncognitoPages?: boolean;
27
+ /**
28
+ * The actual options the browser was launched with, after pre-launch hooks.
29
+ */
30
+ launchOptions?: Dictionary | undefined;
31
+ }
32
+ /**
33
+ * The minimal public contract of a browser controller.
34
+ *
35
+ * Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
36
+ * promises, …) is intentionally **not** part of this contract.
37
+ *
38
+ * @category Browser management
39
+ */
40
+ export interface IBrowserController<Page = unknown> {
41
+ /**
42
+ * A stable identifier for this controller instance. Useful for tracking
43
+ * which browser served which request.
44
+ */
45
+ readonly id: string;
46
+ /**
47
+ * The configuration the underlying browser was launched with — proxy URL,
48
+ * fingerprint, session, launcher-specific options, etc.
49
+ */
50
+ readonly launchContext: IBrowserLaunchContext;
51
+ /**
52
+ * The raw browser handle from the underlying automation library
53
+ * (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
54
+ * Escape hatch for things the controller does not expose directly.
55
+ */
56
+ readonly browser: unknown;
57
+ /**
58
+ * Reads cookies for the given page.
59
+ */
60
+ getCookies(page: Page): Promise<Cookie[]>;
61
+ /**
62
+ * Writes cookies for the given page.
63
+ */
64
+ setCookies(page: Page, cookies: Cookie[]): Promise<void>;
65
+ /**
66
+ * Gracefully closes the browser this controller owns. After this resolves,
67
+ * the controller is no longer usable.
68
+ */
69
+ close(): Promise<void>;
70
+ }
7
71
  export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
8
72
  [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
9
73
  }
@@ -15,12 +79,13 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
15
79
  * for the specialized classes, because it's the same for all of them.
16
80
  * @hideconstructor
17
81
  */
18
- export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
19
- id: string;
82
+ export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
83
+ readonly id: string;
84
+ protected readonly log: CrawleeLogger;
20
85
  /**
21
86
  * The `BrowserPlugin` instance used to launch the browser.
22
87
  */
23
- browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
88
+ readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
24
89
  /**
25
90
  * Browser representation of the underlying automation library.
26
91
  */
@@ -30,12 +95,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
30
95
  */
31
96
  launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
32
97
  /**
33
- * The proxy tier tied to this browser controller.
34
- * `undefined` if no tiered proxy is used.
35
- */
36
- proxyTier?: number;
37
- /**
38
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
98
+ * The proxy URL used by the browser controller.
39
99
  * `undefined` if no proxy is used
40
100
  */
41
101
  proxyUrl?: string;
@@ -104,4 +164,3 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
104
164
  */
105
165
  abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
106
166
  }
107
- //# sourceMappingURL=browser-controller.d.ts.map
@@ -1,7 +1,7 @@
1
+ import { serviceLocator } from '@crawlee/core';
1
2
  import { nanoid } from 'nanoid';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { tryCancel } from '@apify/timeout';
4
- import { log } from '../logger.js';
5
5
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
6
  /**
7
7
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
13
13
  */
14
14
  export class BrowserController extends TypedEmitter {
15
15
  id = nanoid();
16
+ log;
16
17
  /**
17
18
  * The `BrowserPlugin` instance used to launch the browser.
18
19
  */
@@ -26,12 +27,7 @@ export class BrowserController extends TypedEmitter {
26
27
  */
27
28
  launchContext = undefined;
28
29
  /**
29
- * The proxy tier tied to this browser controller.
30
- * `undefined` if no tiered proxy is used.
31
- */
32
- proxyTier;
33
- /**
34
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
30
+ * The proxy URL used by the browser controller.
35
31
  * `undefined` if no proxy is used
36
32
  */
37
33
  proxyUrl;
@@ -49,6 +45,7 @@ export class BrowserController extends TypedEmitter {
49
45
  });
50
46
  constructor(browserPlugin) {
51
47
  super();
48
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
52
49
  this.browserPlugin = browserPlugin;
53
50
  }
54
51
  /**
@@ -89,14 +86,15 @@ export class BrowserController extends TypedEmitter {
89
86
  this.isActive = false;
90
87
  }
91
88
  catch (error) {
92
- log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
89
+ this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
93
90
  }
94
91
  this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
95
- setTimeout(() => {
92
+ const killTimer = setTimeout(() => {
96
93
  this._kill().catch((err) => {
97
- log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
94
+ this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
98
95
  });
99
96
  }, PROCESS_KILL_TIMEOUT_MILLIS);
97
+ killTimer.unref();
100
98
  }
101
99
  /**
102
100
  * Immediately kills the browser process.
@@ -128,4 +126,3 @@ export class BrowserController extends TypedEmitter {
128
126
  return this._getCookies(page);
129
127
  }
130
128
  }
131
- //# sourceMappingURL=browser-controller.js.map
@@ -1,8 +1,9 @@
1
- import { CriticalError } from '@crawlee/core';
1
+ import { type CrawleeLogger, CriticalError } from '@crawlee/core';
2
2
  import type { Dictionary } from '@crawlee/types';
3
3
  import type { LaunchContextOptions } from '../launch-context.js';
4
4
  import { LaunchContext } from '../launch-context.js';
5
- import type { UnwrapPromise } from '../utils.js';
5
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
6
+ import { type UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserController } from './browser-controller.js';
7
8
  /**
8
9
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
@@ -35,6 +36,7 @@ export interface CommonBrowser {
35
36
  export interface CommonPage {
36
37
  close(...args: unknown[]): Promise<unknown>;
37
38
  url(): string | Promise<string>;
39
+ evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
38
40
  }
39
41
  export interface BrowserPluginOptions<LibraryOptions> {
40
42
  /**
@@ -68,6 +70,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
68
70
  * Might cause performance issues, as Crawlee might launch too many browser instances.
69
71
  */
70
72
  browserPerProxy?: boolean;
73
+ /**
74
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
75
+ * This is useful when using HTTPS proxies with self-signed certificates.
76
+ */
77
+ ignoreProxyCertificate?: boolean;
71
78
  }
72
79
  export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
73
80
  }
@@ -78,14 +85,43 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
78
85
  * feed them to {@link BrowserPool} for use.
79
86
  */
80
87
  export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
81
- name: string;
82
- library: Library;
83
- launchOptions: LibraryOptions;
84
- proxyUrl?: string;
85
- userDataDir?: string;
88
+ readonly name: string;
89
+ protected readonly log: CrawleeLogger;
90
+ readonly library: Library;
91
+ readonly launchOptions: LibraryOptions;
92
+ readonly proxyUrl?: string;
93
+ readonly userDataDir?: string;
86
94
  useIncognitoPages: boolean;
87
- browserPerProxy?: boolean;
95
+ readonly browserPerProxy?: boolean;
96
+ readonly ignoreProxyCertificate?: boolean;
97
+ /**
98
+ * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
99
+ * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
100
+ * remote-session policy lives in the pool, not here.
101
+ *
102
+ * @internal
103
+ */
104
+ remoteConnection?: RemoteConnection;
105
+ /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
106
+ remoteConnectionParameters?: RemoteConnectionParameters;
88
107
  constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
108
+ /**
109
+ * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
110
+ * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
111
+ * (e.g. forcing incognito pages).
112
+ *
113
+ * @internal
114
+ */
115
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
116
+ /**
117
+ * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
118
+ * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
119
+ * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
120
+ *
121
+ * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
122
+ * lives here so it stays identical across plugins.
123
+ */
124
+ protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
89
125
  /**
90
126
  * Creates a `LaunchContext` with all the information needed
91
127
  * to launch a browser. Aside from library specific launch options,
@@ -113,4 +149,3 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
113
149
  export declare class BrowserLaunchError extends CriticalError {
114
150
  constructor(...args: ConstructorParameters<typeof CriticalError>);
115
151
  }
116
- //# sourceMappingURL=browser-plugin.d.ts.map
@@ -1,6 +1,7 @@
1
- import { CriticalError } from '@crawlee/core';
1
+ import { CriticalError, serviceLocator } from '@crawlee/core';
2
2
  import merge from 'lodash.merge';
3
3
  import { LaunchContext } from '../launch-context.js';
4
+ import { sanitizeEndpointForLog } from '../utils.js';
4
5
  /**
5
6
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
6
7
  * when Chromium/Chrome browser is launched:
@@ -20,20 +21,73 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
20
21
  */
21
22
  export class BrowserPlugin {
22
23
  name = this.constructor.name;
24
+ log;
23
25
  library;
24
26
  launchOptions;
25
27
  proxyUrl;
26
28
  userDataDir;
27
29
  useIncognitoPages;
28
30
  browserPerProxy;
31
+ ignoreProxyCertificate;
32
+ /**
33
+ * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
34
+ * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
35
+ * remote-session policy lives in the pool, not here.
36
+ *
37
+ * @internal
38
+ */
39
+ remoteConnection;
40
+ /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
41
+ remoteConnectionParameters;
29
42
  constructor(library, options = {}) {
30
- const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
43
+ const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
44
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
31
45
  this.library = library;
32
46
  this.launchOptions = launchOptions;
33
47
  this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
34
48
  this.userDataDir = userDataDir;
35
49
  this.useIncognitoPages = useIncognitoPages;
36
50
  this.browserPerProxy = browserPerProxy;
51
+ this.ignoreProxyCertificate = ignoreProxyCertificate;
52
+ }
53
+ /**
54
+ * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
55
+ * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
56
+ * (e.g. forcing incognito pages).
57
+ *
58
+ * @internal
59
+ */
60
+ useRemoteConnection(connection, parameters = {}) {
61
+ this.remoteConnection = connection;
62
+ this.remoteConnectionParameters = parameters;
63
+ }
64
+ /**
65
+ * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
66
+ * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
67
+ * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
68
+ *
69
+ * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
70
+ * lives here so it stays identical across plugins.
71
+ */
72
+ async _connectToRemoteBrowser(launchContext, connect) {
73
+ const connection = this.remoteConnection;
74
+ let url;
75
+ let token;
76
+ try {
77
+ ({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
78
+ }
79
+ catch (cause) {
80
+ throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
81
+ }
82
+ launchContext._remoteToken = token;
83
+ try {
84
+ return await connect(url);
85
+ }
86
+ catch (cause) {
87
+ await connection.release(token);
88
+ throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
89
+ 'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
90
+ }
37
91
  }
38
92
  /**
39
93
  * Creates a `LaunchContext` with all the information needed
@@ -42,7 +96,7 @@ export class BrowserPlugin {
42
96
  * management of the pool and extra features.
43
97
  */
44
98
  createLaunchContext(options = {}) {
45
- const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
99
+ const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote = !!this.remoteConnection, } = options;
46
100
  return new LaunchContext({
47
101
  id,
48
102
  launchOptions: merge({}, this.launchOptions, launchOptions),
@@ -51,19 +105,21 @@ export class BrowserPlugin {
51
105
  useIncognitoPages,
52
106
  userDataDir,
53
107
  browserPerProxy,
54
- proxyTier,
108
+ ignoreProxyCertificate,
109
+ isRemote,
55
110
  });
56
111
  }
57
112
  /**
58
113
  * Launches the browser using provided launch context.
59
114
  */
60
115
  async launch(launchContext = this.createLaunchContext()) {
116
+ // launchOptions is only used by the local launch path below — remote connections ignore it.
61
117
  launchContext.launchOptions ??= {};
62
118
  const { proxyUrl, launchOptions } = launchContext;
63
- if (proxyUrl) {
119
+ if (proxyUrl && !launchContext.isRemote) {
64
120
  await this._addProxyToLaunchOptions(launchContext);
65
121
  }
66
- if (this._isChromiumBasedBrowser(launchContext)) {
122
+ if (!launchContext.isRemote && this._isChromiumBasedBrowser(launchContext)) {
67
123
  // This will set the args for chromium based browsers to hide the webdriver.
68
124
  launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
69
125
  // When User-Agent is not set, and we're using Chromium in headless mode,
@@ -74,6 +130,9 @@ export class BrowserPlugin {
74
130
  launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
75
131
  }
76
132
  }
133
+ if (launchContext.isRemote) {
134
+ this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
135
+ }
77
136
  return this._launch(launchContext);
78
137
  }
79
138
  _mergeArgsToHideWebdriver(originalArgs) {
@@ -118,4 +177,3 @@ export class BrowserLaunchError extends CriticalError {
118
177
  });
119
178
  }
120
179
  }
121
- //# sourceMappingURL=browser-plugin.js.map
@@ -1,4 +1,7 @@
1
1
  type PromiseVoid = () => Promise<void>;
2
- export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string) => Promise<[string | undefined, PromiseVoid]>;
2
+ export interface AnonymizeProxySugarOptions {
3
+ ignoreProxyCertificate?: boolean;
4
+ }
5
+ /** @internal */
6
+ export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
3
7
  export {};
4
- //# sourceMappingURL=anonymize-proxy.d.ts.map
@@ -1,15 +1,20 @@
1
- import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
2
- export const anonymizeProxySugar = async (proxyUrl, username, password) => {
1
+ /** @internal */
2
+ export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
3
3
  if (proxyUrl) {
4
4
  const url = new URL(proxyUrl);
5
5
  if (username || password) {
6
6
  url.username = username ?? '';
7
7
  url.password = password ?? '';
8
8
  }
9
- if (url.username || url.password) {
9
+ if (url.username || url.password || options?.ignoreProxyCertificate) {
10
10
  // trim off trailing slash if it's present
11
11
  const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
12
- const anonymized = await anonymizeProxy(proxyUrlString);
12
+ const { anonymizeProxy, closeAnonymizedProxy } = await import('proxy-chain');
13
+ const anonymized = await anonymizeProxy({
14
+ url: proxyUrlString,
15
+ port: 0,
16
+ ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
17
+ });
13
18
  return [
14
19
  anonymized,
15
20
  async () => {
@@ -21,4 +26,3 @@ export const anonymizeProxySugar = async (proxyUrl, username, password) => {
21
26
  }
22
27
  return [undefined, async () => { }];
23
28
  };
24
- //# sourceMappingURL=anonymize-proxy.js.map