@crawlee/browser-pool 4.0.0-beta.15 → 4.0.0-beta.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +14 -14
  2. package/abstract-classes/browser-controller.d.ts +74 -14
  3. package/abstract-classes/browser-controller.js +28 -24
  4. package/abstract-classes/browser-plugin.d.ts +48 -15
  5. package/abstract-classes/browser-plugin.js +69 -11
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +100 -28
  9. package/browser-pool.js +199 -79
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +2 -3
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +26 -10
  16. package/fingerprinting/types.d.ts +3 -4
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +5 -6
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +20 -5
  23. package/launch-context.js +21 -10
  24. package/package.json +9 -9
  25. package/playwright/playwright-browser.d.ts +2 -6
  26. package/playwright/playwright-browser.js +16 -17
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +6 -3
  29. package/playwright/playwright-plugin.d.ts +10 -7
  30. package/playwright/playwright-plugin.js +40 -16
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +5 -3
  36. package/puppeteer/puppeteer-plugin.js +83 -46
  37. package/remote-browser-pool.d.ts +165 -0
  38. package/remote-browser-pool.js +195 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/utils.d.ts.map +0 -1
  82. package/utils.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
@@ -1,9 +1,73 @@
1
+ import { type CrawleeLogger } from '@crawlee/core';
1
2
  import type { Cookie, Dictionary } from '@crawlee/types';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
4
5
  import type { LaunchContext } from '../launch-context.js';
5
6
  import type { UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
8
+ /**
9
+ * The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
10
+ * Other fields are only available on the concrete `LaunchContext` class.
11
+ */
12
+ export interface IBrowserLaunchContext {
13
+ /**
14
+ * The proxy URL the browser was launched with, if any.
15
+ */
16
+ proxyUrl?: string;
17
+ /**
18
+ * The fingerprint applied to the browser, if fingerprinting is enabled.
19
+ * Typed as `unknown` here; cast to the concrete `LaunchContext` if you
20
+ * need the structured shape.
21
+ */
22
+ fingerprint?: unknown;
23
+ /**
24
+ * `true` if each page in this browser uses its own context.
25
+ */
26
+ useIncognitoPages?: boolean;
27
+ /**
28
+ * The actual options the browser was launched with, after pre-launch hooks.
29
+ */
30
+ launchOptions?: Dictionary | undefined;
31
+ }
32
+ /**
33
+ * The minimal public contract of a browser controller.
34
+ *
35
+ * Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
36
+ * promises, …) is intentionally **not** part of this contract.
37
+ *
38
+ * @category Browser management
39
+ */
40
+ export interface IBrowserController<Page = unknown> {
41
+ /**
42
+ * A stable identifier for this controller instance. Useful for tracking
43
+ * which browser served which request.
44
+ */
45
+ readonly id: string;
46
+ /**
47
+ * The configuration the underlying browser was launched with — proxy URL,
48
+ * fingerprint, session, launcher-specific options, etc.
49
+ */
50
+ readonly launchContext: IBrowserLaunchContext;
51
+ /**
52
+ * The raw browser handle from the underlying automation library
53
+ * (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
54
+ * Escape hatch for things the controller does not expose directly.
55
+ */
56
+ readonly browser: unknown;
57
+ /**
58
+ * Reads cookies for the given page.
59
+ */
60
+ getCookies(page: Page): Promise<Cookie[]>;
61
+ /**
62
+ * Writes cookies for the given page.
63
+ */
64
+ setCookies(page: Page, cookies: Cookie[]): Promise<void>;
65
+ /**
66
+ * Gracefully closes the browser this controller owns. After this resolves,
67
+ * the controller is no longer usable.
68
+ */
69
+ close(): Promise<void>;
70
+ }
7
71
  export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
8
72
  [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
9
73
  }
@@ -15,12 +79,14 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
15
79
  * for the specialized classes, because it's the same for all of them.
16
80
  * @hideconstructor
17
81
  */
18
- export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
19
- id: string;
82
+ export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
83
+ #private;
84
+ readonly id: string;
85
+ protected readonly log: CrawleeLogger;
20
86
  /**
21
87
  * The `BrowserPlugin` instance used to launch the browser.
22
88
  */
23
- browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
89
+ readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
24
90
  /**
25
91
  * Browser representation of the underlying automation library.
26
92
  */
@@ -30,12 +96,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
30
96
  */
31
97
  launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
32
98
  /**
33
- * The proxy tier tied to this browser controller.
34
- * `undefined` if no tiered proxy is used.
35
- */
36
- proxyTier?: number;
37
- /**
38
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
99
+ * The proxy URL used by the browser controller.
39
100
  * `undefined` if no proxy is used
40
101
  */
41
102
  proxyUrl?: string;
@@ -43,10 +104,10 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
43
104
  activePages: number;
44
105
  totalPages: number;
45
106
  lastPageOpenedAt: number;
46
- private _activate;
47
- private isActivePromise;
48
- private commitBrowser;
49
- private hasBrowserPromise;
107
+ /**
108
+ * Returns a promise that resolves once the browser controller has been activated.
109
+ */
110
+ waitForActive(): Promise<void>;
50
111
  constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
51
112
  /**
52
113
  * Activates the BrowserController. If you try to open new pages before
@@ -104,4 +165,3 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
104
165
  */
105
166
  abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
106
167
  }
107
- //# sourceMappingURL=browser-controller.d.ts.map
@@ -1,7 +1,8 @@
1
+ import { serviceLocator } from '@crawlee/core';
1
2
  import { nanoid } from 'nanoid';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { tryCancel } from '@apify/timeout';
4
- import { log } from '../logger.js';
5
+ import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
5
6
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
7
  /**
7
8
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +14,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
13
14
  */
14
15
  export class BrowserController extends TypedEmitter {
15
16
  id = nanoid();
17
+ log;
16
18
  /**
17
19
  * The `BrowserPlugin` instance used to launch the browser.
18
20
  */
@@ -26,12 +28,7 @@ export class BrowserController extends TypedEmitter {
26
28
  */
27
29
  launchContext = undefined;
28
30
  /**
29
- * The proxy tier tied to this browser controller.
30
- * `undefined` if no tiered proxy is used.
31
- */
32
- proxyTier;
33
- /**
34
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
31
+ * The proxy URL used by the browser controller.
35
32
  * `undefined` if no proxy is used
36
33
  */
37
34
  proxyUrl;
@@ -39,16 +36,23 @@ export class BrowserController extends TypedEmitter {
39
36
  activePages = 0;
40
37
  totalPages = 0;
41
38
  lastPageOpenedAt = Date.now();
42
- _activate;
43
- isActivePromise = new Promise((resolve) => {
44
- this._activate = resolve;
39
+ #activate;
40
+ #isActivePromise = new Promise((resolve) => {
41
+ this.#activate = resolve;
45
42
  });
46
- commitBrowser;
47
- hasBrowserPromise = new Promise((resolve) => {
48
- this.commitBrowser = resolve;
43
+ /**
44
+ * Returns a promise that resolves once the browser controller has been activated.
45
+ */
46
+ async waitForActive() {
47
+ await this.#isActivePromise;
48
+ }
49
+ #commitBrowser;
50
+ #hasBrowserPromise = new Promise((resolve) => {
51
+ this.#commitBrowser = resolve;
49
52
  });
50
53
  constructor(browserPlugin) {
51
54
  super();
55
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
52
56
  this.browserPlugin = browserPlugin;
53
57
  }
54
58
  /**
@@ -61,7 +65,7 @@ export class BrowserController extends TypedEmitter {
61
65
  if (!this.browser) {
62
66
  throw new Error('Cannot activate BrowserController without an assigned browser.');
63
67
  }
64
- this._activate();
68
+ this.#activate();
65
69
  this.isActive = true;
66
70
  }
67
71
  /**
@@ -73,7 +77,7 @@ export class BrowserController extends TypedEmitter {
73
77
  }
74
78
  this.browser = browser;
75
79
  this.launchContext = launchContext;
76
- this.commitBrowser();
80
+ this.#commitBrowser();
77
81
  }
78
82
  /**
79
83
  * Gracefully closes the browser and makes sure
@@ -82,21 +86,22 @@ export class BrowserController extends TypedEmitter {
82
86
  * Emits 'browserClosed' event.
83
87
  */
84
88
  async close() {
85
- await this.hasBrowserPromise;
89
+ await this.#hasBrowserPromise;
86
90
  try {
87
91
  await this._close();
88
92
  // TODO: shouldn't this go in a finally instead?
89
93
  this.isActive = false;
90
94
  }
91
95
  catch (error) {
92
- log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
96
+ this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
93
97
  }
94
- this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
95
- setTimeout(() => {
98
+ this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
99
+ const killTimer = setTimeout(() => {
96
100
  this._kill().catch((err) => {
97
- log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
101
+ this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
98
102
  });
99
103
  }, PROCESS_KILL_TIMEOUT_MILLIS);
104
+ killTimer.unref();
100
105
  }
101
106
  /**
102
107
  * Immediately kills the browser process.
@@ -104,9 +109,9 @@ export class BrowserController extends TypedEmitter {
104
109
  * Emits 'browserClosed' event.
105
110
  */
106
111
  async kill() {
107
- await this.hasBrowserPromise;
112
+ await this.#hasBrowserPromise;
108
113
  await this._kill();
109
- this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
114
+ this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
110
115
  }
111
116
  /**
112
117
  * Opens new browser page.
@@ -115,7 +120,7 @@ export class BrowserController extends TypedEmitter {
115
120
  async newPage(pageOptions) {
116
121
  this.activePages++;
117
122
  this.totalPages++;
118
- await this.isActivePromise;
123
+ await this.#isActivePromise;
119
124
  const page = await this._newPage(pageOptions);
120
125
  tryCancel();
121
126
  this.lastPageOpenedAt = Date.now();
@@ -128,4 +133,3 @@ export class BrowserController extends TypedEmitter {
128
133
  return this._getCookies(page);
129
134
  }
130
135
  }
131
- //# sourceMappingURL=browser-controller.js.map
@@ -1,8 +1,9 @@
1
- import { CriticalError } from '@crawlee/core';
1
+ import { type CrawleeLogger, CriticalError } from '@crawlee/core';
2
2
  import type { Dictionary } from '@crawlee/types';
3
3
  import type { LaunchContextOptions } from '../launch-context.js';
4
4
  import { LaunchContext } from '../launch-context.js';
5
- import type { UnwrapPromise } from '../utils.js';
5
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
6
+ import { type UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserController } from './browser-controller.js';
7
8
  /**
8
9
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
@@ -27,14 +28,13 @@ export interface CommonLibrary {
27
28
  launch(opts?: Dictionary): Promise<CommonBrowser>;
28
29
  name?: () => string;
29
30
  }
30
- /** @internal */
31
31
  export interface CommonBrowser {
32
32
  newPage(...args: unknown[]): Promise<CommonPage>;
33
33
  }
34
- /** @internal */
35
34
  export interface CommonPage {
36
35
  close(...args: unknown[]): Promise<unknown>;
37
36
  url(): string | Promise<string>;
37
+ evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
38
38
  }
39
39
  export interface BrowserPluginOptions<LibraryOptions> {
40
40
  /**
@@ -68,6 +68,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
68
68
  * Might cause performance issues, as Crawlee might launch too many browser instances.
69
69
  */
70
70
  browserPerProxy?: boolean;
71
+ /**
72
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
73
+ * This is useful when using HTTPS proxies with self-signed certificates.
74
+ */
75
+ ignoreProxyCertificate?: boolean;
71
76
  }
72
77
  export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
73
78
  }
@@ -78,14 +83,43 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
78
83
  * feed them to {@link BrowserPool} for use.
79
84
  */
80
85
  export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
81
- name: string;
82
- library: Library;
83
- launchOptions: LibraryOptions;
84
- proxyUrl?: string;
85
- userDataDir?: string;
86
+ readonly name: string;
87
+ protected readonly log: CrawleeLogger;
88
+ readonly library: Library;
89
+ readonly launchOptions: LibraryOptions;
90
+ readonly proxyUrl?: string;
91
+ readonly userDataDir?: string;
86
92
  useIncognitoPages: boolean;
87
- browserPerProxy?: boolean;
93
+ readonly browserPerProxy?: boolean;
94
+ readonly ignoreProxyCertificate?: boolean;
95
+ /**
96
+ * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
97
+ * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
98
+ * remote-session policy lives in the pool, not here.
99
+ *
100
+ * @internal
101
+ */
102
+ remoteConnection?: RemoteConnection;
103
+ /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
104
+ remoteConnectionParameters?: RemoteConnectionParameters;
88
105
  constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
106
+ /**
107
+ * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
108
+ * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
109
+ * (e.g. forcing incognito pages).
110
+ *
111
+ * @internal
112
+ */
113
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
114
+ /**
115
+ * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
116
+ * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
117
+ * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
118
+ *
119
+ * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
120
+ * lives here so it stays identical across plugins.
121
+ */
122
+ protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
89
123
  /**
90
124
  * Creates a `LaunchContext` with all the information needed
91
125
  * to launch a browser. Aside from library specific launch options,
@@ -98,13 +132,13 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
98
132
  * Launches the browser using provided launch context.
99
133
  */
100
134
  launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
101
- private _mergeArgsToHideWebdriver;
102
- protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
135
+ private mergeArgsToHideWebdriver;
136
+ protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
103
137
  /**
104
138
  * @private
105
139
  */
106
- protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
107
- protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
140
+ protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
141
+ protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
108
142
  /**
109
143
  * @private
110
144
  */
@@ -113,4 +147,3 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
113
147
  export declare class BrowserLaunchError extends CriticalError {
114
148
  constructor(...args: ConstructorParameters<typeof CriticalError>);
115
149
  }
116
- //# sourceMappingURL=browser-plugin.d.ts.map
@@ -1,6 +1,7 @@
1
- import { CriticalError } from '@crawlee/core';
1
+ import { CriticalError, serviceLocator } from '@crawlee/core';
2
2
  import merge from 'lodash.merge';
3
3
  import { LaunchContext } from '../launch-context.js';
4
+ import { sanitizeEndpointForLog } from '../utils.js';
4
5
  /**
5
6
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
6
7
  * when Chromium/Chrome browser is launched:
@@ -20,20 +21,73 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
20
21
  */
21
22
  export class BrowserPlugin {
22
23
  name = this.constructor.name;
24
+ log;
23
25
  library;
24
26
  launchOptions;
25
27
  proxyUrl;
26
28
  userDataDir;
27
29
  useIncognitoPages;
28
30
  browserPerProxy;
31
+ ignoreProxyCertificate;
32
+ /**
33
+ * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
34
+ * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
35
+ * remote-session policy lives in the pool, not here.
36
+ *
37
+ * @internal
38
+ */
39
+ remoteConnection;
40
+ /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
41
+ remoteConnectionParameters;
29
42
  constructor(library, options = {}) {
30
- const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
43
+ const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
44
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
31
45
  this.library = library;
32
46
  this.launchOptions = launchOptions;
33
47
  this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
34
48
  this.userDataDir = userDataDir;
35
49
  this.useIncognitoPages = useIncognitoPages;
36
50
  this.browserPerProxy = browserPerProxy;
51
+ this.ignoreProxyCertificate = ignoreProxyCertificate;
52
+ }
53
+ /**
54
+ * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
55
+ * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
56
+ * (e.g. forcing incognito pages).
57
+ *
58
+ * @internal
59
+ */
60
+ useRemoteConnection(connection, parameters = {}) {
61
+ this.remoteConnection = connection;
62
+ this.remoteConnectionParameters = parameters;
63
+ }
64
+ /**
65
+ * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
66
+ * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
67
+ * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
68
+ *
69
+ * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
70
+ * lives here so it stays identical across plugins.
71
+ */
72
+ async connectToRemoteBrowser(launchContext, connect) {
73
+ const connection = this.remoteConnection;
74
+ let url;
75
+ let token;
76
+ try {
77
+ ({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
78
+ }
79
+ catch (cause) {
80
+ throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
81
+ }
82
+ launchContext.remoteToken = token;
83
+ try {
84
+ return await connect(url);
85
+ }
86
+ catch (cause) {
87
+ await connection.release(token);
88
+ throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
89
+ 'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
90
+ }
37
91
  }
38
92
  /**
39
93
  * Creates a `LaunchContext` with all the information needed
@@ -42,7 +96,7 @@ export class BrowserPlugin {
42
96
  * management of the pool and extra features.
43
97
  */
44
98
  createLaunchContext(options = {}) {
45
- const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
99
+ const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote = !!this.remoteConnection, } = options;
46
100
  return new LaunchContext({
47
101
  id,
48
102
  launchOptions: merge({}, this.launchOptions, launchOptions),
@@ -51,21 +105,23 @@ export class BrowserPlugin {
51
105
  useIncognitoPages,
52
106
  userDataDir,
53
107
  browserPerProxy,
54
- proxyTier,
108
+ ignoreProxyCertificate,
109
+ isRemote,
55
110
  });
56
111
  }
57
112
  /**
58
113
  * Launches the browser using provided launch context.
59
114
  */
60
115
  async launch(launchContext = this.createLaunchContext()) {
116
+ // launchOptions is only used by the local launch path below — remote connections ignore it.
61
117
  launchContext.launchOptions ??= {};
62
118
  const { proxyUrl, launchOptions } = launchContext;
63
- if (proxyUrl) {
64
- await this._addProxyToLaunchOptions(launchContext);
119
+ if (proxyUrl && !launchContext.isRemote) {
120
+ await this.addProxyToLaunchOptions(launchContext);
65
121
  }
66
- if (this._isChromiumBasedBrowser(launchContext)) {
122
+ if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
67
123
  // This will set the args for chromium based browsers to hide the webdriver.
68
- launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
124
+ launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
69
125
  // When User-Agent is not set, and we're using Chromium in headless mode,
70
126
  // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
71
127
  // as otherwise 'HeadlessChrome' is present in User-Agent string.
@@ -74,9 +130,12 @@ export class BrowserPlugin {
74
130
  launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
75
131
  }
76
132
  }
133
+ if (launchContext.isRemote) {
134
+ this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
135
+ }
77
136
  return this._launch(launchContext);
78
137
  }
79
- _mergeArgsToHideWebdriver(originalArgs) {
138
+ mergeArgsToHideWebdriver(originalArgs) {
80
139
  if (!originalArgs?.length) {
81
140
  return ['--disable-blink-features=AutomationControlled'];
82
141
  }
@@ -89,7 +148,7 @@ export class BrowserPlugin {
89
148
  }
90
149
  return originalArgs;
91
150
  }
92
- _throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
151
+ throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
93
152
  const errorMessage = ['Failed to launch browser. Please check the following:'];
94
153
  if (executablePath) {
95
154
  errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
@@ -118,4 +177,3 @@ export class BrowserLaunchError extends CriticalError {
118
177
  });
119
178
  }
120
179
  }
121
- //# sourceMappingURL=browser-plugin.js.map
@@ -1,4 +1,7 @@
1
1
  type PromiseVoid = () => Promise<void>;
2
- export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string) => Promise<[string | undefined, PromiseVoid]>;
2
+ export interface AnonymizeProxySugarOptions {
3
+ ignoreProxyCertificate?: boolean;
4
+ }
5
+ /** @internal */
6
+ export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
3
7
  export {};
4
- //# sourceMappingURL=anonymize-proxy.d.ts.map
@@ -1,15 +1,20 @@
1
- import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
2
- export const anonymizeProxySugar = async (proxyUrl, username, password) => {
1
+ /** @internal */
2
+ export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
3
3
  if (proxyUrl) {
4
4
  const url = new URL(proxyUrl);
5
5
  if (username || password) {
6
6
  url.username = username ?? '';
7
7
  url.password = password ?? '';
8
8
  }
9
- if (url.username || url.password) {
9
+ if (url.username || url.password || options?.ignoreProxyCertificate) {
10
10
  // trim off trailing slash if it's present
11
11
  const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
12
- const anonymized = await anonymizeProxy(proxyUrlString);
12
+ const { anonymizeProxy, closeAnonymizedProxy } = await import('proxy-chain');
13
+ const anonymized = await anonymizeProxy({
14
+ url: proxyUrlString,
15
+ port: 0,
16
+ ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
17
+ });
13
18
  return [
14
19
  anonymized,
15
20
  async () => {
@@ -21,4 +26,3 @@ export const anonymizeProxySugar = async (proxyUrl, username, password) => {
21
26
  }
22
27
  return [undefined, async () => { }];
23
28
  };
24
- //# sourceMappingURL=anonymize-proxy.js.map