@crawlee/browser-pool 4.0.0-beta.21 → 4.0.0-beta.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +14 -14
  2. package/abstract-classes/browser-controller.d.ts +100 -29
  3. package/abstract-classes/browser-controller.js +70 -24
  4. package/abstract-classes/browser-plugin.d.ts +54 -23
  5. package/abstract-classes/browser-plugin.js +75 -13
  6. package/anonymize-proxy.d.ts +6 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +123 -49
  9. package/browser-pool.js +320 -147
  10. package/events.d.ts +2 -4
  11. package/events.js +0 -2
  12. package/fingerprinting/hooks.d.ts +0 -1
  13. package/fingerprinting/hooks.js +26 -10
  14. package/fingerprinting/types.d.ts +3 -34
  15. package/fingerprinting/types.js +0 -2
  16. package/fingerprinting/utils.d.ts +0 -1
  17. package/fingerprinting/utils.js +5 -6
  18. package/index.d.ts +14 -7
  19. package/index.js +5 -3
  20. package/launch-context.d.ts +21 -5
  21. package/launch-context.js +21 -10
  22. package/package.json +9 -9
  23. package/playwright/playwright-browser.d.ts +3 -6
  24. package/playwright/playwright-browser.js +17 -17
  25. package/playwright/playwright-controller.d.ts +1 -1
  26. package/playwright/playwright-controller.js +10 -6
  27. package/playwright/playwright-plugin.d.ts +12 -7
  28. package/playwright/playwright-plugin.js +42 -16
  29. package/proxy-server.d.ts +0 -1
  30. package/proxy-server.js +0 -1
  31. package/puppeteer/puppeteer-controller.d.ts +1 -1
  32. package/puppeteer/puppeteer-controller.js +29 -20
  33. package/puppeteer/puppeteer-plugin.d.ts +9 -3
  34. package/puppeteer/puppeteer-plugin.js +87 -46
  35. package/remote-browser-pool.d.ts +178 -0
  36. package/remote-browser-pool.js +206 -0
  37. package/remote-browser-provider.d.ts +83 -0
  38. package/remote-browser-provider.js +67 -0
  39. package/utils.d.ts +7 -1
  40. package/utils.js +19 -1
  41. package/abstract-classes/browser-controller.d.ts.map +0 -1
  42. package/abstract-classes/browser-controller.js.map +0 -1
  43. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  44. package/abstract-classes/browser-plugin.js.map +0 -1
  45. package/anonymize-proxy.d.ts.map +0 -1
  46. package/anonymize-proxy.js.map +0 -1
  47. package/browser-pool.d.ts.map +0 -1
  48. package/browser-pool.js.map +0 -1
  49. package/container-proxy-server.d.ts +0 -12
  50. package/container-proxy-server.d.ts.map +0 -1
  51. package/container-proxy-server.js +0 -43
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/utils.d.ts.map +0 -1
  82. package/utils.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
@@ -1,9 +1,73 @@
1
+ import { type CrawleeLogger } from '@crawlee/core';
1
2
  import type { Cookie, Dictionary } from '@crawlee/types';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
4
5
  import type { LaunchContext } from '../launch-context.js';
5
6
  import type { UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
8
+ /**
9
+ * The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
10
+ * Other fields are only available on the concrete `LaunchContext` class.
11
+ */
12
+ export interface IBrowserLaunchContext {
13
+ /**
14
+ * The proxy URL the browser was launched with, if any.
15
+ */
16
+ proxyUrl?: string;
17
+ /**
18
+ * The fingerprint applied to the browser, if fingerprinting is enabled.
19
+ * Typed as `unknown` here; cast to the concrete `LaunchContext` if you
20
+ * need the structured shape.
21
+ */
22
+ fingerprint?: unknown;
23
+ /**
24
+ * `true` if each page in this browser uses its own context.
25
+ */
26
+ useIncognitoPages?: boolean;
27
+ /**
28
+ * The actual options the browser was launched with, after pre-launch hooks.
29
+ */
30
+ launchOptions?: Dictionary | undefined;
31
+ }
32
+ /**
33
+ * The minimal public contract of a browser controller.
34
+ *
35
+ * Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
36
+ * promises, …) is intentionally **not** part of this contract.
37
+ *
38
+ * @category Browser management
39
+ */
40
+ export interface IBrowserController<Page = unknown> {
41
+ /**
42
+ * A stable identifier for this controller instance. Useful for tracking
43
+ * which browser served which request.
44
+ */
45
+ readonly id: string;
46
+ /**
47
+ * The configuration the underlying browser was launched with — proxy URL,
48
+ * fingerprint, session, launcher-specific options, etc.
49
+ */
50
+ readonly launchContext: IBrowserLaunchContext;
51
+ /**
52
+ * The raw browser handle from the underlying automation library
53
+ * (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
54
+ * Escape hatch for things the controller does not expose directly.
55
+ */
56
+ readonly browser: unknown;
57
+ /**
58
+ * Reads cookies for the given page.
59
+ */
60
+ getCookies(page: Page): Promise<Cookie[]>;
61
+ /**
62
+ * Writes cookies for the given page.
63
+ */
64
+ setCookies(page: Page, cookies: Cookie[]): Promise<void>;
65
+ /**
66
+ * Gracefully closes the browser this controller owns. After this resolves,
67
+ * the controller is no longer usable.
68
+ */
69
+ close(): Promise<void>;
70
+ }
7
71
  export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
8
72
  [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
9
73
  }
@@ -15,12 +79,20 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
15
79
  * for the specialized classes, because it's the same for all of them.
16
80
  * @hideconstructor
17
81
  */
18
- export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
19
- id: string;
82
+ export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
83
+ #private;
84
+ readonly id: string;
85
+ /**
86
+ * Kept `protected` rather than `#private` because the concrete controllers in this package are
87
+ * separate classes — an ES private field would not be visible to them.
88
+ *
89
+ * @internal
90
+ */
91
+ protected readonly log: CrawleeLogger;
20
92
  /**
21
93
  * The `BrowserPlugin` instance used to launch the browser.
22
94
  */
23
- browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
95
+ readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
24
96
  /**
25
97
  * Browser representation of the underlying automation library.
26
98
  */
@@ -30,23 +102,21 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
30
102
  */
31
103
  launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
32
104
  /**
33
- * The proxy tier tied to this browser controller.
34
- * `undefined` if no tiered proxy is used.
35
- */
36
- proxyTier?: number;
37
- /**
38
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
105
+ * The proxy URL used by the browser controller.
39
106
  * `undefined` if no proxy is used
40
107
  */
41
108
  proxyUrl?: string;
109
+ /** @internal */
42
110
  isActive: boolean;
43
111
  activePages: number;
112
+ /** @internal */
44
113
  totalPages: number;
114
+ /** @internal */
45
115
  lastPageOpenedAt: number;
46
- private _activate;
47
- private isActivePromise;
48
- private commitBrowser;
49
- private hasBrowserPromise;
116
+ /**
117
+ * Returns a promise that resolves once the browser controller has been activated.
118
+ */
119
+ waitForActive(): Promise<void>;
50
120
  constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
51
121
  /**
52
122
  * Activates the BrowserController. If you try to open new pages before
@@ -77,31 +147,32 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
77
147
  * @ignore
78
148
  */
79
149
  newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
80
- setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
81
- getCookies(page: NewPageResult): Promise<Cookie[]>;
82
150
  /**
83
- * @private
151
+ * Accounts for a page no longer being open. Callable from either side and safe to call twice:
152
+ * the page's own `close` event, or the pool when it gives up on a close that never settles.
153
+ * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
154
+ * @internal
84
155
  */
85
- protected abstract _close(): Promise<void>;
156
+ registerPageClosed(page: NewPageResult): void;
86
157
  /**
87
- * @private
158
+ * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
159
+ * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
160
+ * or the pool gave up on a close that never settled, and it runs at most once.
161
+ *
162
+ * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
163
+ * now handles, because that event does not arrive for a page the browser never destroyed.
164
+ * @ignore
88
165
  */
166
+ protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
167
+ setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
168
+ getCookies(page: NewPageResult): Promise<Cookie[]>;
169
+ protected abstract _close(): Promise<void>;
89
170
  protected abstract _kill(): Promise<void>;
90
- /**
91
- * @private
92
- */
93
171
  protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
94
- /**
95
- * @private
96
- */
97
172
  protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
98
- /**
99
- * @private
100
- */
101
173
  protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
102
174
  /**
103
- * @private
175
+ * @internal
104
176
  */
105
177
  abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
106
178
  }
107
- //# sourceMappingURL=browser-controller.d.ts.map
@@ -1,7 +1,8 @@
1
+ import { serviceLocator } from '@crawlee/core';
1
2
  import { nanoid } from 'nanoid';
2
3
  import { TypedEmitter } from 'tiny-typed-emitter';
3
4
  import { tryCancel } from '@apify/timeout';
4
- import { log } from '../logger.js';
5
+ import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
5
6
  const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
6
7
  /**
7
8
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +14,13 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
13
14
  */
14
15
  export class BrowserController extends TypedEmitter {
15
16
  id = nanoid();
17
+ /**
18
+ * Kept `protected` rather than `#private` because the concrete controllers in this package are
19
+ * separate classes — an ES private field would not be visible to them.
20
+ *
21
+ * @internal
22
+ */
23
+ log;
16
24
  /**
17
25
  * The `BrowserPlugin` instance used to launch the browser.
18
26
  */
@@ -26,29 +34,36 @@ export class BrowserController extends TypedEmitter {
26
34
  */
27
35
  launchContext = undefined;
28
36
  /**
29
- * The proxy tier tied to this browser controller.
30
- * `undefined` if no tiered proxy is used.
31
- */
32
- proxyTier;
33
- /**
34
- * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
37
+ * The proxy URL used by the browser controller.
35
38
  * `undefined` if no proxy is used
36
39
  */
37
40
  proxyUrl;
41
+ /** @internal */
38
42
  isActive = false;
39
43
  activePages = 0;
44
+ #closedPages = new WeakSet();
45
+ #pageTeardowns = new WeakMap();
46
+ /** @internal */
40
47
  totalPages = 0;
48
+ /** @internal */
41
49
  lastPageOpenedAt = Date.now();
42
- _activate;
43
- isActivePromise = new Promise((resolve) => {
44
- this._activate = resolve;
50
+ #activate;
51
+ #isActivePromise = new Promise((resolve) => {
52
+ this.#activate = resolve;
45
53
  });
46
- commitBrowser;
47
- hasBrowserPromise = new Promise((resolve) => {
48
- this.commitBrowser = resolve;
54
+ /**
55
+ * Returns a promise that resolves once the browser controller has been activated.
56
+ */
57
+ async waitForActive() {
58
+ await this.#isActivePromise;
59
+ }
60
+ #commitBrowser;
61
+ #hasBrowserPromise = new Promise((resolve) => {
62
+ this.#commitBrowser = resolve;
49
63
  });
50
64
  constructor(browserPlugin) {
51
65
  super();
66
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
52
67
  this.browserPlugin = browserPlugin;
53
68
  }
54
69
  /**
@@ -61,7 +76,7 @@ export class BrowserController extends TypedEmitter {
61
76
  if (!this.browser) {
62
77
  throw new Error('Cannot activate BrowserController without an assigned browser.');
63
78
  }
64
- this._activate();
79
+ this.#activate();
65
80
  this.isActive = true;
66
81
  }
67
82
  /**
@@ -73,7 +88,7 @@ export class BrowserController extends TypedEmitter {
73
88
  }
74
89
  this.browser = browser;
75
90
  this.launchContext = launchContext;
76
- this.commitBrowser();
91
+ this.#commitBrowser();
77
92
  }
78
93
  /**
79
94
  * Gracefully closes the browser and makes sure
@@ -82,21 +97,22 @@ export class BrowserController extends TypedEmitter {
82
97
  * Emits 'browserClosed' event.
83
98
  */
84
99
  async close() {
85
- await this.hasBrowserPromise;
100
+ await this.#hasBrowserPromise;
86
101
  try {
87
102
  await this._close();
88
103
  // TODO: shouldn't this go in a finally instead?
89
104
  this.isActive = false;
90
105
  }
91
106
  catch (error) {
92
- log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
107
+ this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
93
108
  }
94
- this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
95
- setTimeout(() => {
109
+ this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
110
+ const killTimer = setTimeout(() => {
96
111
  this._kill().catch((err) => {
97
- log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
112
+ this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
98
113
  });
99
114
  }, PROCESS_KILL_TIMEOUT_MILLIS);
115
+ killTimer.unref();
100
116
  }
101
117
  /**
102
118
  * Immediately kills the browser process.
@@ -104,9 +120,9 @@ export class BrowserController extends TypedEmitter {
104
120
  * Emits 'browserClosed' event.
105
121
  */
106
122
  async kill() {
107
- await this.hasBrowserPromise;
123
+ await this.#hasBrowserPromise;
108
124
  await this._kill();
109
- this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
125
+ this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
110
126
  }
111
127
  /**
112
128
  * Opens new browser page.
@@ -115,12 +131,43 @@ export class BrowserController extends TypedEmitter {
115
131
  async newPage(pageOptions) {
116
132
  this.activePages++;
117
133
  this.totalPages++;
118
- await this.isActivePromise;
134
+ await this.#isActivePromise;
119
135
  const page = await this._newPage(pageOptions);
120
136
  tryCancel();
121
137
  this.lastPageOpenedAt = Date.now();
122
138
  return page;
123
139
  }
140
+ /**
141
+ * Accounts for a page no longer being open. Callable from either side and safe to call twice:
142
+ * the page's own `close` event, or the pool when it gives up on a close that never settles.
143
+ * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
144
+ * @internal
145
+ */
146
+ registerPageClosed(page) {
147
+ if (this.#closedPages.has(page))
148
+ return;
149
+ this.#closedPages.add(page);
150
+ this.activePages--;
151
+ const teardown = this.#pageTeardowns.get(page);
152
+ if (!teardown)
153
+ return;
154
+ this.#pageTeardowns.delete(page);
155
+ teardown().catch((error) => {
156
+ this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
157
+ });
158
+ }
159
+ /**
160
+ * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
161
+ * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
162
+ * or the pool gave up on a close that never settled, and it runs at most once.
163
+ *
164
+ * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
165
+ * now handles, because that event does not arrive for a page the browser never destroyed.
166
+ * @ignore
167
+ */
168
+ registerPageTeardown(page, teardown) {
169
+ this.#pageTeardowns.set(page, teardown);
170
+ }
124
171
  async setCookies(page, cookies) {
125
172
  return this._setCookies(page, cookies);
126
173
  }
@@ -128,4 +175,3 @@ export class BrowserController extends TypedEmitter {
128
175
  return this._getCookies(page);
129
176
  }
130
177
  }
131
- //# sourceMappingURL=browser-controller.js.map
@@ -1,8 +1,9 @@
1
- import { CriticalError } from '@crawlee/core';
1
+ import { type CrawleeLogger, CriticalError } from '@crawlee/core';
2
2
  import type { Dictionary } from '@crawlee/types';
3
3
  import type { LaunchContextOptions } from '../launch-context.js';
4
4
  import { LaunchContext } from '../launch-context.js';
5
- import type { UnwrapPromise } from '../utils.js';
5
+ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
6
+ import { type UnwrapPromise } from '../utils.js';
6
7
  import type { BrowserController } from './browser-controller.js';
7
8
  /**
8
9
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
@@ -11,8 +12,6 @@ import type { BrowserController } from './browser-controller.js';
11
12
  * - without using a fingerprint,
12
13
  * - without specifying a user agent.
13
14
  * Last updated on 2022-05-05.
14
- *
15
- * After you update it here, please update it also in jsdom-crawler.ts
16
15
  */
17
16
  export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
18
17
  /**
@@ -27,14 +26,13 @@ export interface CommonLibrary {
27
26
  launch(opts?: Dictionary): Promise<CommonBrowser>;
28
27
  name?: () => string;
29
28
  }
30
- /** @internal */
31
29
  export interface CommonBrowser {
32
30
  newPage(...args: unknown[]): Promise<CommonPage>;
33
31
  }
34
- /** @internal */
35
32
  export interface CommonPage {
36
33
  close(...args: unknown[]): Promise<unknown>;
37
34
  url(): string | Promise<string>;
35
+ evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
38
36
  }
39
37
  export interface BrowserPluginOptions<LibraryOptions> {
40
38
  /**
@@ -68,6 +66,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
68
66
  * Might cause performance issues, as Crawlee might launch too many browser instances.
69
67
  */
70
68
  browserPerProxy?: boolean;
69
+ /**
70
+ * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
71
+ * This is useful when using HTTPS proxies with self-signed certificates.
72
+ */
73
+ ignoreProxyCertificate?: boolean;
71
74
  }
72
75
  export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
73
76
  }
@@ -78,14 +81,49 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
78
81
  * feed them to {@link BrowserPool} for use.
79
82
  */
80
83
  export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
81
- name: string;
82
- library: Library;
83
- launchOptions: LibraryOptions;
84
- proxyUrl?: string;
85
- userDataDir?: string;
84
+ readonly name: string;
85
+ /**
86
+ * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
87
+ * classes — an ES private field would not be visible to them.
88
+ *
89
+ * @internal
90
+ */
91
+ protected readonly log: CrawleeLogger;
92
+ readonly library: Library;
93
+ readonly launchOptions: LibraryOptions;
94
+ readonly proxyUrl?: string;
95
+ readonly userDataDir?: string;
86
96
  useIncognitoPages: boolean;
87
- browserPerProxy?: boolean;
97
+ readonly browserPerProxy?: boolean;
98
+ readonly ignoreProxyCertificate?: boolean;
99
+ /**
100
+ * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
101
+ * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
102
+ * remote-session policy lives in the pool, not here.
103
+ *
104
+ * @internal
105
+ */
106
+ remoteConnection?: RemoteConnection;
107
+ /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
108
+ remoteConnectionParameters?: RemoteConnectionParameters;
88
109
  constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
110
+ /**
111
+ * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
112
+ * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
113
+ * (e.g. forcing incognito pages).
114
+ *
115
+ * @internal
116
+ */
117
+ useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
118
+ /**
119
+ * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
120
+ * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
121
+ * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
122
+ *
123
+ * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
124
+ * lives here so it stays identical across plugins.
125
+ */
126
+ protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
89
127
  /**
90
128
  * Creates a `LaunchContext` with all the information needed
91
129
  * to launch a browser. Aside from library specific launch options,
@@ -98,19 +136,12 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
98
136
  * Launches the browser using provided launch context.
99
137
  */
100
138
  launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
101
- private _mergeArgsToHideWebdriver;
102
- protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
103
- /**
104
- * @private
105
- */
106
- protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
107
- protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
108
- /**
109
- * @private
110
- */
139
+ private mergeArgsToHideWebdriver;
140
+ protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
141
+ protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
142
+ protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
111
143
  protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
112
144
  }
113
145
  export declare class BrowserLaunchError extends CriticalError {
114
146
  constructor(...args: ConstructorParameters<typeof CriticalError>);
115
147
  }
116
- //# sourceMappingURL=browser-plugin.d.ts.map