@crawlee/browser-pool 4.0.0-beta.13 → 4.0.0-beta.130

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +17 -13
  2. package/abstract-classes/browser-controller.d.ts +70 -13
  3. package/abstract-classes/browser-controller.js +21 -22
  4. package/abstract-classes/browser-plugin.d.ts +48 -15
  5. package/abstract-classes/browser-plugin.js +69 -11
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +100 -28
  9. package/browser-pool.js +211 -83
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +2 -3
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +26 -10
  16. package/fingerprinting/types.d.ts +2 -3
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +5 -6
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +20 -5
  23. package/launch-context.js +21 -10
  24. package/package.json +10 -11
  25. package/playwright/playwright-browser.d.ts +2 -6
  26. package/playwright/playwright-browser.js +16 -17
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +20 -10
  29. package/playwright/playwright-plugin.d.ts +10 -7
  30. package/playwright/playwright-plugin.js +40 -16
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +5 -3
  36. package/puppeteer/puppeteer-plugin.js +83 -46
  37. package/remote-browser-pool.d.ts +164 -0
  38. package/remote-browser-pool.js +192 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/tsconfig.build.tsbuildinfo +0 -1
  82. package/utils.d.ts.map +0 -1
  83. package/utils.js.map +0 -1
package/browser-pool.js CHANGED
@@ -1,15 +1,34 @@
1
+ import { AsyncResource } from 'node:async_hooks';
2
+ import { parseArgument, schemas, SessionError, serviceLocator } from '@crawlee/core';
1
3
  import { FingerprintGenerator } from 'fingerprint-generator';
2
4
  import { FingerprintInjector } from 'fingerprint-injector';
3
5
  import { nanoid } from 'nanoid';
4
- import ow from 'ow';
5
6
  import pLimit from 'p-limit';
6
7
  import QuickLRU from 'quick-lru';
7
8
  import { TypedEmitter } from 'tiny-typed-emitter';
9
+ import { z } from 'zod';
8
10
  import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
11
+ import { BROWSER_POOL_EVENTS } from './events.js';
9
12
  import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
10
- import { log } from './logger.js';
13
+ const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
11
14
  const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
12
15
  const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
16
+ const browserPoolOptionsSchema = z.strictObject({
17
+ browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
18
+ maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
19
+ retireBrowserAfterPageCount: schemas.anyNumber.default(100),
20
+ operationTimeoutSecs: schemas.anyNumber.default(15),
21
+ closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
22
+ retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
23
+ preLaunchHooks: schemas.anyArray.default(() => []),
24
+ postLaunchHooks: schemas.anyArray.default(() => []),
25
+ prePageCreateHooks: schemas.anyArray.default(() => []),
26
+ postPageCreateHooks: schemas.anyArray.default(() => []),
27
+ prePageCloseHooks: schemas.anyArray.default(() => []),
28
+ postPageCloseHooks: schemas.anyArray.default(() => []),
29
+ useFingerprints: z.boolean().default(true),
30
+ fingerprintOptions: schemas.anyObject.default(() => ({})),
31
+ });
13
32
  /**
14
33
  * The `BrowserPool` class is the most important class of the `browser-pool` module.
15
34
  * It manages opening and closing of browsers and their pages and its constructor
@@ -60,6 +79,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
60
79
  export class BrowserPool extends TypedEmitter {
61
80
  browserPlugins;
62
81
  maxOpenPagesPerBrowser;
82
+ maxOpenBrowsers;
63
83
  retireBrowserAfterPageCount;
64
84
  operationTimeoutMillis;
65
85
  closeInactiveBrowserAfterMillis;
@@ -74,35 +94,23 @@ export class BrowserPool extends TypedEmitter {
74
94
  pageCounter = 0;
75
95
  pages = new Map();
76
96
  pageIds = new WeakMap();
97
+ startingBrowserControllers = new Set();
77
98
  activeBrowserControllers = new Set();
78
99
  retiredBrowserControllers = new Set();
79
100
  pageToBrowserController = new WeakMap();
80
101
  fingerprintInjector;
81
102
  fingerprintGenerator;
82
103
  fingerprintCache;
83
- browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
84
- browserRetireInterval;
85
- limiter = pLimit(1);
104
+ // kept as TS-private: tests replace this interval through bracket access
105
+ browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
106
+ #browserRetireInterval;
107
+ #limiter = pLimit(1);
108
+ #log;
86
109
  constructor(options) {
87
110
  super();
111
+ this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
88
112
  this.browserKillerInterval.unref();
89
- ow(options, ow.object.exactShape({
90
- browserPlugins: ow.array.minLength(1),
91
- maxOpenPagesPerBrowser: ow.optional.number,
92
- retireBrowserAfterPageCount: ow.optional.number,
93
- operationTimeoutSecs: ow.optional.number,
94
- closeInactiveBrowserAfterSecs: ow.optional.number,
95
- retireInactiveBrowserAfterSecs: ow.optional.number,
96
- preLaunchHooks: ow.optional.array,
97
- postLaunchHooks: ow.optional.array,
98
- prePageCreateHooks: ow.optional.array,
99
- postPageCreateHooks: ow.optional.array,
100
- prePageCloseHooks: ow.optional.array,
101
- postPageCloseHooks: ow.optional.array,
102
- useFingerprints: ow.optional.boolean,
103
- fingerprintOptions: ow.optional.object,
104
- }));
105
- const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
113
+ const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
106
114
  const firstPluginConstructor = browserPlugins[0].constructor;
107
115
  for (let i = 1; i < browserPlugins.length; i++) {
108
116
  const providedPlugin = browserPlugins[i];
@@ -114,18 +122,19 @@ export class BrowserPool extends TypedEmitter {
114
122
  }
115
123
  this.browserPlugins = browserPlugins;
116
124
  this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
125
+ this.maxOpenBrowsers = Infinity;
117
126
  this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
118
127
  this.operationTimeoutMillis = operationTimeoutSecs * 1000;
119
128
  this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
120
129
  this.useFingerprints = useFingerprints;
121
130
  this.fingerprintOptions = fingerprintOptions;
122
- this.browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
131
+ this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
123
132
  if (controller.activePages === 0 &&
124
133
  controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
125
134
  this.retireBrowserController(controller);
126
135
  }
127
136
  }), retireInactiveBrowserAfterSecs * 1000);
128
- this.browserRetireInterval.unref();
137
+ this.#browserRetireInterval.unref();
129
138
  // hooks
130
139
  this.preLaunchHooks = preLaunchHooks;
131
140
  this.postLaunchHooks = postLaunchHooks;
@@ -135,30 +144,56 @@ export class BrowserPool extends TypedEmitter {
135
144
  this.postPageCloseHooks = postPageCloseHooks;
136
145
  // fingerprinting
137
146
  if (this.useFingerprints) {
138
- this._initializeFingerprinting();
147
+ this.initializeFingerprinting();
139
148
  }
140
149
  }
141
150
  /**
142
151
  * Opens a new page in one of the running browsers or launches
143
152
  * a new browser and opens a page there, if no browsers are active,
144
153
  * or their page limits have been exceeded.
154
+ *
155
+ * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
156
+ * provided, this implementation uses it as a cache key for browser fingerprints (when
157
+ * fingerprinting is enabled) and reads
158
+ * {@link ProxyInfo.url|session.proxyInfo.url} /
159
+ * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
160
+ * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
161
+ * `ignoreTlsErrors` values in the options take precedence.
162
+ *
163
+ * Beyond fingerprint caching and proxy configuration, no other session
164
+ * properties are consumed — cookie and header injection remain the
165
+ * crawler's responsibility.
145
166
  */
146
167
  async newPage(options = {}) {
147
- const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
168
+ const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
148
169
  if (this.pages.has(id)) {
149
170
  throw new Error(`Page with ID: ${id} already exists.`);
150
171
  }
151
172
  if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
152
173
  throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
153
174
  }
175
+ // Bind the limiter callback to the current async-hooks context. p-limit
176
+ // otherwise resumes queued callbacks in the previous task's
177
+ // AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
178
+ // requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
179
+ // fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
180
+ // ESM-only rewrite, so we can't bump it in Crawlee v3.
181
+ // Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
182
+ // ALS-scoped: without it, a queued callback would resume in the previous request's async
183
+ // context and run request B's storage writes inside request A's transaction.
184
+ // TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
154
185
  // Limiter is necessary - https://github.com/apify/crawlee/issues/1126
155
- return this.limiter(async () => {
156
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
186
+ return this.#limiter(AsyncResource.bind(async () => {
187
+ let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
157
188
  if (!browserController)
158
- browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
189
+ browserController = await this.launchBrowser(id, {
190
+ browserPlugin,
191
+ proxyUrl,
192
+ ignoreTlsErrors,
193
+ });
159
194
  tryCancel();
160
- return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
161
- });
195
+ return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
196
+ }));
162
197
  }
163
198
  /**
164
199
  * Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
@@ -166,13 +201,13 @@ export class BrowserPool extends TypedEmitter {
166
201
  * configure the new browser.
167
202
  */
168
203
  async newPageInNewBrowser(options = {}) {
169
- const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;
204
+ const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
170
205
  if (this.pages.has(id)) {
171
206
  throw new Error(`Page with ID: ${id} already exists.`);
172
207
  }
173
- const browserController = await this._launchBrowser(id, { launchOptions, browserPlugin });
208
+ const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
174
209
  tryCancel();
175
- return await this._createPageForBrowser(id, browserController, pageOptions);
210
+ return await this.createPageForBrowser(id, browserController, pageOptions);
176
211
  }
177
212
  /**
178
213
  * Opens new pages with all available plugins and returns an array
@@ -238,7 +273,7 @@ export class BrowserPool extends TypedEmitter {
238
273
  getPageId(page) {
239
274
  return this.pageIds.get(page);
240
275
  }
241
- async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
276
+ async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
242
277
  // This is needed for concurrent newPage calls to wait for the browser launch.
243
278
  // It's not ideal though, we need to come up with a better API.
244
279
  // eslint-disable-next-line dot-notation -- accessing private property
@@ -247,8 +282,14 @@ export class BrowserPool extends TypedEmitter {
247
282
  const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
248
283
  if (finalPageOptions) {
249
284
  Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
285
+ if (ignoreTlsErrors) {
286
+ Object.assign(finalPageOptions, {
287
+ ignoreHTTPSErrors: true,
288
+ acceptInsecureCerts: true,
289
+ });
290
+ }
250
291
  }
251
- await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
292
+ await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
252
293
  tryCancel();
253
294
  let page;
254
295
  try {
@@ -261,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
261
302
  if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
262
303
  this.retireBrowserController(browserController);
263
304
  }
264
- this._overridePageClose(page);
305
+ this.overridePageClose(page);
265
306
  }
266
307
  catch (err) {
267
308
  this.retireBrowserController(browserController);
268
309
  throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
269
310
  }
270
- await this._executeHooks(this.postPageCreateHooks, page, browserController);
311
+ await this.executeHooks(this.postPageCreateHooks, page, browserController);
271
312
  tryCancel();
272
- this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
313
+ this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
273
314
  return page;
274
315
  }
275
316
  /**
@@ -278,11 +319,14 @@ export class BrowserPool extends TypedEmitter {
278
319
  *
279
320
  */
280
321
  retireBrowserController(browserController) {
281
- const hasBeenRetiredOrKilled = !this.activeBrowserControllers.has(browserController);
322
+ const isStarting = this.startingBrowserControllers.has(browserController);
323
+ const isActive = this.activeBrowserControllers.has(browserController);
324
+ const hasBeenRetiredOrKilled = !isStarting && !isActive;
282
325
  if (hasBeenRetiredOrKilled)
283
326
  return;
284
327
  this.retiredBrowserControllers.add(browserController);
285
- this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
328
+ this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
329
+ this.startingBrowserControllers.delete(browserController);
286
330
  this.activeBrowserControllers.delete(browserController);
287
331
  }
288
332
  /**
@@ -294,12 +338,65 @@ export class BrowserPool extends TypedEmitter {
294
338
  if (browserController)
295
339
  this.retireBrowserController(browserController);
296
340
  }
341
+ /**
342
+ * Releases a page back to the pool. The page is closed and, if the
343
+ * optional `error` is a {@link SessionError}, the browser controller
344
+ * that served the page is retired so that its tainted state (cookies,
345
+ * storage, etc.) cannot leak into future sessions.
346
+ *
347
+ * This is the primary way the crawler should return pages to the pool.
348
+ *
349
+ * @param page The page to release.
350
+ * @param options.error The error that caused the page to be released, if any.
351
+ */
352
+ async closePage(page, options) {
353
+ if (options?.error instanceof SessionError) {
354
+ this.retireBrowserByPage(page);
355
+ }
356
+ // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
357
+ await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
358
+ }
359
+ /**
360
+ * Extracts the relevant state (currently just cookies) from a page via its
361
+ * owning {@link BrowserController}. Returns empty state when the page is
362
+ * no longer associated with a controller.
363
+ *
364
+ * As with {@link BrowserPool.injectPageState}, cookies are isolated per
365
+ * page only when the pool is configured with `useIncognitoPages: true`.
366
+ * With the default `useIncognitoPages: false`, the extracted cookies
367
+ * include those set by any sibling page sharing the same browser.
368
+ */
369
+ async extractPageState(page) {
370
+ const controller = this.getBrowserControllerByPage(page);
371
+ if (!controller) {
372
+ return { cookies: [] };
373
+ }
374
+ return { cookies: await controller.getCookies(page) };
375
+ }
376
+ /**
377
+ * Injects state into a page via its owning {@link BrowserController}.
378
+ *
379
+ * No-op when the page is no longer associated with a controller.
380
+ *
381
+ * Note that cookies are isolated per page only when the pool is configured
382
+ * with `useIncognitoPages: true` — each page then gets its own browser
383
+ * context. With the default `useIncognitoPages: false`, all pages in a
384
+ * browser share a single context, so injected cookies are visible to every
385
+ * page served by that browser.
386
+ */
387
+ async injectPageState(page, state) {
388
+ const controller = this.getBrowserControllerByPage(page);
389
+ if (!controller) {
390
+ return;
391
+ }
392
+ await controller.setCookies(page, state.cookies);
393
+ }
297
394
  /**
298
395
  * Removes all active browsers from the pool. The browsers will be
299
396
  * closed after all their pages are closed.
300
397
  */
301
398
  retireAllBrowsers() {
302
- this.activeBrowserControllers.forEach((controller) => {
399
+ [...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
303
400
  this.retireBrowserController(controller);
304
401
  });
305
402
  }
@@ -308,7 +405,7 @@ export class BrowserPool extends TypedEmitter {
308
405
  * @return {Promise<void>}
309
406
  */
310
407
  async closeAllBrowsers() {
311
- const controllers = this._getAllBrowserControllers();
408
+ const controllers = this.getAllBrowserControllers();
312
409
  const promises = [...controllers]
313
410
  .filter((controller) => controller.isActive)
314
411
  .map(async (controller) => controller.close());
@@ -319,55 +416,67 @@ export class BrowserPool extends TypedEmitter {
319
416
  */
320
417
  async destroy() {
321
418
  clearInterval(this.browserKillerInterval);
322
- clearInterval(this.browserRetireInterval);
419
+ clearInterval(this.#browserRetireInterval);
323
420
  this.browserKillerInterval = undefined;
324
- this.browserRetireInterval = undefined;
421
+ this.#browserRetireInterval = undefined;
325
422
  await this.closeAllBrowsers();
326
- this._teardown();
423
+ this.teardown();
327
424
  }
328
- _teardown() {
425
+ teardown() {
426
+ this.startingBrowserControllers.clear();
329
427
  this.activeBrowserControllers.clear();
330
428
  this.retiredBrowserControllers.clear();
331
429
  this.removeAllListeners();
332
430
  }
333
- _getAllBrowserControllers() {
334
- return new Set([...this.activeBrowserControllers, ...this.retiredBrowserControllers]);
431
+ getAllBrowserControllers() {
432
+ return new Set([
433
+ ...this.startingBrowserControllers,
434
+ ...this.activeBrowserControllers,
435
+ ...this.retiredBrowserControllers,
436
+ ]);
335
437
  }
336
- async _launchBrowser(pageId, options) {
337
- const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
438
+ async launchBrowser(pageId, options) {
439
+ const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
338
440
  const browserController = browserPlugin.createController();
339
- this.activeBrowserControllers.add(browserController);
441
+ this.startingBrowserControllers.add(browserController);
340
442
  const launchContext = browserPlugin.createLaunchContext({
341
443
  id: pageId,
342
444
  launchOptions,
343
- proxyTier,
344
445
  proxyUrl,
345
446
  });
447
+ // Disable SSL verification for MITM proxies
448
+ if (ignoreTlsErrors) {
449
+ /**
450
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
451
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
452
+ */
453
+ launchContext.launchOptions.ignoreHTTPSErrors = true;
454
+ launchContext.launchOptions.acceptInsecureCerts = true;
455
+ }
346
456
  try {
347
457
  // If the hooks or the launch fails, we need to delete the controller,
348
458
  // because otherwise it would be stuck in limbo without a browser.
349
- await this._executeHooks(this.preLaunchHooks, pageId, launchContext);
459
+ await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
350
460
  tryCancel();
351
461
  const browser = await browserPlugin.launch(launchContext);
352
462
  tryCancel();
353
463
  browserController.assignBrowser(browser, launchContext);
354
464
  }
355
465
  catch (err) {
356
- this.activeBrowserControllers.delete(browserController);
466
+ this.startingBrowserControllers.delete(browserController);
357
467
  throw err;
358
468
  }
359
- log.debug('Launched new browser.', { id: browserController.id });
360
- browserController.proxyTier = proxyTier;
469
+ this.#log.debug('Launched new browser.', { id: browserController.id });
361
470
  browserController.proxyUrl = proxyUrl;
362
471
  try {
363
472
  // If the launch fails on the post-launch hooks, we need to clean up
364
473
  // both the controller and the browser before throwing.
365
- await this._executeHooks(this.postLaunchHooks, pageId, browserController);
474
+ await this.executeHooks(this.postLaunchHooks, pageId, browserController);
366
475
  }
367
476
  catch (err) {
368
- this.activeBrowserControllers.delete(browserController);
477
+ this.startingBrowserControllers.delete(browserController);
369
478
  browserController.close().catch((closeErr) => {
370
- log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
479
+ this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
371
480
  id: browserController.id,
372
481
  });
373
482
  });
@@ -375,33 +484,33 @@ export class BrowserPool extends TypedEmitter {
375
484
  }
376
485
  tryCancel();
377
486
  browserController.activate();
378
- this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
487
+ this.startingBrowserControllers.delete(browserController);
488
+ this.activeBrowserControllers.add(browserController);
489
+ this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
379
490
  return browserController;
380
491
  }
381
492
  /**
382
493
  * Picks plugins round robin.
383
494
  * @private
384
495
  */
385
- _pickBrowserPlugin() {
496
+ pickBrowserPlugin() {
386
497
  const pluginIndex = this.pageCounter % this.browserPlugins.length;
387
498
  this.pageCounter++;
388
499
  return this.browserPlugins[pluginIndex];
389
500
  }
390
- _pickBrowserWithFreeCapacity(browserPlugin, options) {
501
+ pickBrowserWithFreeCapacity(browserPlugin, options) {
391
502
  return [...this.activeBrowserControllers].find((controller) => {
392
503
  const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
393
504
  const isCorrectPlugin = controller.browserPlugin === browserPlugin;
394
505
  const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
395
- const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
396
506
  return (isCorrectPlugin &&
397
507
  hasCapacity &&
398
- ((!controller.launchContext.browserPerProxy && !options?.proxyTier) ||
399
- (options?.proxyTier && isCorrectProxyTier) ||
508
+ (!controller.launchContext.browserPerProxy ||
400
509
  (options?.proxyUrl && isSameProxyUrl) ||
401
- (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)));
510
+ (!options?.proxyUrl && !controller.proxyUrl)));
402
511
  });
403
512
  }
404
- async _closeInactiveRetiredBrowsers() {
513
+ async closeInactiveRetiredBrowsers() {
405
514
  const closedBrowserIds = [];
406
515
  for (const controller of this.retiredBrowserControllers) {
407
516
  const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
@@ -409,61 +518,81 @@ export class BrowserPool extends TypedEmitter {
409
518
  const isBrowserEmpty = controller.activePages === 0;
410
519
  if (isBrowserIdle || isBrowserEmpty) {
411
520
  const { id } = controller;
412
- log.debug('Closing retired browser.', { id });
521
+ this.#log.debug('Closing retired browser.', { id });
413
522
  await controller.close();
414
523
  this.retiredBrowserControllers.delete(controller);
415
524
  closedBrowserIds.push(id);
416
525
  }
417
526
  }
418
527
  if (closedBrowserIds.length) {
419
- log.debug('Closed retired browsers.', {
528
+ this.#log.debug('Closed retired browsers.', {
420
529
  count: closedBrowserIds.length,
421
530
  closedBrowserIds,
422
531
  });
423
532
  }
424
533
  }
425
- _overridePageClose(page) {
534
+ overridePageClose(page) {
426
535
  const originalPageClose = page.close;
427
536
  const browserController = this.pageToBrowserController.get(page);
428
537
  const pageId = this.getPageId(page);
429
538
  page.close = async (...args) => {
430
- await this._executeHooks(this.prePageCloseHooks, page, browserController);
539
+ await this.executeHooks(this.prePageCloseHooks, page, browserController);
431
540
  await originalPageClose.apply(page, args).catch((err) => {
432
- log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
541
+ this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
433
542
  });
434
- await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
543
+ await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
435
544
  this.pages.delete(pageId);
436
- this._closeRetiredBrowserWithNoPages(browserController);
437
- this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
545
+ this.closeRetiredBrowserWithNoPages(browserController);
546
+ this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
438
547
  };
439
548
  }
440
- async _executeHooks(hooks, ...args) {
549
+ async executeHooks(hooks, ...args) {
441
550
  for (const hook of hooks) {
442
551
  await hook(...args);
443
552
  }
444
553
  }
445
- _closeRetiredBrowserWithNoPages(browserController) {
554
+ closeRetiredBrowserWithNoPages(browserController) {
446
555
  if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
447
556
  // Run this with a delay, otherwise page.close()
448
557
  // might fail with "Protocol error (Target.closeTarget): Target closed."
449
558
  setTimeout(() => {
450
- log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
559
+ this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
451
560
  void browserController.close().finally(() => {
452
561
  this.retiredBrowserControllers.delete(browserController);
453
562
  });
454
563
  }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
455
564
  }
456
565
  }
457
- _initializeFingerprinting() {
566
+ /**
567
+ * Returns `true` if the pool can accept a new browser launch without exceeding
568
+ * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
569
+ */
570
+ hasFreeBrowserSlot() {
571
+ const total = this.startingBrowserControllers.size +
572
+ this.activeBrowserControllers.size +
573
+ this.retiredBrowserControllers.size;
574
+ return total < this.maxOpenBrowsers;
575
+ }
576
+ /**
577
+ * Returns `true` if any active browser has room for another page.
578
+ */
579
+ hasActiveBrowserWithFreeCapacity() {
580
+ for (const controller of this.activeBrowserControllers) {
581
+ if (controller.activePages < this.maxOpenPagesPerBrowser)
582
+ return true;
583
+ }
584
+ return false;
585
+ }
586
+ initializeFingerprinting() {
458
587
  const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
459
588
  this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
460
589
  this.fingerprintInjector = new FingerprintInjector();
461
590
  if (useFingerprintCache) {
462
591
  this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
463
592
  }
464
- this._addFingerprintHooks();
593
+ this.addFingerprintHooks();
465
594
  }
466
- _addFingerprintHooks() {
595
+ addFingerprintHooks() {
467
596
  this.preLaunchHooks = [
468
597
  ...this.preLaunchHooks,
469
598
  // This is flipped because of the fingerprint cache.
@@ -474,4 +603,3 @@ export class BrowserPool extends TypedEmitter {
474
603
  this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
475
604
  }
476
605
  }
477
- //# sourceMappingURL=browser-pool.js.map
@@ -9,4 +9,3 @@ export declare function createProxyServerForContainers(fallbackProxyUrl?: string
9
9
  ipToProxy: Map<string, string>;
10
10
  close(closeConnections: boolean): Promise<void>;
11
11
  }>;
12
- //# sourceMappingURL=container-proxy-server.d.ts.map
@@ -40,4 +40,3 @@ export async function createProxyServerForContainers(fallbackProxyUrl) {
40
40
  },
41
41
  };
42
42
  }
43
- //# sourceMappingURL=container-proxy-server.js.map
package/events.d.ts CHANGED
@@ -1,11 +1,10 @@
1
- export declare const enum BROWSER_POOL_EVENTS {
1
+ export declare enum BROWSER_POOL_EVENTS {
2
2
  BROWSER_LAUNCHED = "browserLaunched",
3
3
  BROWSER_RETIRED = "browserRetired",
4
4
  BROWSER_CLOSED = "browserClosed",
5
5
  PAGE_CREATED = "pageCreated",
6
6
  PAGE_CLOSED = "pageClosed"
7
7
  }
8
- export declare const enum BROWSER_CONTROLLER_EVENTS {
8
+ export declare enum BROWSER_CONTROLLER_EVENTS {
9
9
  BROWSER_CLOSED = "browserClosed"
10
10
  }
11
- //# sourceMappingURL=events.d.ts.map
package/events.js CHANGED
@@ -10,4 +10,3 @@ export var BROWSER_CONTROLLER_EVENTS;
10
10
  (function (BROWSER_CONTROLLER_EVENTS) {
11
11
  BROWSER_CONTROLLER_EVENTS["BROWSER_CLOSED"] = "browserClosed";
12
12
  })(BROWSER_CONTROLLER_EVENTS || (BROWSER_CONTROLLER_EVENTS = {}));
13
- //# sourceMappingURL=events.js.map
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
14
14
  * @internal
15
15
  */
16
16
  export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
17
- //# sourceMappingURL=hooks.d.ts.map