@crawlee/browser-pool 4.0.0-beta.11 → 4.0.0-beta.110

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +17 -13
  2. package/abstract-classes/browser-controller.d.ts +70 -13
  3. package/abstract-classes/browser-controller.js +18 -20
  4. package/abstract-classes/browser-plugin.d.ts +48 -15
  5. package/abstract-classes/browser-plugin.js +69 -11
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +100 -28
  9. package/browser-pool.js +188 -61
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +0 -1
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +26 -10
  16. package/fingerprinting/types.d.ts +0 -1
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +0 -1
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +20 -5
  23. package/launch-context.js +21 -10
  24. package/package.json +8 -9
  25. package/playwright/playwright-browser.d.ts +2 -6
  26. package/playwright/playwright-browser.js +16 -17
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +20 -10
  29. package/playwright/playwright-plugin.d.ts +10 -7
  30. package/playwright/playwright-plugin.js +40 -16
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +5 -3
  36. package/puppeteer/puppeteer-plugin.js +83 -46
  37. package/remote-browser-pool.d.ts +164 -0
  38. package/remote-browser-pool.js +191 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/tsconfig.build.tsbuildinfo +0 -1
  82. package/utils.d.ts.map +0 -1
  83. package/utils.js.map +0 -1
package/browser-pool.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { AsyncResource } from 'node:async_hooks';
2
+ import { SessionError, serviceLocator } from '@crawlee/core';
1
3
  import { FingerprintGenerator } from 'fingerprint-generator';
2
4
  import { FingerprintInjector } from 'fingerprint-injector';
3
5
  import { nanoid } from 'nanoid';
@@ -7,7 +9,7 @@ import QuickLRU from 'quick-lru';
7
9
  import { TypedEmitter } from 'tiny-typed-emitter';
8
10
  import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
9
11
  import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
10
- import { log } from './logger.js';
12
+ const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
11
13
  const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
12
14
  const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
13
15
  /**
@@ -60,6 +62,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
60
62
  export class BrowserPool extends TypedEmitter {
61
63
  browserPlugins;
62
64
  maxOpenPagesPerBrowser;
65
+ maxOpenBrowsers;
63
66
  retireBrowserAfterPageCount;
64
67
  operationTimeoutMillis;
65
68
  closeInactiveBrowserAfterMillis;
@@ -74,17 +77,21 @@ export class BrowserPool extends TypedEmitter {
74
77
  pageCounter = 0;
75
78
  pages = new Map();
76
79
  pageIds = new WeakMap();
80
+ startingBrowserControllers = new Set();
77
81
  activeBrowserControllers = new Set();
78
82
  retiredBrowserControllers = new Set();
79
83
  pageToBrowserController = new WeakMap();
80
84
  fingerprintInjector;
81
85
  fingerprintGenerator;
82
86
  fingerprintCache;
83
- browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
84
- browserRetireInterval;
85
- limiter = pLimit(1);
87
+ // kept as TS-private: tests replace this interval through bracket access
88
+ browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
89
+ #browserRetireInterval;
90
+ #limiter = pLimit(1);
91
+ #log;
86
92
  constructor(options) {
87
93
  super();
94
+ this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
88
95
  this.browserKillerInterval.unref();
89
96
  ow(options, ow.object.exactShape({
90
97
  browserPlugins: ow.array.minLength(1),
@@ -114,18 +121,19 @@ export class BrowserPool extends TypedEmitter {
114
121
  }
115
122
  this.browserPlugins = browserPlugins;
116
123
  this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
124
+ this.maxOpenBrowsers = Infinity;
117
125
  this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
118
126
  this.operationTimeoutMillis = operationTimeoutSecs * 1000;
119
127
  this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
120
128
  this.useFingerprints = useFingerprints;
121
129
  this.fingerprintOptions = fingerprintOptions;
122
- this.browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
130
+ this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
123
131
  if (controller.activePages === 0 &&
124
132
  controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
125
133
  this.retireBrowserController(controller);
126
134
  }
127
135
  }), retireInactiveBrowserAfterSecs * 1000);
128
- this.browserRetireInterval.unref();
136
+ this.#browserRetireInterval.unref();
129
137
  // hooks
130
138
  this.preLaunchHooks = preLaunchHooks;
131
139
  this.postLaunchHooks = postLaunchHooks;
@@ -135,30 +143,56 @@ export class BrowserPool extends TypedEmitter {
135
143
  this.postPageCloseHooks = postPageCloseHooks;
136
144
  // fingerprinting
137
145
  if (this.useFingerprints) {
138
- this._initializeFingerprinting();
146
+ this.initializeFingerprinting();
139
147
  }
140
148
  }
141
149
  /**
142
150
  * Opens a new page in one of the running browsers or launches
143
151
  * a new browser and opens a page there, if no browsers are active,
144
152
  * or their page limits have been exceeded.
153
+ *
154
+ * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
155
+ * provided, this implementation uses it as a cache key for browser fingerprints (when
156
+ * fingerprinting is enabled) and reads
157
+ * {@link ProxyInfo.url|session.proxyInfo.url} /
158
+ * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
159
+ * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
160
+ * `ignoreTlsErrors` values in the options take precedence.
161
+ *
162
+ * Beyond fingerprint caching and proxy configuration, no other session
163
+ * properties are consumed — cookie and header injection remain the
164
+ * crawler's responsibility.
145
165
  */
146
166
  async newPage(options = {}) {
147
- const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
167
+ const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
148
168
  if (this.pages.has(id)) {
149
169
  throw new Error(`Page with ID: ${id} already exists.`);
150
170
  }
151
171
  if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
152
172
  throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
153
173
  }
174
+ // Bind the limiter callback to the current async-hooks context. p-limit
175
+ // otherwise resumes queued callbacks in the previous task's
176
+ // AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
177
+ // requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
178
+ // fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
179
+ // ESM-only rewrite, so we can't bump it in Crawlee v3.
180
+ // Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
181
+ // ALS-scoped: without it, a queued callback would resume in the previous request's async
182
+ // context and run request B's storage writes inside request A's transaction.
183
+ // TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
154
184
  // Limiter is necessary - https://github.com/apify/crawlee/issues/1126
155
- return this.limiter(async () => {
156
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
185
+ return this.#limiter(AsyncResource.bind(async () => {
186
+ let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
157
187
  if (!browserController)
158
- browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
188
+ browserController = await this.launchBrowser(id, {
189
+ browserPlugin,
190
+ proxyUrl,
191
+ ignoreTlsErrors,
192
+ });
159
193
  tryCancel();
160
- return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
161
- });
194
+ return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
195
+ }));
162
196
  }
163
197
  /**
164
198
  * Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
@@ -166,13 +200,13 @@ export class BrowserPool extends TypedEmitter {
166
200
  * configure the new browser.
167
201
  */
168
202
  async newPageInNewBrowser(options = {}) {
169
- const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;
203
+ const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
170
204
  if (this.pages.has(id)) {
171
205
  throw new Error(`Page with ID: ${id} already exists.`);
172
206
  }
173
- const browserController = await this._launchBrowser(id, { launchOptions, browserPlugin });
207
+ const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
174
208
  tryCancel();
175
- return await this._createPageForBrowser(id, browserController, pageOptions);
209
+ return await this.createPageForBrowser(id, browserController, pageOptions);
176
210
  }
177
211
  /**
178
212
  * Opens new pages with all available plugins and returns an array
@@ -238,7 +272,7 @@ export class BrowserPool extends TypedEmitter {
238
272
  getPageId(page) {
239
273
  return this.pageIds.get(page);
240
274
  }
241
- async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
275
+ async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
242
276
  // This is needed for concurrent newPage calls to wait for the browser launch.
243
277
  // It's not ideal though, we need to come up with a better API.
244
278
  // eslint-disable-next-line dot-notation -- accessing private property
@@ -247,8 +281,14 @@ export class BrowserPool extends TypedEmitter {
247
281
  const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
248
282
  if (finalPageOptions) {
249
283
  Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
284
+ if (ignoreTlsErrors) {
285
+ Object.assign(finalPageOptions, {
286
+ ignoreHTTPSErrors: true,
287
+ acceptInsecureCerts: true,
288
+ });
289
+ }
250
290
  }
251
- await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
291
+ await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
252
292
  tryCancel();
253
293
  let page;
254
294
  try {
@@ -261,13 +301,13 @@ export class BrowserPool extends TypedEmitter {
261
301
  if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
262
302
  this.retireBrowserController(browserController);
263
303
  }
264
- this._overridePageClose(page);
304
+ this.overridePageClose(page);
265
305
  }
266
306
  catch (err) {
267
307
  this.retireBrowserController(browserController);
268
308
  throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
269
309
  }
270
- await this._executeHooks(this.postPageCreateHooks, page, browserController);
310
+ await this.executeHooks(this.postPageCreateHooks, page, browserController);
271
311
  tryCancel();
272
312
  this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
273
313
  return page;
@@ -278,11 +318,14 @@ export class BrowserPool extends TypedEmitter {
278
318
  *
279
319
  */
280
320
  retireBrowserController(browserController) {
281
- const hasBeenRetiredOrKilled = !this.activeBrowserControllers.has(browserController);
321
+ const isStarting = this.startingBrowserControllers.has(browserController);
322
+ const isActive = this.activeBrowserControllers.has(browserController);
323
+ const hasBeenRetiredOrKilled = !isStarting && !isActive;
282
324
  if (hasBeenRetiredOrKilled)
283
325
  return;
284
326
  this.retiredBrowserControllers.add(browserController);
285
327
  this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
328
+ this.startingBrowserControllers.delete(browserController);
286
329
  this.activeBrowserControllers.delete(browserController);
287
330
  }
288
331
  /**
@@ -294,12 +337,65 @@ export class BrowserPool extends TypedEmitter {
294
337
  if (browserController)
295
338
  this.retireBrowserController(browserController);
296
339
  }
340
+ /**
341
+ * Releases a page back to the pool. The page is closed and, if the
342
+ * optional `error` is a {@link SessionError}, the browser controller
343
+ * that served the page is retired so that its tainted state (cookies,
344
+ * storage, etc.) cannot leak into future sessions.
345
+ *
346
+ * This is the primary way the crawler should return pages to the pool.
347
+ *
348
+ * @param page The page to release.
349
+ * @param options.error The error that caused the page to be released, if any.
350
+ */
351
+ async closePage(page, options) {
352
+ if (options?.error instanceof SessionError) {
353
+ this.retireBrowserByPage(page);
354
+ }
355
+ // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
356
+ await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
357
+ }
358
+ /**
359
+ * Extracts the relevant state (currently just cookies) from a page via its
360
+ * owning {@link BrowserController}. Returns empty state when the page is
361
+ * no longer associated with a controller.
362
+ *
363
+ * As with {@link BrowserPool.injectPageState}, cookies are isolated per
364
+ * page only when the pool is configured with `useIncognitoPages: true`.
365
+ * With the default `useIncognitoPages: false`, the extracted cookies
366
+ * include those set by any sibling page sharing the same browser.
367
+ */
368
+ async extractPageState(page) {
369
+ const controller = this.getBrowserControllerByPage(page);
370
+ if (!controller) {
371
+ return { cookies: [] };
372
+ }
373
+ return { cookies: await controller.getCookies(page) };
374
+ }
375
+ /**
376
+ * Injects state into a page via its owning {@link BrowserController}.
377
+ *
378
+ * No-op when the page is no longer associated with a controller.
379
+ *
380
+ * Note that cookies are isolated per page only when the pool is configured
381
+ * with `useIncognitoPages: true` — each page then gets its own browser
382
+ * context. With the default `useIncognitoPages: false`, all pages in a
383
+ * browser share a single context, so injected cookies are visible to every
384
+ * page served by that browser.
385
+ */
386
+ async injectPageState(page, state) {
387
+ const controller = this.getBrowserControllerByPage(page);
388
+ if (!controller) {
389
+ return;
390
+ }
391
+ await controller.setCookies(page, state.cookies);
392
+ }
297
393
  /**
298
394
  * Removes all active browsers from the pool. The browsers will be
299
395
  * closed after all their pages are closed.
300
396
  */
301
397
  retireAllBrowsers() {
302
- this.activeBrowserControllers.forEach((controller) => {
398
+ [...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
303
399
  this.retireBrowserController(controller);
304
400
  });
305
401
  }
@@ -308,7 +404,7 @@ export class BrowserPool extends TypedEmitter {
308
404
  * @return {Promise<void>}
309
405
  */
310
406
  async closeAllBrowsers() {
311
- const controllers = this._getAllBrowserControllers();
407
+ const controllers = this.getAllBrowserControllers();
312
408
  const promises = [...controllers]
313
409
  .filter((controller) => controller.isActive)
314
410
  .map(async (controller) => controller.close());
@@ -319,55 +415,67 @@ export class BrowserPool extends TypedEmitter {
319
415
  */
320
416
  async destroy() {
321
417
  clearInterval(this.browserKillerInterval);
322
- clearInterval(this.browserRetireInterval);
418
+ clearInterval(this.#browserRetireInterval);
323
419
  this.browserKillerInterval = undefined;
324
- this.browserRetireInterval = undefined;
420
+ this.#browserRetireInterval = undefined;
325
421
  await this.closeAllBrowsers();
326
- this._teardown();
422
+ this.teardown();
327
423
  }
328
- _teardown() {
424
+ teardown() {
425
+ this.startingBrowserControllers.clear();
329
426
  this.activeBrowserControllers.clear();
330
427
  this.retiredBrowserControllers.clear();
331
428
  this.removeAllListeners();
332
429
  }
333
- _getAllBrowserControllers() {
334
- return new Set([...this.activeBrowserControllers, ...this.retiredBrowserControllers]);
430
+ getAllBrowserControllers() {
431
+ return new Set([
432
+ ...this.startingBrowserControllers,
433
+ ...this.activeBrowserControllers,
434
+ ...this.retiredBrowserControllers,
435
+ ]);
335
436
  }
336
- async _launchBrowser(pageId, options) {
337
- const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
437
+ async launchBrowser(pageId, options) {
438
+ const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
338
439
  const browserController = browserPlugin.createController();
339
- this.activeBrowserControllers.add(browserController);
440
+ this.startingBrowserControllers.add(browserController);
340
441
  const launchContext = browserPlugin.createLaunchContext({
341
442
  id: pageId,
342
443
  launchOptions,
343
- proxyTier,
344
444
  proxyUrl,
345
445
  });
446
+ // Disable SSL verification for MITM proxies
447
+ if (ignoreTlsErrors) {
448
+ /**
449
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
450
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
451
+ */
452
+ launchContext.launchOptions.ignoreHTTPSErrors = true;
453
+ launchContext.launchOptions.acceptInsecureCerts = true;
454
+ }
346
455
  try {
347
456
  // If the hooks or the launch fails, we need to delete the controller,
348
457
  // because otherwise it would be stuck in limbo without a browser.
349
- await this._executeHooks(this.preLaunchHooks, pageId, launchContext);
458
+ await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
350
459
  tryCancel();
351
460
  const browser = await browserPlugin.launch(launchContext);
352
461
  tryCancel();
353
462
  browserController.assignBrowser(browser, launchContext);
354
463
  }
355
464
  catch (err) {
356
- this.activeBrowserControllers.delete(browserController);
465
+ this.startingBrowserControllers.delete(browserController);
357
466
  throw err;
358
467
  }
359
- log.debug('Launched new browser.', { id: browserController.id });
360
- browserController.proxyTier = proxyTier;
468
+ this.#log.debug('Launched new browser.', { id: browserController.id });
361
469
  browserController.proxyUrl = proxyUrl;
362
470
  try {
363
471
  // If the launch fails on the post-launch hooks, we need to clean up
364
472
  // both the controller and the browser before throwing.
365
- await this._executeHooks(this.postLaunchHooks, pageId, browserController);
473
+ await this.executeHooks(this.postLaunchHooks, pageId, browserController);
366
474
  }
367
475
  catch (err) {
368
- this.activeBrowserControllers.delete(browserController);
476
+ this.startingBrowserControllers.delete(browserController);
369
477
  browserController.close().catch((closeErr) => {
370
- log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
478
+ this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
371
479
  id: browserController.id,
372
480
  });
373
481
  });
@@ -375,6 +483,8 @@ export class BrowserPool extends TypedEmitter {
375
483
  }
376
484
  tryCancel();
377
485
  browserController.activate();
486
+ this.startingBrowserControllers.delete(browserController);
487
+ this.activeBrowserControllers.add(browserController);
378
488
  this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
379
489
  return browserController;
380
490
  }
@@ -382,26 +492,24 @@ export class BrowserPool extends TypedEmitter {
382
492
  * Picks plugins round robin.
383
493
  * @private
384
494
  */
385
- _pickBrowserPlugin() {
495
+ pickBrowserPlugin() {
386
496
  const pluginIndex = this.pageCounter % this.browserPlugins.length;
387
497
  this.pageCounter++;
388
498
  return this.browserPlugins[pluginIndex];
389
499
  }
390
- _pickBrowserWithFreeCapacity(browserPlugin, options) {
500
+ pickBrowserWithFreeCapacity(browserPlugin, options) {
391
501
  return [...this.activeBrowserControllers].find((controller) => {
392
502
  const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
393
503
  const isCorrectPlugin = controller.browserPlugin === browserPlugin;
394
504
  const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
395
- const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
396
505
  return (isCorrectPlugin &&
397
506
  hasCapacity &&
398
- ((!controller.launchContext.browserPerProxy && !options?.proxyTier) ||
399
- (options?.proxyTier && isCorrectProxyTier) ||
507
+ (!controller.launchContext.browserPerProxy ||
400
508
  (options?.proxyUrl && isSameProxyUrl) ||
401
- (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)));
509
+ (!options?.proxyUrl && !controller.proxyUrl)));
402
510
  });
403
511
  }
404
- async _closeInactiveRetiredBrowsers() {
512
+ async closeInactiveRetiredBrowsers() {
405
513
  const closedBrowserIds = [];
406
514
  for (const controller of this.retiredBrowserControllers) {
407
515
  const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
@@ -409,61 +517,81 @@ export class BrowserPool extends TypedEmitter {
409
517
  const isBrowserEmpty = controller.activePages === 0;
410
518
  if (isBrowserIdle || isBrowserEmpty) {
411
519
  const { id } = controller;
412
- log.debug('Closing retired browser.', { id });
520
+ this.#log.debug('Closing retired browser.', { id });
413
521
  await controller.close();
414
522
  this.retiredBrowserControllers.delete(controller);
415
523
  closedBrowserIds.push(id);
416
524
  }
417
525
  }
418
526
  if (closedBrowserIds.length) {
419
- log.debug('Closed retired browsers.', {
527
+ this.#log.debug('Closed retired browsers.', {
420
528
  count: closedBrowserIds.length,
421
529
  closedBrowserIds,
422
530
  });
423
531
  }
424
532
  }
425
- _overridePageClose(page) {
533
+ overridePageClose(page) {
426
534
  const originalPageClose = page.close;
427
535
  const browserController = this.pageToBrowserController.get(page);
428
536
  const pageId = this.getPageId(page);
429
537
  page.close = async (...args) => {
430
- await this._executeHooks(this.prePageCloseHooks, page, browserController);
538
+ await this.executeHooks(this.prePageCloseHooks, page, browserController);
431
539
  await originalPageClose.apply(page, args).catch((err) => {
432
- log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
540
+ this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
433
541
  });
434
- await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
542
+ await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
435
543
  this.pages.delete(pageId);
436
- this._closeRetiredBrowserWithNoPages(browserController);
544
+ this.closeRetiredBrowserWithNoPages(browserController);
437
545
  this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
438
546
  };
439
547
  }
440
- async _executeHooks(hooks, ...args) {
548
+ async executeHooks(hooks, ...args) {
441
549
  for (const hook of hooks) {
442
550
  await hook(...args);
443
551
  }
444
552
  }
445
- _closeRetiredBrowserWithNoPages(browserController) {
553
+ closeRetiredBrowserWithNoPages(browserController) {
446
554
  if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
447
555
  // Run this with a delay, otherwise page.close()
448
556
  // might fail with "Protocol error (Target.closeTarget): Target closed."
449
557
  setTimeout(() => {
450
- log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
558
+ this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
451
559
  void browserController.close().finally(() => {
452
560
  this.retiredBrowserControllers.delete(browserController);
453
561
  });
454
562
  }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
455
563
  }
456
564
  }
457
- _initializeFingerprinting() {
565
+ /**
566
+ * Returns `true` if the pool can accept a new browser launch without exceeding
567
+ * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
568
+ */
569
+ hasFreeBrowserSlot() {
570
+ const total = this.startingBrowserControllers.size +
571
+ this.activeBrowserControllers.size +
572
+ this.retiredBrowserControllers.size;
573
+ return total < this.maxOpenBrowsers;
574
+ }
575
+ /**
576
+ * Returns `true` if any active browser has room for another page.
577
+ */
578
+ hasActiveBrowserWithFreeCapacity() {
579
+ for (const controller of this.activeBrowserControllers) {
580
+ if (controller.activePages < this.maxOpenPagesPerBrowser)
581
+ return true;
582
+ }
583
+ return false;
584
+ }
585
+ initializeFingerprinting() {
458
586
  const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
459
587
  this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
460
588
  this.fingerprintInjector = new FingerprintInjector();
461
589
  if (useFingerprintCache) {
462
590
  this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
463
591
  }
464
- this._addFingerprintHooks();
592
+ this.addFingerprintHooks();
465
593
  }
466
- _addFingerprintHooks() {
594
+ addFingerprintHooks() {
467
595
  this.preLaunchHooks = [
468
596
  ...this.preLaunchHooks,
469
597
  // This is flipped because of the fingerprint cache.
@@ -474,4 +602,3 @@ export class BrowserPool extends TypedEmitter {
474
602
  this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
475
603
  }
476
604
  }
477
- //# sourceMappingURL=browser-pool.js.map
@@ -9,4 +9,3 @@ export declare function createProxyServerForContainers(fallbackProxyUrl?: string
9
9
  ipToProxy: Map<string, string>;
10
10
  close(closeConnections: boolean): Promise<void>;
11
11
  }>;
12
- //# sourceMappingURL=container-proxy-server.d.ts.map
@@ -40,4 +40,3 @@ export async function createProxyServerForContainers(fallbackProxyUrl) {
40
40
  },
41
41
  };
42
42
  }
43
- //# sourceMappingURL=container-proxy-server.js.map
package/events.d.ts CHANGED
@@ -8,4 +8,3 @@ export declare const enum BROWSER_POOL_EVENTS {
8
8
  export declare const enum BROWSER_CONTROLLER_EVENTS {
9
9
  BROWSER_CLOSED = "browserClosed"
10
10
  }
11
- //# sourceMappingURL=events.d.ts.map
package/events.js CHANGED
@@ -10,4 +10,3 @@ export var BROWSER_CONTROLLER_EVENTS;
10
10
  (function (BROWSER_CONTROLLER_EVENTS) {
11
11
  BROWSER_CONTROLLER_EVENTS["BROWSER_CLOSED"] = "browserClosed";
12
12
  })(BROWSER_CONTROLLER_EVENTS || (BROWSER_CONTROLLER_EVENTS = {}));
13
- //# sourceMappingURL=events.js.map
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
14
14
  * @internal
15
15
  */
16
16
  export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
17
- //# sourceMappingURL=hooks.d.ts.map
@@ -1,29 +1,42 @@
1
1
  import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
2
2
  import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
3
3
  import { getGeneratorDefaultOptions } from './utils.js';
4
+ function applySessionHints(base, fingerprint) {
5
+ if (!fingerprint)
6
+ return base;
7
+ return {
8
+ ...base,
9
+ ...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
10
+ ...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
11
+ ...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
12
+ };
13
+ }
4
14
  /**
5
15
  * @internal
6
16
  */
7
17
  export function createFingerprintPreLaunchHook(browserPool) {
8
18
  const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
9
19
  return (_pageId, launchContext) => {
20
+ // Remote browsers may have their own fingerprinting — skip local fingerprint injection
21
+ if (launchContext.isRemote)
22
+ return;
10
23
  const { useIncognitoPages } = launchContext;
11
- const cacheKey = launchContext.session?.id ?? launchContext.proxyUrl;
24
+ const session = launchContext.session;
25
+ const cacheKey = session?.id ?? launchContext.proxyUrl;
12
26
  const { launchOptions } = launchContext;
13
- // If no options are passed we try to pass best default options as possible to match browser and OS.
14
- const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
15
27
  let fingerprint;
16
28
  if (cacheKey && fingerprintCache?.has(cacheKey)) {
17
29
  fingerprint = fingerprintCache.get(cacheKey);
18
30
  }
19
- else if (cacheKey) {
20
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
21
- fingerprintCache?.set(cacheKey, fingerprint);
22
- }
23
31
  else {
24
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
32
+ const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
33
+ const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
34
+ fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
35
+ if (cacheKey)
36
+ fingerprintCache?.set(cacheKey, fingerprint);
25
37
  }
26
- launchContext.extend({ fingerprint });
38
+ // `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
39
+ launchContext.fingerprint = fingerprint;
27
40
  if (useIncognitoPages) {
28
41
  return;
29
42
  }
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
41
54
  export function createPrePageCreateHook() {
42
55
  return (_pageId, browserController, pageOptions) => {
43
56
  const { launchContext, browserPlugin } = browserController;
57
+ if (launchContext.isRemote)
58
+ return;
44
59
  const { fingerprint } = launchContext.fingerprint;
45
60
  if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
46
61
  pageOptions.userAgent ??= fingerprint.navigator.userAgent;
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
57
72
  export function createPostPageCreateHook(fingerprintInjector) {
58
73
  return async (page, browserController) => {
59
74
  const { browserPlugin, launchContext } = browserController;
75
+ if (launchContext.isRemote)
76
+ return;
60
77
  const fingerprint = launchContext.fingerprint;
61
78
  // TODO this will require refactoring, we should use common API instead of branching based on plugin type,
62
79
  // and there should be no public methods specific to some browser.
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
79
96
  }
80
97
  };
81
98
  }
82
- //# sourceMappingURL=hooks.js.map
@@ -61,4 +61,3 @@ export declare const enum DeviceCategory {
61
61
  desktop = "desktop"
62
62
  }
63
63
  export {};
64
- //# sourceMappingURL=types.d.ts.map
@@ -32,4 +32,3 @@ export var DeviceCategory;
32
32
  */
33
33
  DeviceCategory["desktop"] = "desktop";
34
34
  })(DeviceCategory || (DeviceCategory = {}));
35
- //# sourceMappingURL=types.js.map
@@ -1,4 +1,3 @@
1
1
  import type { LaunchContext } from '../launch-context.js';
2
2
  import type { FingerprintGeneratorOptions } from './types.js';
3
3
  export declare const getGeneratorDefaultOptions: (launchContext: LaunchContext) => FingerprintGeneratorOptions;
4
- //# sourceMappingURL=utils.d.ts.map
@@ -42,4 +42,3 @@ const getOperatingSystem = () => {
42
42
  return "linux" /* OperatingSystemsName.linux */;
43
43
  }
44
44
  };
45
- //# sourceMappingURL=utils.js.map