@crawlee/browser-pool 4.0.0-beta.1 → 4.0.0-beta.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +17 -13
  2. package/abstract-classes/browser-controller.d.ts +69 -10
  3. package/abstract-classes/browser-controller.js +8 -11
  4. package/abstract-classes/browser-plugin.d.ts +44 -11
  5. package/abstract-classes/browser-plugin.js +65 -7
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +88 -14
  9. package/browser-pool.js +151 -28
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +0 -1
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +24 -9
  16. package/fingerprinting/types.d.ts +0 -1
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +0 -1
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +19 -3
  23. package/launch-context.js +12 -4
  24. package/package.json +8 -9
  25. package/playwright/playwright-browser.d.ts +0 -1
  26. package/playwright/playwright-browser.js +0 -1
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +20 -10
  29. package/playwright/playwright-plugin.d.ts +6 -3
  30. package/playwright/playwright-plugin.js +27 -5
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +3 -1
  36. package/puppeteer/puppeteer-plugin.js +81 -44
  37. package/remote-browser-pool.d.ts +170 -0
  38. package/remote-browser-pool.js +191 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/tsconfig.build.tsbuildinfo +0 -1
  82. package/utils.d.ts.map +0 -1
  83. package/utils.js.map +0 -1
package/browser-pool.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
1
2
  import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
2
3
  import { FingerprintGenerator } from 'fingerprint-generator';
3
4
  import { FingerprintInjector } from 'fingerprint-injector';
@@ -246,9 +247,10 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
246
247
  * });
247
248
  * ```
248
249
  */
249
- export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
250
+ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
250
251
  browserPlugins: BrowserPlugins;
251
252
  maxOpenPagesPerBrowser: number;
253
+ maxOpenBrowsers: number;
252
254
  retireBrowserAfterPageCount: number;
253
255
  operationTimeoutMillis: number;
254
256
  closeInactiveBrowserAfterMillis: number;
@@ -263,6 +265,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
263
265
  pageCounter: number;
264
266
  pages: Map<string, PageReturn>;
265
267
  pageIds: WeakMap<PageReturn, string>;
268
+ startingBrowserControllers: Set<BrowserControllerReturn>;
266
269
  activeBrowserControllers: Set<BrowserControllerReturn>;
267
270
  retiredBrowserControllers: Set<BrowserControllerReturn>;
268
271
  pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
@@ -272,11 +275,24 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
272
275
  private browserKillerInterval?;
273
276
  private browserRetireInterval?;
274
277
  private limiter;
278
+ private log;
275
279
  constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
276
280
  /**
277
281
  * Opens a new page in one of the running browsers or launches
278
282
  * a new browser and opens a page there, if no browsers are active,
279
283
  * or their page limits have been exceeded.
284
+ *
285
+ * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
286
+ * provided, this implementation uses it as a cache key for browser fingerprints (when
287
+ * fingerprinting is enabled) and reads
288
+ * {@link ProxyInfo.url|session.proxyInfo.url} /
289
+ * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
290
+ * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
291
+ * `ignoreTlsErrors` values in the options take precedence.
292
+ *
293
+ * Beyond fingerprint caching and proxy configuration, no other session
294
+ * properties are consumed — cookie and header injection remain the
295
+ * crawler's responsibility.
280
296
  */
281
297
  newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
282
298
  /**
@@ -346,6 +362,43 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
346
362
  * closed after all its pages are closed.
347
363
  */
348
364
  retireBrowserByPage(page: PageReturn): void;
365
+ /**
366
+ * Releases a page back to the pool. The page is closed and, if the
367
+ * optional `error` is a {@link SessionError}, the browser controller
368
+ * that served the page is retired so that its tainted state (cookies,
369
+ * storage, etc.) cannot leak into future sessions.
370
+ *
371
+ * This is the primary way the crawler should return pages to the pool.
372
+ *
373
+ * @param page The page to release.
374
+ * @param options.error The error that caused the page to be released, if any.
375
+ */
376
+ closePage(page: PageReturn, options?: {
377
+ error?: Error;
378
+ }): Promise<void>;
379
+ /**
380
+ * Extracts the relevant state (currently just cookies) from a page via its
381
+ * owning {@link BrowserController}. Returns empty state when the page is
382
+ * no longer associated with a controller.
383
+ *
384
+ * As with {@link BrowserPool.injectPageState}, cookies are isolated per
385
+ * page only when the pool is configured with `useIncognitoPages: true`.
386
+ * With the default `useIncognitoPages: false`, the extracted cookies
387
+ * include those set by any sibling page sharing the same browser.
388
+ */
389
+ extractPageState(page: PageReturn): Promise<PageState>;
390
+ /**
391
+ * Injects state into a page via its owning {@link BrowserController}.
392
+ *
393
+ * No-op when the page is no longer associated with a controller.
394
+ *
395
+ * Note that cookies are isolated per page only when the pool is configured
396
+ * with `useIncognitoPages: true` — each page then gets its own browser
397
+ * context. With the default `useIncognitoPages: false`, all pages in a
398
+ * browser share a single context, so injected cookies are visible to every
399
+ * page served by that browser.
400
+ */
401
+ injectPageState(page: PageReturn, state: PageState): Promise<void>;
349
402
  /**
350
403
  * Removes all active browsers from the pool. The browsers will be
351
404
  * closed after all their pages are closed.
@@ -373,15 +426,45 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
373
426
  private _overridePageClose;
374
427
  private _executeHooks;
375
428
  private _closeRetiredBrowserWithNoPages;
429
+ /**
430
+ * Returns `true` if the pool can accept a new browser launch without exceeding
431
+ * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
432
+ */
433
+ hasFreeBrowserSlot(): boolean;
434
+ /**
435
+ * Returns `true` if any active browser has room for another page.
436
+ */
437
+ hasActiveBrowserWithFreeCapacity(): boolean;
376
438
  private _initializeFingerprinting;
377
439
  private _addFingerprintHooks;
378
440
  }
379
- export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> {
441
+ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
442
+ /**
443
+ * The proxy URL the pool uses internally to route the page: it keys browser
444
+ * reuse (with `browserPerProxy`, only a browser already on this proxy is
445
+ * reused), configures the launched browser, and is applied to incognito
446
+ * pages. When omitted, it is derived from the
447
+ * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
448
+ * here takes precedence.
449
+ *
450
+ * This is an implementation detail of the built-in `BrowserPool`'s proxy
451
+ * handling and is intentionally not part of the {@link IBrowserPool}
452
+ * contract — through that interface the proxy is supplied via the session.
453
+ */
454
+ proxyUrl?: string;
380
455
  /**
381
- * Assign a custom ID to the page. If you don't a random string ID
382
- * will be generated.
456
+ * Disable TLS certificate verification for MITM proxies. Applied both when
457
+ * launching a new browser and when creating a page in an existing one. When
458
+ * omitted, it is derived from the
459
+ * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
460
+ * here takes precedence.
461
+ *
462
+ * This is an implementation detail of the built-in `BrowserPool` and is
463
+ * intentionally not part of the {@link IBrowserPool} contract — through
464
+ * that interface, configure it via the session's `proxyInfo` or through the
465
+ * browser's `launchOptions`.
383
466
  */
384
- id?: string;
467
+ ignoreTlsErrors?: boolean;
385
468
  /**
386
469
  * Some libraries (Playwright) allow you to open new pages with specific
387
470
  * options. Use this property to set those options.
@@ -396,14 +479,6 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
396
479
  * see the `newPageInNewBrowser` function.
397
480
  */
398
481
  browserPlugin?: BP;
399
- /**
400
- * Proxy URL.
401
- */
402
- proxyUrl?: string;
403
- /**
404
- * Proxy tier.
405
- */
406
- proxyTier?: number;
407
482
  }
408
483
  export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
409
484
  /**
@@ -434,4 +509,3 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
434
509
  */
435
510
  launchOptions?: BP['launchOptions'];
436
511
  }
437
- //# sourceMappingURL=browser-pool.d.ts.map
package/browser-pool.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { AsyncResource } from 'node:async_hooks';
2
+ import { SessionError, serviceLocator } from '@crawlee/core';
1
3
  import { FingerprintGenerator } from 'fingerprint-generator';
2
4
  import { FingerprintInjector } from 'fingerprint-injector';
3
5
  import { nanoid } from 'nanoid';
@@ -7,7 +9,7 @@ import QuickLRU from 'quick-lru';
7
9
  import { TypedEmitter } from 'tiny-typed-emitter';
8
10
  import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
9
11
  import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
10
- import { log } from './logger.js';
12
+ const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
11
13
  const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
12
14
  const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
13
15
  /**
@@ -60,6 +62,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
60
62
  export class BrowserPool extends TypedEmitter {
61
63
  browserPlugins;
62
64
  maxOpenPagesPerBrowser;
65
+ maxOpenBrowsers;
63
66
  retireBrowserAfterPageCount;
64
67
  operationTimeoutMillis;
65
68
  closeInactiveBrowserAfterMillis;
@@ -74,6 +77,7 @@ export class BrowserPool extends TypedEmitter {
74
77
  pageCounter = 0;
75
78
  pages = new Map();
76
79
  pageIds = new WeakMap();
80
+ startingBrowserControllers = new Set();
77
81
  activeBrowserControllers = new Set();
78
82
  retiredBrowserControllers = new Set();
79
83
  pageToBrowserController = new WeakMap();
@@ -83,8 +87,10 @@ export class BrowserPool extends TypedEmitter {
83
87
  browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
84
88
  browserRetireInterval;
85
89
  limiter = pLimit(1);
90
+ log;
86
91
  constructor(options) {
87
92
  super();
93
+ this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
88
94
  this.browserKillerInterval.unref();
89
95
  ow(options, ow.object.exactShape({
90
96
  browserPlugins: ow.array.minLength(1),
@@ -114,6 +120,7 @@ export class BrowserPool extends TypedEmitter {
114
120
  }
115
121
  this.browserPlugins = browserPlugins;
116
122
  this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
123
+ this.maxOpenBrowsers = Infinity;
117
124
  this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
118
125
  this.operationTimeoutMillis = operationTimeoutSecs * 1000;
119
126
  this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
@@ -142,23 +149,46 @@ export class BrowserPool extends TypedEmitter {
142
149
  * Opens a new page in one of the running browsers or launches
143
150
  * a new browser and opens a page there, if no browsers are active,
144
151
  * or their page limits have been exceeded.
152
+ *
153
+ * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
154
+ * provided, this implementation uses it as a cache key for browser fingerprints (when
155
+ * fingerprinting is enabled) and reads
156
+ * {@link ProxyInfo.url|session.proxyInfo.url} /
157
+ * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
158
+ * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
159
+ * `ignoreTlsErrors` values in the options take precedence.
160
+ *
161
+ * Beyond fingerprint caching and proxy configuration, no other session
162
+ * properties are consumed — cookie and header injection remain the
163
+ * crawler's responsibility.
145
164
  */
146
165
  async newPage(options = {}) {
147
- const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
166
+ const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
148
167
  if (this.pages.has(id)) {
149
168
  throw new Error(`Page with ID: ${id} already exists.`);
150
169
  }
151
170
  if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
152
171
  throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
153
172
  }
173
+ // Bind the limiter callback to the current async-hooks context. p-limit
174
+ // otherwise resumes queued callbacks in the previous task's
175
+ // AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
176
+ // requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
177
+ // fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
178
+ // ESM-only rewrite, so we can't bump it in Crawlee v3.
179
+ // TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
154
180
  // Limiter is necessary - https://github.com/apify/crawlee/issues/1126
155
- return this.limiter(async () => {
156
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
181
+ return this.limiter(AsyncResource.bind(async () => {
182
+ let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
157
183
  if (!browserController)
158
- browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
184
+ browserController = await this._launchBrowser(id, {
185
+ browserPlugin,
186
+ proxyUrl,
187
+ ignoreTlsErrors,
188
+ });
159
189
  tryCancel();
160
- return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
161
- });
190
+ return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
191
+ }));
162
192
  }
163
193
  /**
164
194
  * Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
@@ -238,7 +268,7 @@ export class BrowserPool extends TypedEmitter {
238
268
  getPageId(page) {
239
269
  return this.pageIds.get(page);
240
270
  }
241
- async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
271
+ async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
242
272
  // This is needed for concurrent newPage calls to wait for the browser launch.
243
273
  // It's not ideal though, we need to come up with a better API.
244
274
  // eslint-disable-next-line dot-notation -- accessing private property
@@ -247,6 +277,12 @@ export class BrowserPool extends TypedEmitter {
247
277
  const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
248
278
  if (finalPageOptions) {
249
279
  Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
280
+ if (ignoreTlsErrors) {
281
+ Object.assign(finalPageOptions, {
282
+ ignoreHTTPSErrors: true,
283
+ acceptInsecureCerts: true,
284
+ });
285
+ }
250
286
  }
251
287
  await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
252
288
  tryCancel();
@@ -278,11 +314,14 @@ export class BrowserPool extends TypedEmitter {
278
314
  *
279
315
  */
280
316
  retireBrowserController(browserController) {
281
- const hasBeenRetiredOrKilled = !this.activeBrowserControllers.has(browserController);
317
+ const isStarting = this.startingBrowserControllers.has(browserController);
318
+ const isActive = this.activeBrowserControllers.has(browserController);
319
+ const hasBeenRetiredOrKilled = !isStarting && !isActive;
282
320
  if (hasBeenRetiredOrKilled)
283
321
  return;
284
322
  this.retiredBrowserControllers.add(browserController);
285
323
  this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
324
+ this.startingBrowserControllers.delete(browserController);
286
325
  this.activeBrowserControllers.delete(browserController);
287
326
  }
288
327
  /**
@@ -294,12 +333,65 @@ export class BrowserPool extends TypedEmitter {
294
333
  if (browserController)
295
334
  this.retireBrowserController(browserController);
296
335
  }
336
+ /**
337
+ * Releases a page back to the pool. The page is closed and, if the
338
+ * optional `error` is a {@link SessionError}, the browser controller
339
+ * that served the page is retired so that its tainted state (cookies,
340
+ * storage, etc.) cannot leak into future sessions.
341
+ *
342
+ * This is the primary way the crawler should return pages to the pool.
343
+ *
344
+ * @param page The page to release.
345
+ * @param options.error The error that caused the page to be released, if any.
346
+ */
347
+ async closePage(page, options) {
348
+ if (options?.error instanceof SessionError) {
349
+ this.retireBrowserByPage(page);
350
+ }
351
+ // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
352
+ await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
353
+ }
354
+ /**
355
+ * Extracts the relevant state (currently just cookies) from a page via its
356
+ * owning {@link BrowserController}. Returns empty state when the page is
357
+ * no longer associated with a controller.
358
+ *
359
+ * As with {@link BrowserPool.injectPageState}, cookies are isolated per
360
+ * page only when the pool is configured with `useIncognitoPages: true`.
361
+ * With the default `useIncognitoPages: false`, the extracted cookies
362
+ * include those set by any sibling page sharing the same browser.
363
+ */
364
+ async extractPageState(page) {
365
+ const controller = this.getBrowserControllerByPage(page);
366
+ if (!controller) {
367
+ return { cookies: [] };
368
+ }
369
+ return { cookies: await controller.getCookies(page) };
370
+ }
371
+ /**
372
+ * Injects state into a page via its owning {@link BrowserController}.
373
+ *
374
+ * No-op when the page is no longer associated with a controller.
375
+ *
376
+ * Note that cookies are isolated per page only when the pool is configured
377
+ * with `useIncognitoPages: true` — each page then gets its own browser
378
+ * context. With the default `useIncognitoPages: false`, all pages in a
379
+ * browser share a single context, so injected cookies are visible to every
380
+ * page served by that browser.
381
+ */
382
+ async injectPageState(page, state) {
383
+ const controller = this.getBrowserControllerByPage(page);
384
+ if (!controller) {
385
+ return;
386
+ }
387
+ await controller.setCookies(page, state.cookies);
388
+ }
297
389
  /**
298
390
  * Removes all active browsers from the pool. The browsers will be
299
391
  * closed after all their pages are closed.
300
392
  */
301
393
  retireAllBrowsers() {
302
- this.activeBrowserControllers.forEach((controller) => {
394
+ [...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
303
395
  this.retireBrowserController(controller);
304
396
  });
305
397
  }
@@ -326,23 +418,36 @@ export class BrowserPool extends TypedEmitter {
326
418
  this._teardown();
327
419
  }
328
420
  _teardown() {
421
+ this.startingBrowserControllers.clear();
329
422
  this.activeBrowserControllers.clear();
330
423
  this.retiredBrowserControllers.clear();
331
424
  this.removeAllListeners();
332
425
  }
333
426
  _getAllBrowserControllers() {
334
- return new Set([...this.activeBrowserControllers, ...this.retiredBrowserControllers]);
427
+ return new Set([
428
+ ...this.startingBrowserControllers,
429
+ ...this.activeBrowserControllers,
430
+ ...this.retiredBrowserControllers,
431
+ ]);
335
432
  }
336
433
  async _launchBrowser(pageId, options) {
337
- const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
434
+ const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
338
435
  const browserController = browserPlugin.createController();
339
- this.activeBrowserControllers.add(browserController);
436
+ this.startingBrowserControllers.add(browserController);
340
437
  const launchContext = browserPlugin.createLaunchContext({
341
438
  id: pageId,
342
439
  launchOptions,
343
- proxyTier,
344
440
  proxyUrl,
345
441
  });
442
+ // Disable SSL verification for MITM proxies
443
+ if (ignoreTlsErrors) {
444
+ /**
445
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
446
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
447
+ */
448
+ launchContext.launchOptions.ignoreHTTPSErrors = true;
449
+ launchContext.launchOptions.acceptInsecureCerts = true;
450
+ }
346
451
  try {
347
452
  // If the hooks or the launch fails, we need to delete the controller,
348
453
  // because otherwise it would be stuck in limbo without a browser.
@@ -353,11 +458,10 @@ export class BrowserPool extends TypedEmitter {
353
458
  browserController.assignBrowser(browser, launchContext);
354
459
  }
355
460
  catch (err) {
356
- this.activeBrowserControllers.delete(browserController);
461
+ this.startingBrowserControllers.delete(browserController);
357
462
  throw err;
358
463
  }
359
- log.debug('Launched new browser.', { id: browserController.id });
360
- browserController.proxyTier = proxyTier;
464
+ this.log.debug('Launched new browser.', { id: browserController.id });
361
465
  browserController.proxyUrl = proxyUrl;
362
466
  try {
363
467
  // If the launch fails on the post-launch hooks, we need to clean up
@@ -365,9 +469,9 @@ export class BrowserPool extends TypedEmitter {
365
469
  await this._executeHooks(this.postLaunchHooks, pageId, browserController);
366
470
  }
367
471
  catch (err) {
368
- this.activeBrowserControllers.delete(browserController);
472
+ this.startingBrowserControllers.delete(browserController);
369
473
  browserController.close().catch((closeErr) => {
370
- log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
474
+ this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
371
475
  id: browserController.id,
372
476
  });
373
477
  });
@@ -375,6 +479,8 @@ export class BrowserPool extends TypedEmitter {
375
479
  }
376
480
  tryCancel();
377
481
  browserController.activate();
482
+ this.startingBrowserControllers.delete(browserController);
483
+ this.activeBrowserControllers.add(browserController);
378
484
  this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
379
485
  return browserController;
380
486
  }
@@ -392,13 +498,11 @@ export class BrowserPool extends TypedEmitter {
392
498
  const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
393
499
  const isCorrectPlugin = controller.browserPlugin === browserPlugin;
394
500
  const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
395
- const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
396
501
  return (isCorrectPlugin &&
397
502
  hasCapacity &&
398
- ((!controller.launchContext.browserPerProxy && !options?.proxyTier) ||
399
- (options?.proxyTier && isCorrectProxyTier) ||
503
+ (!controller.launchContext.browserPerProxy ||
400
504
  (options?.proxyUrl && isSameProxyUrl) ||
401
- (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)));
505
+ (!options?.proxyUrl && !controller.proxyUrl)));
402
506
  });
403
507
  }
404
508
  async _closeInactiveRetiredBrowsers() {
@@ -409,14 +513,14 @@ export class BrowserPool extends TypedEmitter {
409
513
  const isBrowserEmpty = controller.activePages === 0;
410
514
  if (isBrowserIdle || isBrowserEmpty) {
411
515
  const { id } = controller;
412
- log.debug('Closing retired browser.', { id });
516
+ this.log.debug('Closing retired browser.', { id });
413
517
  await controller.close();
414
518
  this.retiredBrowserControllers.delete(controller);
415
519
  closedBrowserIds.push(id);
416
520
  }
417
521
  }
418
522
  if (closedBrowserIds.length) {
419
- log.debug('Closed retired browsers.', {
523
+ this.log.debug('Closed retired browsers.', {
420
524
  count: closedBrowserIds.length,
421
525
  closedBrowserIds,
422
526
  });
@@ -429,7 +533,7 @@ export class BrowserPool extends TypedEmitter {
429
533
  page.close = async (...args) => {
430
534
  await this._executeHooks(this.prePageCloseHooks, page, browserController);
431
535
  await originalPageClose.apply(page, args).catch((err) => {
432
- log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
536
+ this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
433
537
  });
434
538
  await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
435
539
  this.pages.delete(pageId);
@@ -447,13 +551,33 @@ export class BrowserPool extends TypedEmitter {
447
551
  // Run this with a delay, otherwise page.close()
448
552
  // might fail with "Protocol error (Target.closeTarget): Target closed."
449
553
  setTimeout(() => {
450
- log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
554
+ this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
451
555
  void browserController.close().finally(() => {
452
556
  this.retiredBrowserControllers.delete(browserController);
453
557
  });
454
558
  }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
455
559
  }
456
560
  }
561
+ /**
562
+ * Returns `true` if the pool can accept a new browser launch without exceeding
563
+ * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
564
+ */
565
+ hasFreeBrowserSlot() {
566
+ const total = this.startingBrowserControllers.size +
567
+ this.activeBrowserControllers.size +
568
+ this.retiredBrowserControllers.size;
569
+ return total < this.maxOpenBrowsers;
570
+ }
571
+ /**
572
+ * Returns `true` if any active browser has room for another page.
573
+ */
574
+ hasActiveBrowserWithFreeCapacity() {
575
+ for (const controller of this.activeBrowserControllers) {
576
+ if (controller.activePages < this.maxOpenPagesPerBrowser)
577
+ return true;
578
+ }
579
+ return false;
580
+ }
457
581
  _initializeFingerprinting() {
458
582
  const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
459
583
  this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
@@ -474,4 +598,3 @@ export class BrowserPool extends TypedEmitter {
474
598
  this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
475
599
  }
476
600
  }
477
- //# sourceMappingURL=browser-pool.js.map
@@ -9,4 +9,3 @@ export declare function createProxyServerForContainers(fallbackProxyUrl?: string
9
9
  ipToProxy: Map<string, string>;
10
10
  close(closeConnections: boolean): Promise<void>;
11
11
  }>;
12
- //# sourceMappingURL=container-proxy-server.d.ts.map
@@ -40,4 +40,3 @@ export async function createProxyServerForContainers(fallbackProxyUrl) {
40
40
  },
41
41
  };
42
42
  }
43
- //# sourceMappingURL=container-proxy-server.js.map
package/events.d.ts CHANGED
@@ -8,4 +8,3 @@ export declare const enum BROWSER_POOL_EVENTS {
8
8
  export declare const enum BROWSER_CONTROLLER_EVENTS {
9
9
  BROWSER_CLOSED = "browserClosed"
10
10
  }
11
- //# sourceMappingURL=events.d.ts.map
package/events.js CHANGED
@@ -10,4 +10,3 @@ export var BROWSER_CONTROLLER_EVENTS;
10
10
  (function (BROWSER_CONTROLLER_EVENTS) {
11
11
  BROWSER_CONTROLLER_EVENTS["BROWSER_CLOSED"] = "browserClosed";
12
12
  })(BROWSER_CONTROLLER_EVENTS || (BROWSER_CONTROLLER_EVENTS = {}));
13
- //# sourceMappingURL=events.js.map
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
14
14
  * @internal
15
15
  */
16
16
  export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
17
- //# sourceMappingURL=hooks.d.ts.map
@@ -1,27 +1,39 @@
1
1
  import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
2
2
  import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
3
3
  import { getGeneratorDefaultOptions } from './utils.js';
4
+ function applySessionHints(base, fingerprint) {
5
+ if (!fingerprint)
6
+ return base;
7
+ return {
8
+ ...base,
9
+ ...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
10
+ ...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
11
+ ...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
12
+ };
13
+ }
4
14
  /**
5
15
  * @internal
6
16
  */
7
17
  export function createFingerprintPreLaunchHook(browserPool) {
8
18
  const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
9
19
  return (_pageId, launchContext) => {
20
+ // Remote browsers may have their own fingerprinting — skip local fingerprint injection
21
+ if (launchContext.isRemote)
22
+ return;
10
23
  const { useIncognitoPages } = launchContext;
11
- const cacheKey = launchContext.session?.id ?? launchContext.proxyUrl;
24
+ const session = launchContext.session;
25
+ const cacheKey = session?.id ?? launchContext.proxyUrl;
12
26
  const { launchOptions } = launchContext;
13
- // If no options are passed we try to pass best default options as possible to match browser and OS.
14
- const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
15
27
  let fingerprint;
16
28
  if (cacheKey && fingerprintCache?.has(cacheKey)) {
17
29
  fingerprint = fingerprintCache.get(cacheKey);
18
30
  }
19
- else if (cacheKey) {
20
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
21
- fingerprintCache?.set(cacheKey, fingerprint);
22
- }
23
31
  else {
24
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
32
+ const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
33
+ const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
34
+ fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
35
+ if (cacheKey)
36
+ fingerprintCache?.set(cacheKey, fingerprint);
25
37
  }
26
38
  launchContext.extend({ fingerprint });
27
39
  if (useIncognitoPages) {
@@ -41,6 +53,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
41
53
  export function createPrePageCreateHook() {
42
54
  return (_pageId, browserController, pageOptions) => {
43
55
  const { launchContext, browserPlugin } = browserController;
56
+ if (launchContext.isRemote)
57
+ return;
44
58
  const { fingerprint } = launchContext.fingerprint;
45
59
  if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
46
60
  pageOptions.userAgent ??= fingerprint.navigator.userAgent;
@@ -57,6 +71,8 @@ export function createPrePageCreateHook() {
57
71
  export function createPostPageCreateHook(fingerprintInjector) {
58
72
  return async (page, browserController) => {
59
73
  const { browserPlugin, launchContext } = browserController;
74
+ if (launchContext.isRemote)
75
+ return;
60
76
  const fingerprint = launchContext.fingerprint;
61
77
  // TODO this will require refactoring, we should use common API instead of branching based on plugin type,
62
78
  // and there should be no public methods specific to some browser.
@@ -79,4 +95,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
79
95
  }
80
96
  };
81
97
  }
82
- //# sourceMappingURL=hooks.js.map
@@ -61,4 +61,3 @@ export declare const enum DeviceCategory {
61
61
  desktop = "desktop"
62
62
  }
63
63
  export {};
64
- //# sourceMappingURL=types.d.ts.map
@@ -32,4 +32,3 @@ export var DeviceCategory;
32
32
  */
33
33
  DeviceCategory["desktop"] = "desktop";
34
34
  })(DeviceCategory || (DeviceCategory = {}));
35
- //# sourceMappingURL=types.js.map
@@ -1,4 +1,3 @@
1
1
  import type { LaunchContext } from '../launch-context.js';
2
2
  import type { FingerprintGeneratorOptions } from './types.js';
3
3
  export declare const getGeneratorDefaultOptions: (launchContext: LaunchContext) => FingerprintGeneratorOptions;
4
- //# sourceMappingURL=utils.d.ts.map
@@ -42,4 +42,3 @@ const getOperatingSystem = () => {
42
42
  return "linux" /* OperatingSystemsName.linux */;
43
43
  }
44
44
  };
45
- //# sourceMappingURL=utils.js.map