@crawlee/browser-pool 4.0.0-beta.16 → 4.0.0-beta.161

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +14 -14
  2. package/abstract-classes/browser-controller.d.ts +74 -14
  3. package/abstract-classes/browser-controller.js +28 -24
  4. package/abstract-classes/browser-plugin.d.ts +48 -15
  5. package/abstract-classes/browser-plugin.js +69 -11
  6. package/anonymize-proxy.d.ts +5 -2
  7. package/anonymize-proxy.js +9 -5
  8. package/browser-pool.d.ts +100 -28
  9. package/browser-pool.js +199 -79
  10. package/container-proxy-server.d.ts +0 -1
  11. package/container-proxy-server.js +0 -1
  12. package/events.d.ts +2 -3
  13. package/events.js +0 -1
  14. package/fingerprinting/hooks.d.ts +0 -1
  15. package/fingerprinting/hooks.js +26 -10
  16. package/fingerprinting/types.d.ts +3 -4
  17. package/fingerprinting/types.js +0 -1
  18. package/fingerprinting/utils.d.ts +0 -1
  19. package/fingerprinting/utils.js +5 -6
  20. package/index.d.ts +14 -7
  21. package/index.js +5 -3
  22. package/launch-context.d.ts +20 -5
  23. package/launch-context.js +21 -10
  24. package/package.json +9 -9
  25. package/playwright/playwright-browser.d.ts +2 -6
  26. package/playwright/playwright-browser.js +16 -17
  27. package/playwright/playwright-controller.d.ts +0 -1
  28. package/playwright/playwright-controller.js +6 -3
  29. package/playwright/playwright-plugin.d.ts +10 -7
  30. package/playwright/playwright-plugin.js +40 -16
  31. package/proxy-server.d.ts +0 -1
  32. package/proxy-server.js +0 -1
  33. package/puppeteer/puppeteer-controller.d.ts +0 -1
  34. package/puppeteer/puppeteer-controller.js +16 -9
  35. package/puppeteer/puppeteer-plugin.d.ts +5 -3
  36. package/puppeteer/puppeteer-plugin.js +83 -46
  37. package/remote-browser-pool.d.ts +165 -0
  38. package/remote-browser-pool.js +195 -0
  39. package/remote-browser-provider.d.ts +83 -0
  40. package/remote-browser-provider.js +67 -0
  41. package/utils.d.ts +7 -1
  42. package/utils.js +19 -1
  43. package/abstract-classes/browser-controller.d.ts.map +0 -1
  44. package/abstract-classes/browser-controller.js.map +0 -1
  45. package/abstract-classes/browser-plugin.d.ts.map +0 -1
  46. package/abstract-classes/browser-plugin.js.map +0 -1
  47. package/anonymize-proxy.d.ts.map +0 -1
  48. package/anonymize-proxy.js.map +0 -1
  49. package/browser-pool.d.ts.map +0 -1
  50. package/browser-pool.js.map +0 -1
  51. package/container-proxy-server.d.ts.map +0 -1
  52. package/container-proxy-server.js.map +0 -1
  53. package/events.d.ts.map +0 -1
  54. package/events.js.map +0 -1
  55. package/fingerprinting/hooks.d.ts.map +0 -1
  56. package/fingerprinting/hooks.js.map +0 -1
  57. package/fingerprinting/types.d.ts.map +0 -1
  58. package/fingerprinting/types.js.map +0 -1
  59. package/fingerprinting/utils.d.ts.map +0 -1
  60. package/fingerprinting/utils.js.map +0 -1
  61. package/index.d.ts.map +0 -1
  62. package/index.js.map +0 -1
  63. package/launch-context.d.ts.map +0 -1
  64. package/launch-context.js.map +0 -1
  65. package/logger.d.ts +0 -3
  66. package/logger.d.ts.map +0 -1
  67. package/logger.js +0 -5
  68. package/logger.js.map +0 -1
  69. package/playwright/playwright-browser.d.ts.map +0 -1
  70. package/playwright/playwright-browser.js.map +0 -1
  71. package/playwright/playwright-controller.d.ts.map +0 -1
  72. package/playwright/playwright-controller.js.map +0 -1
  73. package/playwright/playwright-plugin.d.ts.map +0 -1
  74. package/playwright/playwright-plugin.js.map +0 -1
  75. package/proxy-server.d.ts.map +0 -1
  76. package/proxy-server.js.map +0 -1
  77. package/puppeteer/puppeteer-controller.d.ts.map +0 -1
  78. package/puppeteer/puppeteer-controller.js.map +0 -1
  79. package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
  80. package/puppeteer/puppeteer-plugin.js.map +0 -1
  81. package/utils.d.ts.map +0 -1
  82. package/utils.js.map +0 -1
package/browser-pool.js CHANGED
@@ -1,15 +1,35 @@
1
+ import { AsyncResource } from 'node:async_hooks';
2
+ import { SessionError, serviceLocator } from '@crawlee/core';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
1
4
  import { FingerprintGenerator } from 'fingerprint-generator';
2
5
  import { FingerprintInjector } from 'fingerprint-injector';
3
6
  import { nanoid } from 'nanoid';
4
- import ow from 'ow';
5
7
  import pLimit from 'p-limit';
6
8
  import QuickLRU from 'quick-lru';
7
9
  import { TypedEmitter } from 'tiny-typed-emitter';
10
+ import { z } from 'zod';
8
11
  import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
12
+ import { BROWSER_POOL_EVENTS } from './events.js';
9
13
  import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
10
- import { log } from './logger.js';
14
+ const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
11
15
  const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
12
16
  const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
17
+ const browserPoolOptionsSchema = z.strictObject({
18
+ browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
19
+ maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
20
+ retireBrowserAfterPageCount: schemas.anyNumber.default(100),
21
+ operationTimeoutSecs: schemas.anyNumber.default(15),
22
+ closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
23
+ retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
24
+ preLaunchHooks: schemas.anyArray.default(() => []),
25
+ postLaunchHooks: schemas.anyArray.default(() => []),
26
+ prePageCreateHooks: schemas.anyArray.default(() => []),
27
+ postPageCreateHooks: schemas.anyArray.default(() => []),
28
+ prePageCloseHooks: schemas.anyArray.default(() => []),
29
+ postPageCloseHooks: schemas.anyArray.default(() => []),
30
+ useFingerprints: z.boolean().default(true),
31
+ fingerprintOptions: schemas.anyObject.default(() => ({})),
32
+ });
13
33
  /**
14
34
  * The `BrowserPool` class is the most important class of the `browser-pool` module.
15
35
  * It manages opening and closing of browsers and their pages and its constructor
@@ -60,6 +80,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
60
80
  export class BrowserPool extends TypedEmitter {
61
81
  browserPlugins;
62
82
  maxOpenPagesPerBrowser;
83
+ maxOpenBrowsers;
63
84
  retireBrowserAfterPageCount;
64
85
  operationTimeoutMillis;
65
86
  closeInactiveBrowserAfterMillis;
@@ -81,29 +102,16 @@ export class BrowserPool extends TypedEmitter {
81
102
  fingerprintInjector;
82
103
  fingerprintGenerator;
83
104
  fingerprintCache;
84
- browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
85
- browserRetireInterval;
86
- limiter = pLimit(1);
105
+ // kept as TS-private: tests replace this interval through bracket access
106
+ browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
107
+ #browserRetireInterval;
108
+ #limiter = pLimit(1);
109
+ #log;
87
110
  constructor(options) {
88
111
  super();
112
+ this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
89
113
  this.browserKillerInterval.unref();
90
- ow(options, ow.object.exactShape({
91
- browserPlugins: ow.array.minLength(1),
92
- maxOpenPagesPerBrowser: ow.optional.number,
93
- retireBrowserAfterPageCount: ow.optional.number,
94
- operationTimeoutSecs: ow.optional.number,
95
- closeInactiveBrowserAfterSecs: ow.optional.number,
96
- retireInactiveBrowserAfterSecs: ow.optional.number,
97
- preLaunchHooks: ow.optional.array,
98
- postLaunchHooks: ow.optional.array,
99
- prePageCreateHooks: ow.optional.array,
100
- postPageCreateHooks: ow.optional.array,
101
- prePageCloseHooks: ow.optional.array,
102
- postPageCloseHooks: ow.optional.array,
103
- useFingerprints: ow.optional.boolean,
104
- fingerprintOptions: ow.optional.object,
105
- }));
106
- const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
114
+ const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
107
115
  const firstPluginConstructor = browserPlugins[0].constructor;
108
116
  for (let i = 1; i < browserPlugins.length; i++) {
109
117
  const providedPlugin = browserPlugins[i];
@@ -115,18 +123,19 @@ export class BrowserPool extends TypedEmitter {
115
123
  }
116
124
  this.browserPlugins = browserPlugins;
117
125
  this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
126
+ this.maxOpenBrowsers = Infinity;
118
127
  this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
119
128
  this.operationTimeoutMillis = operationTimeoutSecs * 1000;
120
129
  this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
121
130
  this.useFingerprints = useFingerprints;
122
131
  this.fingerprintOptions = fingerprintOptions;
123
- this.browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
132
+ this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
124
133
  if (controller.activePages === 0 &&
125
134
  controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
126
135
  this.retireBrowserController(controller);
127
136
  }
128
137
  }), retireInactiveBrowserAfterSecs * 1000);
129
- this.browserRetireInterval.unref();
138
+ this.#browserRetireInterval.unref();
130
139
  // hooks
131
140
  this.preLaunchHooks = preLaunchHooks;
132
141
  this.postLaunchHooks = postLaunchHooks;
@@ -136,30 +145,56 @@ export class BrowserPool extends TypedEmitter {
136
145
  this.postPageCloseHooks = postPageCloseHooks;
137
146
  // fingerprinting
138
147
  if (this.useFingerprints) {
139
- this._initializeFingerprinting();
148
+ this.initializeFingerprinting();
140
149
  }
141
150
  }
142
151
  /**
143
152
  * Opens a new page in one of the running browsers or launches
144
153
  * a new browser and opens a page there, if no browsers are active,
145
154
  * or their page limits have been exceeded.
155
+ *
156
+ * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
157
+ * provided, this implementation uses it as a cache key for browser fingerprints (when
158
+ * fingerprinting is enabled) and reads
159
+ * {@link ProxyInfo.url|session.proxyInfo.url} /
160
+ * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
161
+ * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
162
+ * `ignoreTlsErrors` values in the options take precedence.
163
+ *
164
+ * Beyond fingerprint caching and proxy configuration, no other session
165
+ * properties are consumed — cookie and header injection remain the
166
+ * crawler's responsibility.
146
167
  */
147
168
  async newPage(options = {}) {
148
- const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
169
+ const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
149
170
  if (this.pages.has(id)) {
150
171
  throw new Error(`Page with ID: ${id} already exists.`);
151
172
  }
152
173
  if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
153
174
  throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
154
175
  }
176
+ // Bind the limiter callback to the current async-hooks context. p-limit
177
+ // otherwise resumes queued callbacks in the previous task's
178
+ // AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
179
+ // requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
180
+ // fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
181
+ // ESM-only rewrite, so we can't bump it in Crawlee v3.
182
+ // Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
183
+ // ALS-scoped: without it, a queued callback would resume in the previous request's async
184
+ // context and run request B's storage writes inside request A's transaction.
185
+ // TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
155
186
  // Limiter is necessary - https://github.com/apify/crawlee/issues/1126
156
- return this.limiter(async () => {
157
- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
187
+ return this.#limiter(AsyncResource.bind(async () => {
188
+ let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
158
189
  if (!browserController)
159
- browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
190
+ browserController = await this.launchBrowser(id, {
191
+ browserPlugin,
192
+ proxyUrl,
193
+ ignoreTlsErrors,
194
+ });
160
195
  tryCancel();
161
- return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
162
- });
196
+ return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
197
+ }));
163
198
  }
164
199
  /**
165
200
  * Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
@@ -167,13 +202,13 @@ export class BrowserPool extends TypedEmitter {
167
202
  * configure the new browser.
168
203
  */
169
204
  async newPageInNewBrowser(options = {}) {
170
- const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;
205
+ const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
171
206
  if (this.pages.has(id)) {
172
207
  throw new Error(`Page with ID: ${id} already exists.`);
173
208
  }
174
- const browserController = await this._launchBrowser(id, { launchOptions, browserPlugin });
209
+ const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
175
210
  tryCancel();
176
- return await this._createPageForBrowser(id, browserController, pageOptions);
211
+ return await this.createPageForBrowser(id, browserController, pageOptions);
177
212
  }
178
213
  /**
179
214
  * Opens new pages with all available plugins and returns an array
@@ -239,17 +274,22 @@ export class BrowserPool extends TypedEmitter {
239
274
  getPageId(page) {
240
275
  return this.pageIds.get(page);
241
276
  }
242
- async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
277
+ async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
243
278
  // This is needed for concurrent newPage calls to wait for the browser launch.
244
279
  // It's not ideal though, we need to come up with a better API.
245
- // eslint-disable-next-line dot-notation -- accessing private property
246
- await browserController['isActivePromise'];
280
+ await browserController.waitForActive();
247
281
  tryCancel();
248
282
  const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
249
283
  if (finalPageOptions) {
250
284
  Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
285
+ if (ignoreTlsErrors) {
286
+ Object.assign(finalPageOptions, {
287
+ ignoreHTTPSErrors: true,
288
+ acceptInsecureCerts: true,
289
+ });
290
+ }
251
291
  }
252
- await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
292
+ await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
253
293
  tryCancel();
254
294
  let page;
255
295
  try {
@@ -262,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
262
302
  if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
263
303
  this.retireBrowserController(browserController);
264
304
  }
265
- this._overridePageClose(page);
305
+ this.overridePageClose(page);
266
306
  }
267
307
  catch (err) {
268
308
  this.retireBrowserController(browserController);
269
309
  throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
270
310
  }
271
- await this._executeHooks(this.postPageCreateHooks, page, browserController);
311
+ await this.executeHooks(this.postPageCreateHooks, page, browserController);
272
312
  tryCancel();
273
- this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
313
+ this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
274
314
  return page;
275
315
  }
276
316
  /**
@@ -285,7 +325,7 @@ export class BrowserPool extends TypedEmitter {
285
325
  if (hasBeenRetiredOrKilled)
286
326
  return;
287
327
  this.retiredBrowserControllers.add(browserController);
288
- this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
328
+ this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
289
329
  this.startingBrowserControllers.delete(browserController);
290
330
  this.activeBrowserControllers.delete(browserController);
291
331
  }
@@ -298,6 +338,59 @@ export class BrowserPool extends TypedEmitter {
298
338
  if (browserController)
299
339
  this.retireBrowserController(browserController);
300
340
  }
341
+ /**
342
+ * Releases a page back to the pool. The page is closed and, if the
343
+ * optional `error` is a {@link SessionError}, the browser controller
344
+ * that served the page is retired so that its tainted state (cookies,
345
+ * storage, etc.) cannot leak into future sessions.
346
+ *
347
+ * This is the primary way the crawler should return pages to the pool.
348
+ *
349
+ * @param page The page to release.
350
+ * @param options.error The error that caused the page to be released, if any.
351
+ */
352
+ async closePage(page, options) {
353
+ if (options?.error instanceof SessionError) {
354
+ this.retireBrowserByPage(page);
355
+ }
356
+ // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
357
+ await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
358
+ }
359
+ /**
360
+ * Extracts the relevant state (currently just cookies) from a page via its
361
+ * owning {@link BrowserController}. Returns empty state when the page is
362
+ * no longer associated with a controller.
363
+ *
364
+ * As with {@link BrowserPool.injectPageState}, cookies are isolated per
365
+ * page only when the pool is configured with `useIncognitoPages: true`.
366
+ * With the default `useIncognitoPages: false`, the extracted cookies
367
+ * include those set by any sibling page sharing the same browser.
368
+ */
369
+ async extractPageState(page) {
370
+ const controller = this.getBrowserControllerByPage(page);
371
+ if (!controller) {
372
+ return { cookies: [] };
373
+ }
374
+ return { cookies: await controller.getCookies(page) };
375
+ }
376
+ /**
377
+ * Injects state into a page via its owning {@link BrowserController}.
378
+ *
379
+ * No-op when the page is no longer associated with a controller.
380
+ *
381
+ * Note that cookies are isolated per page only when the pool is configured
382
+ * with `useIncognitoPages: true` — each page then gets its own browser
383
+ * context. With the default `useIncognitoPages: false`, all pages in a
384
+ * browser share a single context, so injected cookies are visible to every
385
+ * page served by that browser.
386
+ */
387
+ async injectPageState(page, state) {
388
+ const controller = this.getBrowserControllerByPage(page);
389
+ if (!controller) {
390
+ return;
391
+ }
392
+ await controller.setCookies(page, state.cookies);
393
+ }
301
394
  /**
302
395
  * Removes all active browsers from the pool. The browsers will be
303
396
  * closed after all their pages are closed.
@@ -312,50 +405,61 @@ export class BrowserPool extends TypedEmitter {
312
405
  * @return {Promise<void>}
313
406
  */
314
407
  async closeAllBrowsers() {
315
- const controllers = this._getAllBrowserControllers();
408
+ const controllers = this.getAllBrowserControllers();
316
409
  const promises = [...controllers]
317
410
  .filter((controller) => controller.isActive)
318
411
  .map(async (controller) => controller.close());
319
412
  await Promise.all(promises);
320
413
  }
414
+ async [Symbol.asyncDispose]() {
415
+ await this.destroy();
416
+ }
321
417
  /**
322
418
  * Closes all managed browsers and tears down the pool.
323
419
  */
324
420
  async destroy() {
325
421
  clearInterval(this.browserKillerInterval);
326
- clearInterval(this.browserRetireInterval);
422
+ clearInterval(this.#browserRetireInterval);
327
423
  this.browserKillerInterval = undefined;
328
- this.browserRetireInterval = undefined;
424
+ this.#browserRetireInterval = undefined;
329
425
  await this.closeAllBrowsers();
330
- this._teardown();
426
+ this.teardown();
331
427
  }
332
- _teardown() {
428
+ teardown() {
333
429
  this.startingBrowserControllers.clear();
334
430
  this.activeBrowserControllers.clear();
335
431
  this.retiredBrowserControllers.clear();
336
432
  this.removeAllListeners();
337
433
  }
338
- _getAllBrowserControllers() {
434
+ getAllBrowserControllers() {
339
435
  return new Set([
340
436
  ...this.startingBrowserControllers,
341
437
  ...this.activeBrowserControllers,
342
438
  ...this.retiredBrowserControllers,
343
439
  ]);
344
440
  }
345
- async _launchBrowser(pageId, options) {
346
- const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
441
+ async launchBrowser(pageId, options) {
442
+ const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
347
443
  const browserController = browserPlugin.createController();
348
444
  this.startingBrowserControllers.add(browserController);
349
445
  const launchContext = browserPlugin.createLaunchContext({
350
446
  id: pageId,
351
447
  launchOptions,
352
- proxyTier,
353
448
  proxyUrl,
354
449
  });
450
+ // Disable SSL verification for MITM proxies
451
+ if (ignoreTlsErrors) {
452
+ /**
453
+ * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
454
+ * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
455
+ */
456
+ launchContext.launchOptions.ignoreHTTPSErrors = true;
457
+ launchContext.launchOptions.acceptInsecureCerts = true;
458
+ }
355
459
  try {
356
460
  // If the hooks or the launch fails, we need to delete the controller,
357
461
  // because otherwise it would be stuck in limbo without a browser.
358
- await this._executeHooks(this.preLaunchHooks, pageId, launchContext);
462
+ await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
359
463
  tryCancel();
360
464
  const browser = await browserPlugin.launch(launchContext);
361
465
  tryCancel();
@@ -365,18 +469,17 @@ export class BrowserPool extends TypedEmitter {
365
469
  this.startingBrowserControllers.delete(browserController);
366
470
  throw err;
367
471
  }
368
- log.debug('Launched new browser.', { id: browserController.id });
369
- browserController.proxyTier = proxyTier;
472
+ this.#log.debug('Launched new browser.', { id: browserController.id });
370
473
  browserController.proxyUrl = proxyUrl;
371
474
  try {
372
475
  // If the launch fails on the post-launch hooks, we need to clean up
373
476
  // both the controller and the browser before throwing.
374
- await this._executeHooks(this.postLaunchHooks, pageId, browserController);
477
+ await this.executeHooks(this.postLaunchHooks, pageId, browserController);
375
478
  }
376
479
  catch (err) {
377
480
  this.startingBrowserControllers.delete(browserController);
378
481
  browserController.close().catch((closeErr) => {
379
- log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
482
+ this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
380
483
  id: browserController.id,
381
484
  });
382
485
  });
@@ -386,33 +489,31 @@ export class BrowserPool extends TypedEmitter {
386
489
  browserController.activate();
387
490
  this.startingBrowserControllers.delete(browserController);
388
491
  this.activeBrowserControllers.add(browserController);
389
- this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
492
+ this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
390
493
  return browserController;
391
494
  }
392
495
  /**
393
496
  * Picks plugins round robin.
394
497
  * @private
395
498
  */
396
- _pickBrowserPlugin() {
499
+ pickBrowserPlugin() {
397
500
  const pluginIndex = this.pageCounter % this.browserPlugins.length;
398
501
  this.pageCounter++;
399
502
  return this.browserPlugins[pluginIndex];
400
503
  }
401
- _pickBrowserWithFreeCapacity(browserPlugin, options) {
504
+ pickBrowserWithFreeCapacity(browserPlugin, options) {
402
505
  return [...this.activeBrowserControllers].find((controller) => {
403
506
  const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
404
507
  const isCorrectPlugin = controller.browserPlugin === browserPlugin;
405
508
  const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
406
- const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
407
509
  return (isCorrectPlugin &&
408
510
  hasCapacity &&
409
- ((!controller.launchContext.browserPerProxy && !options?.proxyTier) ||
410
- (options?.proxyTier && isCorrectProxyTier) ||
511
+ (!controller.launchContext.browserPerProxy ||
411
512
  (options?.proxyUrl && isSameProxyUrl) ||
412
- (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)));
513
+ (!options?.proxyUrl && !controller.proxyUrl)));
413
514
  });
414
515
  }
415
- async _closeInactiveRetiredBrowsers() {
516
+ async closeInactiveRetiredBrowsers() {
416
517
  const closedBrowserIds = [];
417
518
  for (const controller of this.retiredBrowserControllers) {
418
519
  const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
@@ -420,61 +521,81 @@ export class BrowserPool extends TypedEmitter {
420
521
  const isBrowserEmpty = controller.activePages === 0;
421
522
  if (isBrowserIdle || isBrowserEmpty) {
422
523
  const { id } = controller;
423
- log.debug('Closing retired browser.', { id });
524
+ this.#log.debug('Closing retired browser.', { id });
424
525
  await controller.close();
425
526
  this.retiredBrowserControllers.delete(controller);
426
527
  closedBrowserIds.push(id);
427
528
  }
428
529
  }
429
530
  if (closedBrowserIds.length) {
430
- log.debug('Closed retired browsers.', {
531
+ this.#log.debug('Closed retired browsers.', {
431
532
  count: closedBrowserIds.length,
432
533
  closedBrowserIds,
433
534
  });
434
535
  }
435
536
  }
436
- _overridePageClose(page) {
537
+ overridePageClose(page) {
437
538
  const originalPageClose = page.close;
438
539
  const browserController = this.pageToBrowserController.get(page);
439
540
  const pageId = this.getPageId(page);
440
541
  page.close = async (...args) => {
441
- await this._executeHooks(this.prePageCloseHooks, page, browserController);
542
+ await this.executeHooks(this.prePageCloseHooks, page, browserController);
442
543
  await originalPageClose.apply(page, args).catch((err) => {
443
- log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
544
+ this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
444
545
  });
445
- await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
546
+ await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
446
547
  this.pages.delete(pageId);
447
- this._closeRetiredBrowserWithNoPages(browserController);
448
- this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
548
+ this.closeRetiredBrowserWithNoPages(browserController);
549
+ this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
449
550
  };
450
551
  }
451
- async _executeHooks(hooks, ...args) {
552
+ async executeHooks(hooks, ...args) {
452
553
  for (const hook of hooks) {
453
554
  await hook(...args);
454
555
  }
455
556
  }
456
- _closeRetiredBrowserWithNoPages(browserController) {
557
+ closeRetiredBrowserWithNoPages(browserController) {
457
558
  if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
458
559
  // Run this with a delay, otherwise page.close()
459
560
  // might fail with "Protocol error (Target.closeTarget): Target closed."
460
561
  setTimeout(() => {
461
- log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
562
+ this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
462
563
  void browserController.close().finally(() => {
463
564
  this.retiredBrowserControllers.delete(browserController);
464
565
  });
465
566
  }, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
466
567
  }
467
568
  }
468
- _initializeFingerprinting() {
569
+ /**
570
+ * Returns `true` if the pool can accept a new browser launch without exceeding
571
+ * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
572
+ */
573
+ hasFreeBrowserSlot() {
574
+ const total = this.startingBrowserControllers.size +
575
+ this.activeBrowserControllers.size +
576
+ this.retiredBrowserControllers.size;
577
+ return total < this.maxOpenBrowsers;
578
+ }
579
+ /**
580
+ * Returns `true` if any active browser has room for another page.
581
+ */
582
+ hasActiveBrowserWithFreeCapacity() {
583
+ for (const controller of this.activeBrowserControllers) {
584
+ if (controller.activePages < this.maxOpenPagesPerBrowser)
585
+ return true;
586
+ }
587
+ return false;
588
+ }
589
+ initializeFingerprinting() {
469
590
  const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
470
591
  this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
471
592
  this.fingerprintInjector = new FingerprintInjector();
472
593
  if (useFingerprintCache) {
473
594
  this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
474
595
  }
475
- this._addFingerprintHooks();
596
+ this.addFingerprintHooks();
476
597
  }
477
- _addFingerprintHooks() {
598
+ addFingerprintHooks() {
478
599
  this.preLaunchHooks = [
479
600
  ...this.preLaunchHooks,
480
601
  // This is flipped because of the fingerprint cache.
@@ -485,4 +606,3 @@ export class BrowserPool extends TypedEmitter {
485
606
  this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
486
607
  }
487
608
  }
488
- //# sourceMappingURL=browser-pool.js.map
@@ -9,4 +9,3 @@ export declare function createProxyServerForContainers(fallbackProxyUrl?: string
9
9
  ipToProxy: Map<string, string>;
10
10
  close(closeConnections: boolean): Promise<void>;
11
11
  }>;
12
- //# sourceMappingURL=container-proxy-server.d.ts.map
@@ -40,4 +40,3 @@ export async function createProxyServerForContainers(fallbackProxyUrl) {
40
40
  },
41
41
  };
42
42
  }
43
- //# sourceMappingURL=container-proxy-server.js.map
package/events.d.ts CHANGED
@@ -1,11 +1,10 @@
1
- export declare const enum BROWSER_POOL_EVENTS {
1
+ export declare enum BROWSER_POOL_EVENTS {
2
2
  BROWSER_LAUNCHED = "browserLaunched",
3
3
  BROWSER_RETIRED = "browserRetired",
4
4
  BROWSER_CLOSED = "browserClosed",
5
5
  PAGE_CREATED = "pageCreated",
6
6
  PAGE_CLOSED = "pageClosed"
7
7
  }
8
- export declare const enum BROWSER_CONTROLLER_EVENTS {
8
+ export declare enum BROWSER_CONTROLLER_EVENTS {
9
9
  BROWSER_CLOSED = "browserClosed"
10
10
  }
11
- //# sourceMappingURL=events.d.ts.map
package/events.js CHANGED
@@ -10,4 +10,3 @@ export var BROWSER_CONTROLLER_EVENTS;
10
10
  (function (BROWSER_CONTROLLER_EVENTS) {
11
11
  BROWSER_CONTROLLER_EVENTS["BROWSER_CLOSED"] = "browserClosed";
12
12
  })(BROWSER_CONTROLLER_EVENTS || (BROWSER_CONTROLLER_EVENTS = {}));
13
- //# sourceMappingURL=events.js.map
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
14
14
  * @internal
15
15
  */
16
16
  export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
17
- //# sourceMappingURL=hooks.d.ts.map
@@ -1,29 +1,42 @@
1
1
  import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
2
2
  import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
3
3
  import { getGeneratorDefaultOptions } from './utils.js';
4
+ function applySessionHints(base, fingerprint) {
5
+ if (!fingerprint)
6
+ return base;
7
+ return {
8
+ ...base,
9
+ ...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
10
+ ...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
11
+ ...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
12
+ };
13
+ }
4
14
  /**
5
15
  * @internal
6
16
  */
7
17
  export function createFingerprintPreLaunchHook(browserPool) {
8
18
  const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
9
19
  return (_pageId, launchContext) => {
20
+ // Remote browsers may have their own fingerprinting — skip local fingerprint injection
21
+ if (launchContext.isRemote)
22
+ return;
10
23
  const { useIncognitoPages } = launchContext;
11
- const cacheKey = launchContext.session?.id ?? launchContext.proxyUrl;
24
+ const session = launchContext.session;
25
+ const cacheKey = session?.id ?? launchContext.proxyUrl;
12
26
  const { launchOptions } = launchContext;
13
- // If no options are passed we try to pass best default options as possible to match browser and OS.
14
- const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
15
27
  let fingerprint;
16
28
  if (cacheKey && fingerprintCache?.has(cacheKey)) {
17
29
  fingerprint = fingerprintCache.get(cacheKey);
18
30
  }
19
- else if (cacheKey) {
20
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
21
- fingerprintCache?.set(cacheKey, fingerprint);
22
- }
23
31
  else {
24
- fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
32
+ const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
33
+ const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
34
+ fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
35
+ if (cacheKey)
36
+ fingerprintCache?.set(cacheKey, fingerprint);
25
37
  }
26
- launchContext.extend({ fingerprint });
38
+ // `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
39
+ launchContext.fingerprint = fingerprint;
27
40
  if (useIncognitoPages) {
28
41
  return;
29
42
  }
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
41
54
  export function createPrePageCreateHook() {
42
55
  return (_pageId, browserController, pageOptions) => {
43
56
  const { launchContext, browserPlugin } = browserController;
57
+ if (launchContext.isRemote)
58
+ return;
44
59
  const { fingerprint } = launchContext.fingerprint;
45
60
  if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
46
61
  pageOptions.userAgent ??= fingerprint.navigator.userAgent;
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
57
72
  export function createPostPageCreateHook(fingerprintInjector) {
58
73
  return async (page, browserController) => {
59
74
  const { browserPlugin, launchContext } = browserController;
75
+ if (launchContext.isRemote)
76
+ return;
60
77
  const fingerprint = launchContext.fingerprint;
61
78
  // TODO this will require refactoring, we should use common API instead of branching based on plugin type,
62
79
  // and there should be no public methods specific to some browser.
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
79
96
  }
80
97
  };
81
98
  }
82
- //# sourceMappingURL=hooks.js.map