@crawlee/browser-pool 4.0.0-beta.11 → 4.0.0-beta.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/abstract-classes/browser-controller.d.ts +70 -13
- package/abstract-classes/browser-controller.js +18 -20
- package/abstract-classes/browser-plugin.d.ts +48 -15
- package/abstract-classes/browser-plugin.js +69 -11
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +100 -28
- package/browser-pool.js +188 -61
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +0 -1
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +0 -1
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +0 -1
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +20 -5
- package/launch-context.js +21 -10
- package/package.json +8 -9
- package/playwright/playwright-browser.d.ts +2 -6
- package/playwright/playwright-browser.js +16 -17
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +20 -10
- package/playwright/playwright-plugin.d.ts +10 -7
- package/playwright/playwright-plugin.js +40 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +5 -3
- package/puppeteer/puppeteer-plugin.js +83 -46
- package/remote-browser-pool.d.ts +164 -0
- package/remote-browser-pool.js +191 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/browser-pool.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
1
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
5
|
import { nanoid } from 'nanoid';
|
|
@@ -7,7 +9,7 @@ import QuickLRU from 'quick-lru';
|
|
|
7
9
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
8
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
9
11
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
|
|
12
|
+
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
11
13
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
14
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
13
15
|
/**
|
|
@@ -60,6 +62,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
|
60
62
|
export class BrowserPool extends TypedEmitter {
|
|
61
63
|
browserPlugins;
|
|
62
64
|
maxOpenPagesPerBrowser;
|
|
65
|
+
maxOpenBrowsers;
|
|
63
66
|
retireBrowserAfterPageCount;
|
|
64
67
|
operationTimeoutMillis;
|
|
65
68
|
closeInactiveBrowserAfterMillis;
|
|
@@ -74,17 +77,21 @@ export class BrowserPool extends TypedEmitter {
|
|
|
74
77
|
pageCounter = 0;
|
|
75
78
|
pages = new Map();
|
|
76
79
|
pageIds = new WeakMap();
|
|
80
|
+
startingBrowserControllers = new Set();
|
|
77
81
|
activeBrowserControllers = new Set();
|
|
78
82
|
retiredBrowserControllers = new Set();
|
|
79
83
|
pageToBrowserController = new WeakMap();
|
|
80
84
|
fingerprintInjector;
|
|
81
85
|
fingerprintGenerator;
|
|
82
86
|
fingerprintCache;
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
87
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
88
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
89
|
+
#browserRetireInterval;
|
|
90
|
+
#limiter = pLimit(1);
|
|
91
|
+
#log;
|
|
86
92
|
constructor(options) {
|
|
87
93
|
super();
|
|
94
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
88
95
|
this.browserKillerInterval.unref();
|
|
89
96
|
ow(options, ow.object.exactShape({
|
|
90
97
|
browserPlugins: ow.array.minLength(1),
|
|
@@ -114,18 +121,19 @@ export class BrowserPool extends TypedEmitter {
|
|
|
114
121
|
}
|
|
115
122
|
this.browserPlugins = browserPlugins;
|
|
116
123
|
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
124
|
+
this.maxOpenBrowsers = Infinity;
|
|
117
125
|
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
118
126
|
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
119
127
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
120
128
|
this.useFingerprints = useFingerprints;
|
|
121
129
|
this.fingerprintOptions = fingerprintOptions;
|
|
122
|
-
this
|
|
130
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
123
131
|
if (controller.activePages === 0 &&
|
|
124
132
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
125
133
|
this.retireBrowserController(controller);
|
|
126
134
|
}
|
|
127
135
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
128
|
-
this
|
|
136
|
+
this.#browserRetireInterval.unref();
|
|
129
137
|
// hooks
|
|
130
138
|
this.preLaunchHooks = preLaunchHooks;
|
|
131
139
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -135,30 +143,56 @@ export class BrowserPool extends TypedEmitter {
|
|
|
135
143
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
136
144
|
// fingerprinting
|
|
137
145
|
if (this.useFingerprints) {
|
|
138
|
-
this.
|
|
146
|
+
this.initializeFingerprinting();
|
|
139
147
|
}
|
|
140
148
|
}
|
|
141
149
|
/**
|
|
142
150
|
* Opens a new page in one of the running browsers or launches
|
|
143
151
|
* a new browser and opens a page there, if no browsers are active,
|
|
144
152
|
* or their page limits have been exceeded.
|
|
153
|
+
*
|
|
154
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
155
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
156
|
+
* fingerprinting is enabled) and reads
|
|
157
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
158
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
159
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
160
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
161
|
+
*
|
|
162
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
163
|
+
* properties are consumed — cookie and header injection remain the
|
|
164
|
+
* crawler's responsibility.
|
|
145
165
|
*/
|
|
146
166
|
async newPage(options = {}) {
|
|
147
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
167
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
148
168
|
if (this.pages.has(id)) {
|
|
149
169
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
150
170
|
}
|
|
151
171
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
152
172
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
153
173
|
}
|
|
174
|
+
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
175
|
+
// otherwise resumes queued callbacks in the previous task's
|
|
176
|
+
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
177
|
+
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
178
|
+
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
179
|
+
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
180
|
+
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
181
|
+
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
182
|
+
// context and run request B's storage writes inside request A's transaction.
|
|
183
|
+
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
154
184
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
155
|
-
return this
|
|
156
|
-
let browserController = this.
|
|
185
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
186
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
157
187
|
if (!browserController)
|
|
158
|
-
browserController = await this.
|
|
188
|
+
browserController = await this.launchBrowser(id, {
|
|
189
|
+
browserPlugin,
|
|
190
|
+
proxyUrl,
|
|
191
|
+
ignoreTlsErrors,
|
|
192
|
+
});
|
|
159
193
|
tryCancel();
|
|
160
|
-
return await this.
|
|
161
|
-
});
|
|
194
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
195
|
+
}));
|
|
162
196
|
}
|
|
163
197
|
/**
|
|
164
198
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -166,13 +200,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
166
200
|
* configure the new browser.
|
|
167
201
|
*/
|
|
168
202
|
async newPageInNewBrowser(options = {}) {
|
|
169
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
203
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
170
204
|
if (this.pages.has(id)) {
|
|
171
205
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
172
206
|
}
|
|
173
|
-
const browserController = await this.
|
|
207
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
174
208
|
tryCancel();
|
|
175
|
-
return await this.
|
|
209
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
176
210
|
}
|
|
177
211
|
/**
|
|
178
212
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -238,7 +272,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
238
272
|
getPageId(page) {
|
|
239
273
|
return this.pageIds.get(page);
|
|
240
274
|
}
|
|
241
|
-
async
|
|
275
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
242
276
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
243
277
|
// It's not ideal though, we need to come up with a better API.
|
|
244
278
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -247,8 +281,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
247
281
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
248
282
|
if (finalPageOptions) {
|
|
249
283
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
284
|
+
if (ignoreTlsErrors) {
|
|
285
|
+
Object.assign(finalPageOptions, {
|
|
286
|
+
ignoreHTTPSErrors: true,
|
|
287
|
+
acceptInsecureCerts: true,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
250
290
|
}
|
|
251
|
-
await this.
|
|
291
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
252
292
|
tryCancel();
|
|
253
293
|
let page;
|
|
254
294
|
try {
|
|
@@ -261,13 +301,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
261
301
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
262
302
|
this.retireBrowserController(browserController);
|
|
263
303
|
}
|
|
264
|
-
this.
|
|
304
|
+
this.overridePageClose(page);
|
|
265
305
|
}
|
|
266
306
|
catch (err) {
|
|
267
307
|
this.retireBrowserController(browserController);
|
|
268
308
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
269
309
|
}
|
|
270
|
-
await this.
|
|
310
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
271
311
|
tryCancel();
|
|
272
312
|
this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
|
|
273
313
|
return page;
|
|
@@ -278,11 +318,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
278
318
|
*
|
|
279
319
|
*/
|
|
280
320
|
retireBrowserController(browserController) {
|
|
281
|
-
const
|
|
321
|
+
const isStarting = this.startingBrowserControllers.has(browserController);
|
|
322
|
+
const isActive = this.activeBrowserControllers.has(browserController);
|
|
323
|
+
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
282
324
|
if (hasBeenRetiredOrKilled)
|
|
283
325
|
return;
|
|
284
326
|
this.retiredBrowserControllers.add(browserController);
|
|
285
327
|
this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
|
|
328
|
+
this.startingBrowserControllers.delete(browserController);
|
|
286
329
|
this.activeBrowserControllers.delete(browserController);
|
|
287
330
|
}
|
|
288
331
|
/**
|
|
@@ -294,12 +337,65 @@ export class BrowserPool extends TypedEmitter {
|
|
|
294
337
|
if (browserController)
|
|
295
338
|
this.retireBrowserController(browserController);
|
|
296
339
|
}
|
|
340
|
+
/**
|
|
341
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
342
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
343
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
344
|
+
* storage, etc.) cannot leak into future sessions.
|
|
345
|
+
*
|
|
346
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
347
|
+
*
|
|
348
|
+
* @param page The page to release.
|
|
349
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
350
|
+
*/
|
|
351
|
+
async closePage(page, options) {
|
|
352
|
+
if (options?.error instanceof SessionError) {
|
|
353
|
+
this.retireBrowserByPage(page);
|
|
354
|
+
}
|
|
355
|
+
// Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
|
|
356
|
+
await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
360
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
361
|
+
* no longer associated with a controller.
|
|
362
|
+
*
|
|
363
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
364
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
365
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
366
|
+
* include those set by any sibling page sharing the same browser.
|
|
367
|
+
*/
|
|
368
|
+
async extractPageState(page) {
|
|
369
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
370
|
+
if (!controller) {
|
|
371
|
+
return { cookies: [] };
|
|
372
|
+
}
|
|
373
|
+
return { cookies: await controller.getCookies(page) };
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
377
|
+
*
|
|
378
|
+
* No-op when the page is no longer associated with a controller.
|
|
379
|
+
*
|
|
380
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
381
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
382
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
383
|
+
* browser share a single context, so injected cookies are visible to every
|
|
384
|
+
* page served by that browser.
|
|
385
|
+
*/
|
|
386
|
+
async injectPageState(page, state) {
|
|
387
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
388
|
+
if (!controller) {
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
await controller.setCookies(page, state.cookies);
|
|
392
|
+
}
|
|
297
393
|
/**
|
|
298
394
|
* Removes all active browsers from the pool. The browsers will be
|
|
299
395
|
* closed after all their pages are closed.
|
|
300
396
|
*/
|
|
301
397
|
retireAllBrowsers() {
|
|
302
|
-
this.activeBrowserControllers.forEach((controller) => {
|
|
398
|
+
[...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
303
399
|
this.retireBrowserController(controller);
|
|
304
400
|
});
|
|
305
401
|
}
|
|
@@ -308,7 +404,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
308
404
|
* @return {Promise<void>}
|
|
309
405
|
*/
|
|
310
406
|
async closeAllBrowsers() {
|
|
311
|
-
const controllers = this.
|
|
407
|
+
const controllers = this.getAllBrowserControllers();
|
|
312
408
|
const promises = [...controllers]
|
|
313
409
|
.filter((controller) => controller.isActive)
|
|
314
410
|
.map(async (controller) => controller.close());
|
|
@@ -319,55 +415,67 @@ export class BrowserPool extends TypedEmitter {
|
|
|
319
415
|
*/
|
|
320
416
|
async destroy() {
|
|
321
417
|
clearInterval(this.browserKillerInterval);
|
|
322
|
-
clearInterval(this
|
|
418
|
+
clearInterval(this.#browserRetireInterval);
|
|
323
419
|
this.browserKillerInterval = undefined;
|
|
324
|
-
this
|
|
420
|
+
this.#browserRetireInterval = undefined;
|
|
325
421
|
await this.closeAllBrowsers();
|
|
326
|
-
this.
|
|
422
|
+
this.teardown();
|
|
327
423
|
}
|
|
328
|
-
|
|
424
|
+
teardown() {
|
|
425
|
+
this.startingBrowserControllers.clear();
|
|
329
426
|
this.activeBrowserControllers.clear();
|
|
330
427
|
this.retiredBrowserControllers.clear();
|
|
331
428
|
this.removeAllListeners();
|
|
332
429
|
}
|
|
333
|
-
|
|
334
|
-
return new Set([
|
|
430
|
+
getAllBrowserControllers() {
|
|
431
|
+
return new Set([
|
|
432
|
+
...this.startingBrowserControllers,
|
|
433
|
+
...this.activeBrowserControllers,
|
|
434
|
+
...this.retiredBrowserControllers,
|
|
435
|
+
]);
|
|
335
436
|
}
|
|
336
|
-
async
|
|
337
|
-
const { browserPlugin, launchOptions,
|
|
437
|
+
async launchBrowser(pageId, options) {
|
|
438
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
338
439
|
const browserController = browserPlugin.createController();
|
|
339
|
-
this.
|
|
440
|
+
this.startingBrowserControllers.add(browserController);
|
|
340
441
|
const launchContext = browserPlugin.createLaunchContext({
|
|
341
442
|
id: pageId,
|
|
342
443
|
launchOptions,
|
|
343
|
-
proxyTier,
|
|
344
444
|
proxyUrl,
|
|
345
445
|
});
|
|
446
|
+
// Disable SSL verification for MITM proxies
|
|
447
|
+
if (ignoreTlsErrors) {
|
|
448
|
+
/**
|
|
449
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
450
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
451
|
+
*/
|
|
452
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
453
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
454
|
+
}
|
|
346
455
|
try {
|
|
347
456
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
348
457
|
// because otherwise it would be stuck in limbo without a browser.
|
|
349
|
-
await this.
|
|
458
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
350
459
|
tryCancel();
|
|
351
460
|
const browser = await browserPlugin.launch(launchContext);
|
|
352
461
|
tryCancel();
|
|
353
462
|
browserController.assignBrowser(browser, launchContext);
|
|
354
463
|
}
|
|
355
464
|
catch (err) {
|
|
356
|
-
this.
|
|
465
|
+
this.startingBrowserControllers.delete(browserController);
|
|
357
466
|
throw err;
|
|
358
467
|
}
|
|
359
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
360
|
-
browserController.proxyTier = proxyTier;
|
|
468
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
361
469
|
browserController.proxyUrl = proxyUrl;
|
|
362
470
|
try {
|
|
363
471
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
364
472
|
// both the controller and the browser before throwing.
|
|
365
|
-
await this.
|
|
473
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
366
474
|
}
|
|
367
475
|
catch (err) {
|
|
368
|
-
this.
|
|
476
|
+
this.startingBrowserControllers.delete(browserController);
|
|
369
477
|
browserController.close().catch((closeErr) => {
|
|
370
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
478
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
371
479
|
id: browserController.id,
|
|
372
480
|
});
|
|
373
481
|
});
|
|
@@ -375,6 +483,8 @@ export class BrowserPool extends TypedEmitter {
|
|
|
375
483
|
}
|
|
376
484
|
tryCancel();
|
|
377
485
|
browserController.activate();
|
|
486
|
+
this.startingBrowserControllers.delete(browserController);
|
|
487
|
+
this.activeBrowserControllers.add(browserController);
|
|
378
488
|
this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
|
|
379
489
|
return browserController;
|
|
380
490
|
}
|
|
@@ -382,26 +492,24 @@ export class BrowserPool extends TypedEmitter {
|
|
|
382
492
|
* Picks plugins round robin.
|
|
383
493
|
* @private
|
|
384
494
|
*/
|
|
385
|
-
|
|
495
|
+
pickBrowserPlugin() {
|
|
386
496
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
387
497
|
this.pageCounter++;
|
|
388
498
|
return this.browserPlugins[pluginIndex];
|
|
389
499
|
}
|
|
390
|
-
|
|
500
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
391
501
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
392
502
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
393
503
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
394
504
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
395
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
396
505
|
return (isCorrectPlugin &&
|
|
397
506
|
hasCapacity &&
|
|
398
|
-
(
|
|
399
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
507
|
+
(!controller.launchContext.browserPerProxy ||
|
|
400
508
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
401
|
-
(!options?.proxyUrl && !
|
|
509
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
402
510
|
});
|
|
403
511
|
}
|
|
404
|
-
async
|
|
512
|
+
async closeInactiveRetiredBrowsers() {
|
|
405
513
|
const closedBrowserIds = [];
|
|
406
514
|
for (const controller of this.retiredBrowserControllers) {
|
|
407
515
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -409,61 +517,81 @@ export class BrowserPool extends TypedEmitter {
|
|
|
409
517
|
const isBrowserEmpty = controller.activePages === 0;
|
|
410
518
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
411
519
|
const { id } = controller;
|
|
412
|
-
log.debug('Closing retired browser.', { id });
|
|
520
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
413
521
|
await controller.close();
|
|
414
522
|
this.retiredBrowserControllers.delete(controller);
|
|
415
523
|
closedBrowserIds.push(id);
|
|
416
524
|
}
|
|
417
525
|
}
|
|
418
526
|
if (closedBrowserIds.length) {
|
|
419
|
-
log.debug('Closed retired browsers.', {
|
|
527
|
+
this.#log.debug('Closed retired browsers.', {
|
|
420
528
|
count: closedBrowserIds.length,
|
|
421
529
|
closedBrowserIds,
|
|
422
530
|
});
|
|
423
531
|
}
|
|
424
532
|
}
|
|
425
|
-
|
|
533
|
+
overridePageClose(page) {
|
|
426
534
|
const originalPageClose = page.close;
|
|
427
535
|
const browserController = this.pageToBrowserController.get(page);
|
|
428
536
|
const pageId = this.getPageId(page);
|
|
429
537
|
page.close = async (...args) => {
|
|
430
|
-
await this.
|
|
538
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
431
539
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
432
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
540
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
433
541
|
});
|
|
434
|
-
await this.
|
|
542
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
435
543
|
this.pages.delete(pageId);
|
|
436
|
-
this.
|
|
544
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
437
545
|
this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
|
|
438
546
|
};
|
|
439
547
|
}
|
|
440
|
-
async
|
|
548
|
+
async executeHooks(hooks, ...args) {
|
|
441
549
|
for (const hook of hooks) {
|
|
442
550
|
await hook(...args);
|
|
443
551
|
}
|
|
444
552
|
}
|
|
445
|
-
|
|
553
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
446
554
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
447
555
|
// Run this with a delay, otherwise page.close()
|
|
448
556
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
449
557
|
setTimeout(() => {
|
|
450
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
558
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
451
559
|
void browserController.close().finally(() => {
|
|
452
560
|
this.retiredBrowserControllers.delete(browserController);
|
|
453
561
|
});
|
|
454
562
|
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
|
|
455
563
|
}
|
|
456
564
|
}
|
|
457
|
-
|
|
565
|
+
/**
|
|
566
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
567
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
568
|
+
*/
|
|
569
|
+
hasFreeBrowserSlot() {
|
|
570
|
+
const total = this.startingBrowserControllers.size +
|
|
571
|
+
this.activeBrowserControllers.size +
|
|
572
|
+
this.retiredBrowserControllers.size;
|
|
573
|
+
return total < this.maxOpenBrowsers;
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* Returns `true` if any active browser has room for another page.
|
|
577
|
+
*/
|
|
578
|
+
hasActiveBrowserWithFreeCapacity() {
|
|
579
|
+
for (const controller of this.activeBrowserControllers) {
|
|
580
|
+
if (controller.activePages < this.maxOpenPagesPerBrowser)
|
|
581
|
+
return true;
|
|
582
|
+
}
|
|
583
|
+
return false;
|
|
584
|
+
}
|
|
585
|
+
initializeFingerprinting() {
|
|
458
586
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
459
587
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
460
588
|
this.fingerprintInjector = new FingerprintInjector();
|
|
461
589
|
if (useFingerprintCache) {
|
|
462
590
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
463
591
|
}
|
|
464
|
-
this.
|
|
592
|
+
this.addFingerprintHooks();
|
|
465
593
|
}
|
|
466
|
-
|
|
594
|
+
addFingerprintHooks() {
|
|
467
595
|
this.preLaunchHooks = [
|
|
468
596
|
...this.preLaunchHooks,
|
|
469
597
|
// This is flipped because of the fingerprint cache.
|
|
@@ -474,4 +602,3 @@ export class BrowserPool extends TypedEmitter {
|
|
|
474
602
|
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
475
603
|
}
|
|
476
604
|
}
|
|
477
|
-
//# sourceMappingURL=browser-pool.js.map
|
package/events.d.ts
CHANGED
package/events.js
CHANGED
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -1,29 +1,42 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
3
|
import { getGeneratorDefaultOptions } from './utils.js';
|
|
4
|
+
function applySessionHints(base, fingerprint) {
|
|
5
|
+
if (!fingerprint)
|
|
6
|
+
return base;
|
|
7
|
+
return {
|
|
8
|
+
...base,
|
|
9
|
+
...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
|
|
10
|
+
...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
|
|
11
|
+
...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
/**
|
|
5
15
|
* @internal
|
|
6
16
|
*/
|
|
7
17
|
export function createFingerprintPreLaunchHook(browserPool) {
|
|
8
18
|
const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
|
|
9
19
|
return (_pageId, launchContext) => {
|
|
20
|
+
// Remote browsers may have their own fingerprinting — skip local fingerprint injection
|
|
21
|
+
if (launchContext.isRemote)
|
|
22
|
+
return;
|
|
10
23
|
const { useIncognitoPages } = launchContext;
|
|
11
|
-
const
|
|
24
|
+
const session = launchContext.session;
|
|
25
|
+
const cacheKey = session?.id ?? launchContext.proxyUrl;
|
|
12
26
|
const { launchOptions } = launchContext;
|
|
13
|
-
// If no options are passed we try to pass best default options as possible to match browser and OS.
|
|
14
|
-
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
15
27
|
let fingerprint;
|
|
16
28
|
if (cacheKey && fingerprintCache?.has(cacheKey)) {
|
|
17
29
|
fingerprint = fingerprintCache.get(cacheKey);
|
|
18
30
|
}
|
|
19
|
-
else if (cacheKey) {
|
|
20
|
-
fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
|
|
21
|
-
fingerprintCache?.set(cacheKey, fingerprint);
|
|
22
|
-
}
|
|
23
31
|
else {
|
|
24
|
-
|
|
32
|
+
const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
33
|
+
const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
|
|
34
|
+
fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
|
|
35
|
+
if (cacheKey)
|
|
36
|
+
fingerprintCache?.set(cacheKey, fingerprint);
|
|
25
37
|
}
|
|
26
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
27
40
|
if (useIncognitoPages) {
|
|
28
41
|
return;
|
|
29
42
|
}
|
|
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
41
54
|
export function createPrePageCreateHook() {
|
|
42
55
|
return (_pageId, browserController, pageOptions) => {
|
|
43
56
|
const { launchContext, browserPlugin } = browserController;
|
|
57
|
+
if (launchContext.isRemote)
|
|
58
|
+
return;
|
|
44
59
|
const { fingerprint } = launchContext.fingerprint;
|
|
45
60
|
if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
|
|
46
61
|
pageOptions.userAgent ??= fingerprint.navigator.userAgent;
|
|
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
|
|
|
57
72
|
export function createPostPageCreateHook(fingerprintInjector) {
|
|
58
73
|
return async (page, browserController) => {
|
|
59
74
|
const { browserPlugin, launchContext } = browserController;
|
|
75
|
+
if (launchContext.isRemote)
|
|
76
|
+
return;
|
|
60
77
|
const fingerprint = launchContext.fingerprint;
|
|
61
78
|
// TODO this will require refactoring, we should use common API instead of branching based on plugin type,
|
|
62
79
|
// and there should be no public methods specific to some browser.
|
|
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
|
|
|
79
96
|
}
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
|
-
//# sourceMappingURL=hooks.js.map
|
package/fingerprinting/types.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
import type { LaunchContext } from '../launch-context.js';
|
|
2
2
|
import type { FingerprintGeneratorOptions } from './types.js';
|
|
3
3
|
export declare const getGeneratorDefaultOptions: (launchContext: LaunchContext) => FingerprintGeneratorOptions;
|
|
4
|
-
//# sourceMappingURL=utils.d.ts.map
|
package/fingerprinting/utils.js
CHANGED