@crawlee/browser-pool 4.0.0-rc.0 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/abstract-classes/browser-controller.d.ts +30 -16
- package/abstract-classes/browser-controller.js +50 -3
- package/abstract-classes/browser-plugin.d.ts +7 -9
- package/abstract-classes/browser-plugin.js +8 -4
- package/anonymize-proxy.d.ts +1 -0
- package/browser-pool.d.ts +28 -37
- package/browser-pool.js +171 -112
- package/events.d.ts +0 -1
- package/events.js +0 -1
- package/fingerprinting/types.d.ts +1 -31
- package/fingerprinting/types.js +0 -1
- package/index.d.ts +1 -1
- package/launch-context.d.ts +1 -0
- package/package.json +8 -7
- package/playwright/playwright-browser.d.ts +1 -0
- package/playwright/playwright-browser.js +1 -0
- package/playwright/playwright-controller.d.ts +1 -0
- package/playwright/playwright-controller.js +4 -3
- package/playwright/playwright-plugin.d.ts +2 -1
- package/playwright/playwright-plugin.js +5 -3
- package/puppeteer/puppeteer-controller.d.ts +1 -0
- package/puppeteer/puppeteer-controller.js +13 -11
- package/puppeteer/puppeteer-plugin.d.ts +5 -1
- package/puppeteer/puppeteer-plugin.js +5 -1
- package/remote-browser-pool.d.ts +16 -10
- package/remote-browser-pool.js +19 -5
- package/container-proxy-server.d.ts +0 -11
- package/container-proxy-server.js +0 -42
package/browser-pool.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
-
import {
|
|
2
|
+
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
4
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
4
5
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
5
6
|
import { nanoid } from 'nanoid';
|
|
@@ -78,38 +79,43 @@ const browserPoolOptionsSchema = z.strictObject({
|
|
|
78
79
|
*/
|
|
79
80
|
export class BrowserPool extends TypedEmitter {
|
|
80
81
|
browserPlugins;
|
|
81
|
-
|
|
82
|
+
/** @internal */
|
|
82
83
|
maxOpenBrowsers;
|
|
83
|
-
|
|
84
|
-
operationTimeoutMillis;
|
|
85
|
-
closeInactiveBrowserAfterMillis;
|
|
86
|
-
useFingerprints;
|
|
84
|
+
/** @internal */
|
|
87
85
|
fingerprintOptions;
|
|
88
|
-
|
|
89
|
-
postLaunchHooks;
|
|
90
|
-
prePageCreateHooks;
|
|
91
|
-
postPageCreateHooks;
|
|
92
|
-
prePageCloseHooks;
|
|
93
|
-
postPageCloseHooks;
|
|
94
|
-
pageCounter = 0;
|
|
95
|
-
pages = new Map();
|
|
96
|
-
pageIds = new WeakMap();
|
|
97
|
-
startingBrowserControllers = new Set();
|
|
98
|
-
activeBrowserControllers = new Set();
|
|
99
|
-
retiredBrowserControllers = new Set();
|
|
100
|
-
pageToBrowserController = new WeakMap();
|
|
86
|
+
/** @internal */
|
|
101
87
|
fingerprintInjector;
|
|
102
88
|
fingerprintGenerator;
|
|
89
|
+
/** @internal */
|
|
103
90
|
fingerprintCache;
|
|
104
|
-
|
|
105
|
-
|
|
91
|
+
#maxOpenPagesPerBrowser;
|
|
92
|
+
#retireBrowserAfterPageCount;
|
|
93
|
+
#operationTimeoutMillis;
|
|
94
|
+
#closeInactiveBrowserAfterMillis;
|
|
95
|
+
#preLaunchHooks;
|
|
96
|
+
#postLaunchHooks;
|
|
97
|
+
#prePageCreateHooks;
|
|
98
|
+
#postPageCreateHooks;
|
|
99
|
+
#prePageCloseHooks;
|
|
100
|
+
#postPageCloseHooks;
|
|
101
|
+
#pageCounter = 0;
|
|
102
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- tests observe page tracking and controller retirement directly
|
|
103
|
+
pages = new Map();
|
|
104
|
+
#pageIds = new WeakMap();
|
|
105
|
+
#startingBrowserControllers = new Set();
|
|
106
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
|
|
107
|
+
activeBrowserControllers = new Set();
|
|
108
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
|
|
109
|
+
retiredBrowserControllers = new Set();
|
|
110
|
+
#pageToBrowserController = new WeakMap();
|
|
111
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- tests replace this interval through bracket access
|
|
112
|
+
browserKillerInterval;
|
|
106
113
|
#browserRetireInterval;
|
|
107
114
|
#limiter = pLimit(1);
|
|
108
115
|
#log;
|
|
109
116
|
constructor(options) {
|
|
110
117
|
super();
|
|
111
118
|
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
112
|
-
this.browserKillerInterval.unref();
|
|
113
119
|
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
114
120
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
115
121
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
@@ -121,13 +127,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
121
127
|
}
|
|
122
128
|
}
|
|
123
129
|
this.browserPlugins = browserPlugins;
|
|
124
|
-
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
125
130
|
this.maxOpenBrowsers = Infinity;
|
|
126
|
-
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
127
|
-
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
128
|
-
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
129
|
-
this.useFingerprints = useFingerprints;
|
|
130
131
|
this.fingerprintOptions = fingerprintOptions;
|
|
132
|
+
this.#maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
133
|
+
this.#retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
134
|
+
this.#operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
135
|
+
this.#closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
136
|
+
// Sweeping slower than the window it enforces would round any sub-10s
|
|
137
|
+
// `closeInactiveBrowserAfterSecs` up to the sweep period.
|
|
138
|
+
this.browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), Math.min(BROWSER_KILLER_INTERVAL_MILLIS, this.#closeInactiveBrowserAfterMillis));
|
|
139
|
+
this.browserKillerInterval.unref();
|
|
131
140
|
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
132
141
|
if (controller.activePages === 0 &&
|
|
133
142
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
@@ -136,15 +145,23 @@ export class BrowserPool extends TypedEmitter {
|
|
|
136
145
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
137
146
|
this.#browserRetireInterval.unref();
|
|
138
147
|
// hooks
|
|
139
|
-
this
|
|
140
|
-
this
|
|
141
|
-
this
|
|
142
|
-
this
|
|
143
|
-
this
|
|
144
|
-
this
|
|
148
|
+
this.#preLaunchHooks = preLaunchHooks;
|
|
149
|
+
this.#postLaunchHooks = postLaunchHooks;
|
|
150
|
+
this.#prePageCreateHooks = prePageCreateHooks;
|
|
151
|
+
this.#postPageCreateHooks = postPageCreateHooks;
|
|
152
|
+
this.#prePageCloseHooks = prePageCloseHooks;
|
|
153
|
+
this.#postPageCloseHooks = postPageCloseHooks;
|
|
145
154
|
// fingerprinting
|
|
146
|
-
if (
|
|
147
|
-
this
|
|
155
|
+
if (useFingerprints) {
|
|
156
|
+
this.#initializeFingerprinting();
|
|
157
|
+
// The fingerprint pre-launch hook goes last because of the fingerprint cache.
|
|
158
|
+
// It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
|
|
159
|
+
this.#preLaunchHooks = [...this.#preLaunchHooks, createFingerprintPreLaunchHook(this)];
|
|
160
|
+
this.#prePageCreateHooks = [createPrePageCreateHook(), ...this.#prePageCreateHooks];
|
|
161
|
+
this.#postPageCreateHooks = [
|
|
162
|
+
createPostPageCreateHook(this.fingerprintInjector),
|
|
163
|
+
...this.#postPageCreateHooks,
|
|
164
|
+
];
|
|
148
165
|
}
|
|
149
166
|
}
|
|
150
167
|
/**
|
|
@@ -165,7 +182,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
165
182
|
* crawler's responsibility.
|
|
166
183
|
*/
|
|
167
184
|
async newPage(options = {}) {
|
|
168
|
-
const { id = nanoid(), pageOptions, browserPlugin = this
|
|
185
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.#pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
169
186
|
if (this.pages.has(id)) {
|
|
170
187
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
171
188
|
}
|
|
@@ -184,15 +201,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
184
201
|
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
185
202
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
186
203
|
return this.#limiter(AsyncResource.bind(async () => {
|
|
187
|
-
let browserController = this
|
|
204
|
+
let browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
188
205
|
if (!browserController)
|
|
189
|
-
browserController = await this
|
|
206
|
+
browserController = await this.#launchBrowser(id, {
|
|
190
207
|
browserPlugin,
|
|
191
208
|
proxyUrl,
|
|
192
209
|
ignoreTlsErrors,
|
|
193
210
|
});
|
|
194
211
|
tryCancel();
|
|
195
|
-
return await this
|
|
212
|
+
return await this.#createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
196
213
|
}));
|
|
197
214
|
}
|
|
198
215
|
/**
|
|
@@ -201,13 +218,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
201
218
|
* configure the new browser.
|
|
202
219
|
*/
|
|
203
220
|
async newPageInNewBrowser(options = {}) {
|
|
204
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this
|
|
221
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.#pickBrowserPlugin() } = options;
|
|
205
222
|
if (this.pages.has(id)) {
|
|
206
223
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
207
224
|
}
|
|
208
|
-
const browserController = await this
|
|
225
|
+
const browserController = await this.#launchBrowser(id, { launchOptions, browserPlugin });
|
|
209
226
|
tryCancel();
|
|
210
|
-
return await this
|
|
227
|
+
return await this.#createPageForBrowser(id, browserController, pageOptions);
|
|
211
228
|
}
|
|
212
229
|
/**
|
|
213
230
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -253,7 +270,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
253
270
|
* @param page - Browser plugin page
|
|
254
271
|
*/
|
|
255
272
|
getBrowserControllerByPage(page) {
|
|
256
|
-
return this
|
|
273
|
+
return this.#pageToBrowserController.get(page);
|
|
257
274
|
}
|
|
258
275
|
/**
|
|
259
276
|
* If you provided a custom ID to one of your pages or saved the
|
|
@@ -271,13 +288,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
271
288
|
* until it's closed.
|
|
272
289
|
*/
|
|
273
290
|
getPageId(page) {
|
|
274
|
-
return this
|
|
291
|
+
return this.#pageIds.get(page);
|
|
275
292
|
}
|
|
276
|
-
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
293
|
+
async #createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
277
294
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
278
295
|
// It's not ideal though, we need to come up with a better API.
|
|
279
|
-
|
|
280
|
-
await browserController['isActivePromise'];
|
|
296
|
+
await browserController.waitForActive();
|
|
281
297
|
tryCancel();
|
|
282
298
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
283
299
|
if (finalPageOptions) {
|
|
@@ -289,26 +305,26 @@ export class BrowserPool extends TypedEmitter {
|
|
|
289
305
|
});
|
|
290
306
|
}
|
|
291
307
|
}
|
|
292
|
-
await this
|
|
308
|
+
await this.#executeHooks(this.#prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
293
309
|
tryCancel();
|
|
294
310
|
let page;
|
|
295
311
|
try {
|
|
296
|
-
page = (await addTimeoutToPromise(async () => browserController.newPage(finalPageOptions), this
|
|
312
|
+
page = (await addTimeoutToPromise(async () => browserController.newPage(finalPageOptions), this.#operationTimeoutMillis, 'browserController.newPage() timed out.'));
|
|
297
313
|
tryCancel();
|
|
298
314
|
this.pages.set(pageId, page);
|
|
299
|
-
this
|
|
300
|
-
this
|
|
315
|
+
this.#pageIds.set(page, pageId);
|
|
316
|
+
this.#pageToBrowserController.set(page, browserController);
|
|
301
317
|
// if you synchronously trigger a lot of page launches, browser will not get retired soon enough. Not sure if it's a problem, let's monitor it.
|
|
302
|
-
if (browserController.totalPages >= this
|
|
318
|
+
if (browserController.totalPages >= this.#retireBrowserAfterPageCount) {
|
|
303
319
|
this.retireBrowserController(browserController);
|
|
304
320
|
}
|
|
305
|
-
this
|
|
321
|
+
this.#overridePageClose(page);
|
|
306
322
|
}
|
|
307
323
|
catch (err) {
|
|
308
324
|
this.retireBrowserController(browserController);
|
|
309
325
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
310
326
|
}
|
|
311
|
-
await this
|
|
327
|
+
await this.#executeHooks(this.#postPageCreateHooks, page, browserController);
|
|
312
328
|
tryCancel();
|
|
313
329
|
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
314
330
|
return page;
|
|
@@ -319,14 +335,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
319
335
|
*
|
|
320
336
|
*/
|
|
321
337
|
retireBrowserController(browserController) {
|
|
322
|
-
const isStarting = this
|
|
338
|
+
const isStarting = this.#startingBrowserControllers.has(browserController);
|
|
323
339
|
const isActive = this.activeBrowserControllers.has(browserController);
|
|
324
340
|
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
325
341
|
if (hasBeenRetiredOrKilled)
|
|
326
342
|
return;
|
|
327
343
|
this.retiredBrowserControllers.add(browserController);
|
|
328
344
|
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
329
|
-
this
|
|
345
|
+
this.#startingBrowserControllers.delete(browserController);
|
|
330
346
|
this.activeBrowserControllers.delete(browserController);
|
|
331
347
|
}
|
|
332
348
|
/**
|
|
@@ -340,8 +356,9 @@ export class BrowserPool extends TypedEmitter {
|
|
|
340
356
|
}
|
|
341
357
|
/**
|
|
342
358
|
* Releases a page back to the pool. The page is closed and, if the
|
|
343
|
-
* optional `error` is a {@link SessionError}
|
|
344
|
-
*
|
|
359
|
+
* optional `error` is a {@link SessionError} (including
|
|
360
|
+
* {@link SessionRetiredError}), the browser controller that served
|
|
361
|
+
* the page is retired so that the finished session's state (cookies,
|
|
345
362
|
* storage, etc.) cannot leak into future sessions.
|
|
346
363
|
*
|
|
347
364
|
* This is the primary way the crawler should return pages to the pool.
|
|
@@ -353,8 +370,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
353
370
|
if (options?.error instanceof SessionError) {
|
|
354
371
|
this.retireBrowserByPage(page);
|
|
355
372
|
}
|
|
356
|
-
//
|
|
357
|
-
|
|
373
|
+
// The bound on this lives in `overridePageClose`, where it covers the pool's own
|
|
374
|
+
// bookkeeping as well. Wrapping it here instead would time out the override and abandon
|
|
375
|
+
// that bookkeeping, leaving the browser holding a slot it can never get back.
|
|
376
|
+
await page.close();
|
|
358
377
|
}
|
|
359
378
|
/**
|
|
360
379
|
* Extracts the relevant state (currently just cookies) from a page via its
|
|
@@ -396,7 +415,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
396
415
|
* closed after all their pages are closed.
|
|
397
416
|
*/
|
|
398
417
|
retireAllBrowsers() {
|
|
399
|
-
[...this
|
|
418
|
+
[...this.#startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
400
419
|
this.retireBrowserController(controller);
|
|
401
420
|
});
|
|
402
421
|
}
|
|
@@ -405,40 +424,48 @@ export class BrowserPool extends TypedEmitter {
|
|
|
405
424
|
* @return {Promise<void>}
|
|
406
425
|
*/
|
|
407
426
|
async closeAllBrowsers() {
|
|
408
|
-
const controllers = this
|
|
427
|
+
const controllers = this.#getAllBrowserControllers();
|
|
409
428
|
const promises = [...controllers]
|
|
410
429
|
.filter((controller) => controller.isActive)
|
|
411
430
|
.map(async (controller) => controller.close());
|
|
412
431
|
await Promise.all(promises);
|
|
413
432
|
}
|
|
433
|
+
async [Symbol.asyncDispose]() {
|
|
434
|
+
await this.destroy();
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
|
|
438
|
+
* browsers when a run ends and may start another run on the same pool.
|
|
439
|
+
*/
|
|
440
|
+
async releaseAllBrowsers() {
|
|
441
|
+
await this.closeAllBrowsers();
|
|
442
|
+
this.#startingBrowserControllers.clear();
|
|
443
|
+
this.activeBrowserControllers.clear();
|
|
444
|
+
this.retiredBrowserControllers.clear();
|
|
445
|
+
}
|
|
414
446
|
/**
|
|
415
|
-
* Closes all managed browsers and tears down
|
|
447
|
+
* Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
|
|
448
|
+
* re-armed and its listeners are dropped, so it cannot be used again.
|
|
416
449
|
*/
|
|
417
450
|
async destroy() {
|
|
418
451
|
clearInterval(this.browserKillerInterval);
|
|
419
452
|
clearInterval(this.#browserRetireInterval);
|
|
420
453
|
this.browserKillerInterval = undefined;
|
|
421
454
|
this.#browserRetireInterval = undefined;
|
|
422
|
-
await this.
|
|
423
|
-
this.teardown();
|
|
424
|
-
}
|
|
425
|
-
teardown() {
|
|
426
|
-
this.startingBrowserControllers.clear();
|
|
427
|
-
this.activeBrowserControllers.clear();
|
|
428
|
-
this.retiredBrowserControllers.clear();
|
|
455
|
+
await this.releaseAllBrowsers();
|
|
429
456
|
this.removeAllListeners();
|
|
430
457
|
}
|
|
431
|
-
getAllBrowserControllers() {
|
|
458
|
+
#getAllBrowserControllers() {
|
|
432
459
|
return new Set([
|
|
433
|
-
...this
|
|
460
|
+
...this.#startingBrowserControllers,
|
|
434
461
|
...this.activeBrowserControllers,
|
|
435
462
|
...this.retiredBrowserControllers,
|
|
436
463
|
]);
|
|
437
464
|
}
|
|
438
|
-
async launchBrowser(pageId, options) {
|
|
465
|
+
async #launchBrowser(pageId, options) {
|
|
439
466
|
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
440
467
|
const browserController = browserPlugin.createController();
|
|
441
|
-
this
|
|
468
|
+
this.#startingBrowserControllers.add(browserController);
|
|
442
469
|
const launchContext = browserPlugin.createLaunchContext({
|
|
443
470
|
id: pageId,
|
|
444
471
|
launchOptions,
|
|
@@ -456,14 +483,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
456
483
|
try {
|
|
457
484
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
458
485
|
// because otherwise it would be stuck in limbo without a browser.
|
|
459
|
-
await this
|
|
486
|
+
await this.#executeHooks(this.#preLaunchHooks, pageId, launchContext);
|
|
460
487
|
tryCancel();
|
|
461
488
|
const browser = await browserPlugin.launch(launchContext);
|
|
462
489
|
tryCancel();
|
|
463
490
|
browserController.assignBrowser(browser, launchContext);
|
|
464
491
|
}
|
|
465
492
|
catch (err) {
|
|
466
|
-
this
|
|
493
|
+
this.#startingBrowserControllers.delete(browserController);
|
|
467
494
|
throw err;
|
|
468
495
|
}
|
|
469
496
|
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
@@ -471,10 +498,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
471
498
|
try {
|
|
472
499
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
473
500
|
// both the controller and the browser before throwing.
|
|
474
|
-
await this
|
|
501
|
+
await this.#executeHooks(this.#postLaunchHooks, pageId, browserController);
|
|
475
502
|
}
|
|
476
503
|
catch (err) {
|
|
477
|
-
this
|
|
504
|
+
this.#startingBrowserControllers.delete(browserController);
|
|
478
505
|
browserController.close().catch((closeErr) => {
|
|
479
506
|
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
480
507
|
id: browserController.id,
|
|
@@ -484,23 +511,22 @@ export class BrowserPool extends TypedEmitter {
|
|
|
484
511
|
}
|
|
485
512
|
tryCancel();
|
|
486
513
|
browserController.activate();
|
|
487
|
-
this
|
|
514
|
+
this.#startingBrowserControllers.delete(browserController);
|
|
488
515
|
this.activeBrowserControllers.add(browserController);
|
|
489
516
|
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
490
517
|
return browserController;
|
|
491
518
|
}
|
|
492
519
|
/**
|
|
493
520
|
* Picks plugins round robin.
|
|
494
|
-
* @private
|
|
495
521
|
*/
|
|
496
|
-
pickBrowserPlugin() {
|
|
497
|
-
const pluginIndex = this
|
|
498
|
-
this
|
|
522
|
+
#pickBrowserPlugin() {
|
|
523
|
+
const pluginIndex = this.#pageCounter % this.browserPlugins.length;
|
|
524
|
+
this.#pageCounter++;
|
|
499
525
|
return this.browserPlugins[pluginIndex];
|
|
500
526
|
}
|
|
501
|
-
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
527
|
+
#pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
502
528
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
503
|
-
const hasCapacity = controller.activePages < this
|
|
529
|
+
const hasCapacity = controller.activePages < this.#maxOpenPagesPerBrowser;
|
|
504
530
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
505
531
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
506
532
|
return (isCorrectPlugin &&
|
|
@@ -510,11 +536,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
510
536
|
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
511
537
|
});
|
|
512
538
|
}
|
|
539
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
|
|
513
540
|
async closeInactiveRetiredBrowsers() {
|
|
514
541
|
const closedBrowserIds = [];
|
|
515
542
|
for (const controller of this.retiredBrowserControllers) {
|
|
516
543
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
517
|
-
const isBrowserIdle = millisSinceLastPageOpened >= this
|
|
544
|
+
const isBrowserIdle = millisSinceLastPageOpened >= this.#closeInactiveBrowserAfterMillis;
|
|
518
545
|
const isBrowserEmpty = controller.activePages === 0;
|
|
519
546
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
520
547
|
const { id } = controller;
|
|
@@ -531,27 +558,61 @@ export class BrowserPool extends TypedEmitter {
|
|
|
531
558
|
});
|
|
532
559
|
}
|
|
533
560
|
}
|
|
534
|
-
overridePageClose(page) {
|
|
561
|
+
#overridePageClose(page) {
|
|
535
562
|
const originalPageClose = page.close;
|
|
536
|
-
const browserController = this
|
|
563
|
+
const browserController = this.#pageToBrowserController.get(page);
|
|
537
564
|
const pageId = this.getPageId(page);
|
|
538
565
|
page.close = async (...args) => {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
566
|
+
// A browser can acknowledge the close and then never destroy the target, in which case
|
|
567
|
+
// this never settles and the page's own `close` event never fires either (Chromium
|
|
568
|
+
// 536385539). Bound the whole sequence so nothing here can hang the caller, then
|
|
569
|
+
// reconcile the pool regardless of the outcome, so a page that refuses to close cannot
|
|
570
|
+
// leave the pool believing it is still open. `Promise.race` rather than
|
|
571
|
+
// `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
|
|
572
|
+
// so a timeout here would cancel the task of whoever awaited `page.close()`.
|
|
573
|
+
let pageClosed = false;
|
|
574
|
+
const closing = (async () => {
|
|
575
|
+
await this.#executeHooks(this.#prePageCloseHooks, page, browserController);
|
|
576
|
+
await originalPageClose.apply(page, args).catch((err) => {
|
|
577
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
578
|
+
});
|
|
579
|
+
// Whether the page itself is gone. Tracked separately from the sequence finishing,
|
|
580
|
+
// so that a slow hook does not get the browser retired.
|
|
581
|
+
pageClosed = true;
|
|
582
|
+
await this.#executeHooks(this.#postPageCloseHooks, pageId, browserController);
|
|
583
|
+
})();
|
|
584
|
+
let timeout;
|
|
585
|
+
const finished = await Promise.race([
|
|
586
|
+
closing.then(() => true, (err) => {
|
|
587
|
+
this.#log.warning(`Closing a page failed, releasing it from the pool anyway.\nCause:${err.message}`, { id: browserController.id, pageId });
|
|
588
|
+
return true;
|
|
589
|
+
}),
|
|
590
|
+
new Promise((resolve) => {
|
|
591
|
+
timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
|
|
592
|
+
}),
|
|
593
|
+
]);
|
|
594
|
+
clearTimeout(timeout);
|
|
595
|
+
if (!finished) {
|
|
596
|
+
this.#log.warning(`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
|
|
597
|
+
'releasing it from the pool anyway.', { id: browserController.id, pageId });
|
|
598
|
+
}
|
|
599
|
+
if (!pageClosed) {
|
|
600
|
+
// The page is still attached, so this browser cannot be trusted with more work.
|
|
601
|
+
// Retiring it lets the reclamation below close the process once its pages drain.
|
|
602
|
+
this.retireBrowserController(browserController);
|
|
603
|
+
}
|
|
604
|
+
browserController.registerPageClosed(page);
|
|
544
605
|
this.pages.delete(pageId);
|
|
545
|
-
this
|
|
606
|
+
this.#closeRetiredBrowserWithNoPages(browserController);
|
|
546
607
|
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
547
608
|
};
|
|
548
609
|
}
|
|
549
|
-
async executeHooks(hooks, ...args) {
|
|
610
|
+
async #executeHooks(hooks, ...args) {
|
|
550
611
|
for (const hook of hooks) {
|
|
551
612
|
await hook(...args);
|
|
552
613
|
}
|
|
553
614
|
}
|
|
554
|
-
closeRetiredBrowserWithNoPages(browserController) {
|
|
615
|
+
#closeRetiredBrowserWithNoPages(browserController) {
|
|
555
616
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
556
617
|
// Run this with a delay, otherwise page.close()
|
|
557
618
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
@@ -564,42 +625,40 @@ export class BrowserPool extends TypedEmitter {
|
|
|
564
625
|
}
|
|
565
626
|
}
|
|
566
627
|
/**
|
|
567
|
-
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
568
|
-
*
|
|
628
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
|
|
629
|
+
* Counts starting, active, and retired browsers.
|
|
630
|
+
*
|
|
631
|
+
* A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
|
|
632
|
+
* has set a cap — {@link RemoteBrowserPool} does, from its own
|
|
633
|
+
* {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
|
|
634
|
+
* `BrowserPoolOptions` key for it.
|
|
635
|
+
*
|
|
636
|
+
* @internal
|
|
569
637
|
*/
|
|
570
638
|
hasFreeBrowserSlot() {
|
|
571
|
-
const total = this
|
|
639
|
+
const total = this.#startingBrowserControllers.size +
|
|
572
640
|
this.activeBrowserControllers.size +
|
|
573
641
|
this.retiredBrowserControllers.size;
|
|
574
642
|
return total < this.maxOpenBrowsers;
|
|
575
643
|
}
|
|
576
644
|
/**
|
|
577
645
|
* Returns `true` if any active browser has room for another page.
|
|
646
|
+
*
|
|
647
|
+
* @internal
|
|
578
648
|
*/
|
|
579
649
|
hasActiveBrowserWithFreeCapacity() {
|
|
580
650
|
for (const controller of this.activeBrowserControllers) {
|
|
581
|
-
if (controller.activePages < this
|
|
651
|
+
if (controller.activePages < this.#maxOpenPagesPerBrowser)
|
|
582
652
|
return true;
|
|
583
653
|
}
|
|
584
654
|
return false;
|
|
585
655
|
}
|
|
586
|
-
initializeFingerprinting() {
|
|
656
|
+
#initializeFingerprinting() {
|
|
587
657
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
588
658
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
589
659
|
this.fingerprintInjector = new FingerprintInjector();
|
|
590
660
|
if (useFingerprintCache) {
|
|
591
661
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
592
662
|
}
|
|
593
|
-
this.addFingerprintHooks();
|
|
594
|
-
}
|
|
595
|
-
addFingerprintHooks() {
|
|
596
|
-
this.preLaunchHooks = [
|
|
597
|
-
...this.preLaunchHooks,
|
|
598
|
-
// This is flipped because of the fingerprint cache.
|
|
599
|
-
// It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
|
|
600
|
-
createFingerprintPreLaunchHook(this),
|
|
601
|
-
];
|
|
602
|
-
this.prePageCreateHooks = [createPrePageCreateHook(), ...this.prePageCreateHooks];
|
|
603
|
-
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
604
663
|
}
|
|
605
664
|
}
|
package/events.d.ts
CHANGED
package/events.js
CHANGED
|
@@ -2,7 +2,6 @@ export var BROWSER_POOL_EVENTS;
|
|
|
2
2
|
(function (BROWSER_POOL_EVENTS) {
|
|
3
3
|
BROWSER_POOL_EVENTS["BROWSER_LAUNCHED"] = "browserLaunched";
|
|
4
4
|
BROWSER_POOL_EVENTS["BROWSER_RETIRED"] = "browserRetired";
|
|
5
|
-
BROWSER_POOL_EVENTS["BROWSER_CLOSED"] = "browserClosed";
|
|
6
5
|
BROWSER_POOL_EVENTS["PAGE_CREATED"] = "pageCreated";
|
|
7
6
|
BROWSER_POOL_EVENTS["PAGE_CLOSED"] = "pageClosed";
|
|
8
7
|
})(BROWSER_POOL_EVENTS || (BROWSER_POOL_EVENTS = {}));
|
|
@@ -1,41 +1,12 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export interface FingerprintGenerator {
|
|
3
|
-
getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
|
|
4
|
-
}
|
|
5
|
-
export interface GetFingerprintReturn {
|
|
6
|
-
fingerprint: Fingerprint;
|
|
7
|
-
}
|
|
1
|
+
import type { FingerprintGeneratorOptions as FingerprintOptionsOriginal } from 'fingerprint-generator';
|
|
8
2
|
export interface FingerprintGeneratorOptions extends Partial<FingerprintOptionsOriginal> {
|
|
9
3
|
}
|
|
10
|
-
declare const SUPPORTED_HTTP_VERSIONS: readonly ["1", "2"];
|
|
11
|
-
/**
|
|
12
|
-
* String specifying the HTTP version to use.
|
|
13
|
-
*/
|
|
14
|
-
type HttpVersion = (typeof SUPPORTED_HTTP_VERSIONS)[number];
|
|
15
4
|
export declare enum BrowserName {
|
|
16
5
|
chrome = "chrome",
|
|
17
6
|
firefox = "firefox",
|
|
18
7
|
safari = "safari",
|
|
19
8
|
edge = "edge"
|
|
20
9
|
}
|
|
21
|
-
export interface BrowserSpecification {
|
|
22
|
-
/**
|
|
23
|
-
* String representing the browser name.
|
|
24
|
-
*/
|
|
25
|
-
name: BrowserName;
|
|
26
|
-
/**
|
|
27
|
-
* Minimum version of browser used.
|
|
28
|
-
*/
|
|
29
|
-
minVersion?: number;
|
|
30
|
-
/**
|
|
31
|
-
* Maximum version of browser used.
|
|
32
|
-
*/
|
|
33
|
-
maxVersion?: number;
|
|
34
|
-
/**
|
|
35
|
-
* HTTP version to be used for header generation (the headers differ depending on the version).
|
|
36
|
-
*/
|
|
37
|
-
httpVersion?: HttpVersion;
|
|
38
|
-
}
|
|
39
10
|
export declare enum OperatingSystemsName {
|
|
40
11
|
linux = "linux",
|
|
41
12
|
macos = "macos",
|
|
@@ -60,4 +31,3 @@ export declare enum DeviceCategory {
|
|
|
60
31
|
*/
|
|
61
32
|
desktop = "desktop"
|
|
62
33
|
}
|
|
63
|
-
export {};
|
package/fingerprinting/types.js
CHANGED
package/index.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export * from './browser-pool.js';
|
|
|
28
28
|
export * from './playwright/playwright-plugin.js';
|
|
29
29
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
30
30
|
export * from './events.js';
|
|
31
|
-
export type {
|
|
31
|
+
export type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
|
|
32
32
|
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
33
33
|
export type { BrowserControllerEvents, IBrowserController, IBrowserLaunchContext, } from './abstract-classes/browser-controller.js';
|
|
34
34
|
export { BrowserController } from './abstract-classes/browser-controller.js';
|
package/launch-context.d.ts
CHANGED
|
@@ -52,6 +52,7 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
52
52
|
* Whether this launch context represents a connection to a remote browser
|
|
53
53
|
* rather than a locally launched one.
|
|
54
54
|
* @default false
|
|
55
|
+
* @internal
|
|
55
56
|
*/
|
|
56
57
|
isRemote?: boolean;
|
|
57
58
|
}
|