@crawlee/browser-pool 4.0.0-beta.98 → 4.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstract-classes/browser-controller.d.ts +1 -3
- package/abstract-classes/browser-controller.js +13 -11
- package/abstract-classes/browser-plugin.d.ts +5 -7
- package/abstract-classes/browser-plugin.js +7 -7
- package/browser-pool.d.ts +13 -15
- package/browser-pool.js +75 -70
- package/events.d.ts +2 -2
- package/fingerprinting/hooks.js +2 -1
- package/fingerprinting/types.d.ts +2 -2
- package/fingerprinting/utils.js +5 -5
- package/launch-context.d.ts +2 -3
- package/launch-context.js +11 -8
- package/package.json +6 -6
- package/playwright/playwright-browser.d.ts +2 -5
- package/playwright/playwright-browser.js +16 -16
- package/playwright/playwright-plugin.d.ts +4 -4
- package/playwright/playwright-plugin.js +14 -12
- package/puppeteer/puppeteer-plugin.d.ts +2 -2
- package/puppeteer/puppeteer-plugin.js +4 -4
- package/remote-browser-pool.d.ts +7 -15
- package/remote-browser-pool.js +48 -47
|
@@ -80,6 +80,7 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
|
|
|
80
80
|
* @hideconstructor
|
|
81
81
|
*/
|
|
82
82
|
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
|
|
83
|
+
#private;
|
|
83
84
|
readonly id: string;
|
|
84
85
|
protected readonly log: CrawleeLogger;
|
|
85
86
|
/**
|
|
@@ -103,10 +104,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
|
|
|
103
104
|
activePages: number;
|
|
104
105
|
totalPages: number;
|
|
105
106
|
lastPageOpenedAt: number;
|
|
106
|
-
private _activate;
|
|
107
107
|
private isActivePromise;
|
|
108
|
-
private commitBrowser;
|
|
109
|
-
private hasBrowserPromise;
|
|
110
108
|
constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
111
109
|
/**
|
|
112
110
|
* Activates the BrowserController. If you try to open new pages before
|
|
@@ -2,6 +2,7 @@ import { serviceLocator } from '@crawlee/core';
|
|
|
2
2
|
import { nanoid } from 'nanoid';
|
|
3
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
4
4
|
import { tryCancel } from '@apify/timeout';
|
|
5
|
+
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
5
6
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
7
|
/**
|
|
7
8
|
* The `BrowserController` serves two purposes. First, it is the base class that
|
|
@@ -35,13 +36,14 @@ export class BrowserController extends TypedEmitter {
|
|
|
35
36
|
activePages = 0;
|
|
36
37
|
totalPages = 0;
|
|
37
38
|
lastPageOpenedAt = Date.now();
|
|
38
|
-
|
|
39
|
+
#activate;
|
|
40
|
+
// kept as TS-private: `BrowserPool` awaits it through cross-object bracket access
|
|
39
41
|
isActivePromise = new Promise((resolve) => {
|
|
40
|
-
this
|
|
42
|
+
this.#activate = resolve;
|
|
41
43
|
});
|
|
42
|
-
commitBrowser;
|
|
43
|
-
hasBrowserPromise = new Promise((resolve) => {
|
|
44
|
-
this
|
|
44
|
+
#commitBrowser;
|
|
45
|
+
#hasBrowserPromise = new Promise((resolve) => {
|
|
46
|
+
this.#commitBrowser = resolve;
|
|
45
47
|
});
|
|
46
48
|
constructor(browserPlugin) {
|
|
47
49
|
super();
|
|
@@ -58,7 +60,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
58
60
|
if (!this.browser) {
|
|
59
61
|
throw new Error('Cannot activate BrowserController without an assigned browser.');
|
|
60
62
|
}
|
|
61
|
-
this
|
|
63
|
+
this.#activate();
|
|
62
64
|
this.isActive = true;
|
|
63
65
|
}
|
|
64
66
|
/**
|
|
@@ -70,7 +72,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
70
72
|
}
|
|
71
73
|
this.browser = browser;
|
|
72
74
|
this.launchContext = launchContext;
|
|
73
|
-
this
|
|
75
|
+
this.#commitBrowser();
|
|
74
76
|
}
|
|
75
77
|
/**
|
|
76
78
|
* Gracefully closes the browser and makes sure
|
|
@@ -79,7 +81,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
79
81
|
* Emits 'browserClosed' event.
|
|
80
82
|
*/
|
|
81
83
|
async close() {
|
|
82
|
-
await this
|
|
84
|
+
await this.#hasBrowserPromise;
|
|
83
85
|
try {
|
|
84
86
|
await this._close();
|
|
85
87
|
// TODO: shouldn't this go in a finally instead?
|
|
@@ -88,7 +90,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
88
90
|
catch (error) {
|
|
89
91
|
this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
90
92
|
}
|
|
91
|
-
this.emit(
|
|
93
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
92
94
|
const killTimer = setTimeout(() => {
|
|
93
95
|
this._kill().catch((err) => {
|
|
94
96
|
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
@@ -102,9 +104,9 @@ export class BrowserController extends TypedEmitter {
|
|
|
102
104
|
* Emits 'browserClosed' event.
|
|
103
105
|
*/
|
|
104
106
|
async kill() {
|
|
105
|
-
await this
|
|
107
|
+
await this.#hasBrowserPromise;
|
|
106
108
|
await this._kill();
|
|
107
|
-
this.emit(
|
|
109
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
108
110
|
}
|
|
109
111
|
/**
|
|
110
112
|
* Opens new browser page.
|
|
@@ -28,11 +28,9 @@ export interface CommonLibrary {
|
|
|
28
28
|
launch(opts?: Dictionary): Promise<CommonBrowser>;
|
|
29
29
|
name?: () => string;
|
|
30
30
|
}
|
|
31
|
-
/** @internal */
|
|
32
31
|
export interface CommonBrowser {
|
|
33
32
|
newPage(...args: unknown[]): Promise<CommonPage>;
|
|
34
33
|
}
|
|
35
|
-
/** @internal */
|
|
36
34
|
export interface CommonPage {
|
|
37
35
|
close(...args: unknown[]): Promise<unknown>;
|
|
38
36
|
url(): string | Promise<string>;
|
|
@@ -121,7 +119,7 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
121
119
|
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
122
120
|
* lives here so it stays identical across plugins.
|
|
123
121
|
*/
|
|
124
|
-
protected
|
|
122
|
+
protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
|
|
125
123
|
/**
|
|
126
124
|
* Creates a `LaunchContext` with all the information needed
|
|
127
125
|
* to launch a browser. Aside from library specific launch options,
|
|
@@ -134,13 +132,13 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
134
132
|
* Launches the browser using provided launch context.
|
|
135
133
|
*/
|
|
136
134
|
launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
|
137
|
-
private
|
|
138
|
-
protected
|
|
135
|
+
private mergeArgsToHideWebdriver;
|
|
136
|
+
protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
|
|
139
137
|
/**
|
|
140
138
|
* @private
|
|
141
139
|
*/
|
|
142
|
-
protected abstract
|
|
143
|
-
protected abstract
|
|
140
|
+
protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
|
|
141
|
+
protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
|
|
144
142
|
/**
|
|
145
143
|
* @private
|
|
146
144
|
*/
|
|
@@ -69,7 +69,7 @@ export class BrowserPlugin {
|
|
|
69
69
|
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
70
70
|
* lives here so it stays identical across plugins.
|
|
71
71
|
*/
|
|
72
|
-
async
|
|
72
|
+
async connectToRemoteBrowser(launchContext, connect) {
|
|
73
73
|
const connection = this.remoteConnection;
|
|
74
74
|
let url;
|
|
75
75
|
let token;
|
|
@@ -79,7 +79,7 @@ export class BrowserPlugin {
|
|
|
79
79
|
catch (cause) {
|
|
80
80
|
throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
|
|
81
81
|
}
|
|
82
|
-
launchContext.
|
|
82
|
+
launchContext.remoteToken = token;
|
|
83
83
|
try {
|
|
84
84
|
return await connect(url);
|
|
85
85
|
}
|
|
@@ -117,11 +117,11 @@ export class BrowserPlugin {
|
|
|
117
117
|
launchContext.launchOptions ??= {};
|
|
118
118
|
const { proxyUrl, launchOptions } = launchContext;
|
|
119
119
|
if (proxyUrl && !launchContext.isRemote) {
|
|
120
|
-
await this.
|
|
120
|
+
await this.addProxyToLaunchOptions(launchContext);
|
|
121
121
|
}
|
|
122
|
-
if (!launchContext.isRemote && this.
|
|
122
|
+
if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
|
|
123
123
|
// This will set the args for chromium based browsers to hide the webdriver.
|
|
124
|
-
launchOptions.args = this.
|
|
124
|
+
launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
|
|
125
125
|
// When User-Agent is not set, and we're using Chromium in headless mode,
|
|
126
126
|
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
|
|
127
127
|
// as otherwise 'HeadlessChrome' is present in User-Agent string.
|
|
@@ -135,7 +135,7 @@ export class BrowserPlugin {
|
|
|
135
135
|
}
|
|
136
136
|
return this._launch(launchContext);
|
|
137
137
|
}
|
|
138
|
-
|
|
138
|
+
mergeArgsToHideWebdriver(originalArgs) {
|
|
139
139
|
if (!originalArgs?.length) {
|
|
140
140
|
return ['--disable-blink-features=AutomationControlled'];
|
|
141
141
|
}
|
|
@@ -148,7 +148,7 @@ export class BrowserPlugin {
|
|
|
148
148
|
}
|
|
149
149
|
return originalArgs;
|
|
150
150
|
}
|
|
151
|
-
|
|
151
|
+
throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
|
|
152
152
|
const errorMessage = ['Failed to launch browser. Please check the following:'];
|
|
153
153
|
if (executablePath) {
|
|
154
154
|
errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
|
package/browser-pool.d.ts
CHANGED
|
@@ -248,6 +248,7 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
|
|
|
248
248
|
* ```
|
|
249
249
|
*/
|
|
250
250
|
export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
|
|
251
|
+
#private;
|
|
251
252
|
browserPlugins: BrowserPlugins;
|
|
252
253
|
maxOpenPagesPerBrowser: number;
|
|
253
254
|
maxOpenBrowsers: number;
|
|
@@ -273,9 +274,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
273
274
|
fingerprintGenerator?: FingerprintGenerator;
|
|
274
275
|
fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
|
|
275
276
|
private browserKillerInterval?;
|
|
276
|
-
private browserRetireInterval?;
|
|
277
|
-
private limiter;
|
|
278
|
-
private log;
|
|
279
277
|
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
|
280
278
|
/**
|
|
281
279
|
* Opens a new page in one of the running browsers or launches
|
|
@@ -350,7 +348,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
350
348
|
* until it's closed.
|
|
351
349
|
*/
|
|
352
350
|
getPageId(page: PageReturn): string | undefined;
|
|
353
|
-
private
|
|
351
|
+
private createPageForBrowser;
|
|
354
352
|
/**
|
|
355
353
|
* Removes a browser controller from the pool. The underlying
|
|
356
354
|
* browser will be closed after all its pages are closed.
|
|
@@ -413,19 +411,19 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
413
411
|
* Closes all managed browsers and tears down the pool.
|
|
414
412
|
*/
|
|
415
413
|
destroy(): Promise<void>;
|
|
416
|
-
private
|
|
417
|
-
private
|
|
418
|
-
private
|
|
414
|
+
private teardown;
|
|
415
|
+
private getAllBrowserControllers;
|
|
416
|
+
private launchBrowser;
|
|
419
417
|
/**
|
|
420
418
|
* Picks plugins round robin.
|
|
421
419
|
* @private
|
|
422
420
|
*/
|
|
423
|
-
private
|
|
424
|
-
private
|
|
425
|
-
private
|
|
426
|
-
private
|
|
427
|
-
private
|
|
428
|
-
private
|
|
421
|
+
private pickBrowserPlugin;
|
|
422
|
+
private pickBrowserWithFreeCapacity;
|
|
423
|
+
private closeInactiveRetiredBrowsers;
|
|
424
|
+
private overridePageClose;
|
|
425
|
+
private executeHooks;
|
|
426
|
+
private closeRetiredBrowserWithNoPages;
|
|
429
427
|
/**
|
|
430
428
|
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
431
429
|
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
@@ -435,8 +433,8 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
435
433
|
* Returns `true` if any active browser has room for another page.
|
|
436
434
|
*/
|
|
437
435
|
hasActiveBrowserWithFreeCapacity(): boolean;
|
|
438
|
-
private
|
|
439
|
-
private
|
|
436
|
+
private initializeFingerprinting;
|
|
437
|
+
private addFingerprintHooks;
|
|
440
438
|
}
|
|
441
439
|
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
|
|
442
440
|
/**
|
package/browser-pool.js
CHANGED
|
@@ -1,17 +1,34 @@
|
|
|
1
1
|
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
-
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
2
|
+
import { parseArgument, schemas, SessionError, serviceLocator } from '@crawlee/core';
|
|
3
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
4
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
5
5
|
import { nanoid } from 'nanoid';
|
|
6
|
-
import ow from 'ow';
|
|
7
6
|
import pLimit from 'p-limit';
|
|
8
7
|
import QuickLRU from 'quick-lru';
|
|
9
8
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
9
|
+
import { z } from 'zod';
|
|
10
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
|
+
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
11
12
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
12
13
|
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
13
14
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
14
15
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
16
|
+
const browserPoolOptionsSchema = z.strictObject({
|
|
17
|
+
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
18
|
+
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
19
|
+
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
20
|
+
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
21
|
+
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
22
|
+
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
|
|
23
|
+
preLaunchHooks: schemas.anyArray.default(() => []),
|
|
24
|
+
postLaunchHooks: schemas.anyArray.default(() => []),
|
|
25
|
+
prePageCreateHooks: schemas.anyArray.default(() => []),
|
|
26
|
+
postPageCreateHooks: schemas.anyArray.default(() => []),
|
|
27
|
+
prePageCloseHooks: schemas.anyArray.default(() => []),
|
|
28
|
+
postPageCloseHooks: schemas.anyArray.default(() => []),
|
|
29
|
+
useFingerprints: z.boolean().default(true),
|
|
30
|
+
fingerprintOptions: schemas.anyObject.default(() => ({})),
|
|
31
|
+
});
|
|
15
32
|
/**
|
|
16
33
|
* The `BrowserPool` class is the most important class of the `browser-pool` module.
|
|
17
34
|
* It manages opening and closing of browsers and their pages and its constructor
|
|
@@ -84,31 +101,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
84
101
|
fingerprintInjector;
|
|
85
102
|
fingerprintGenerator;
|
|
86
103
|
fingerprintCache;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
104
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
105
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
106
|
+
#browserRetireInterval;
|
|
107
|
+
#limiter = pLimit(1);
|
|
108
|
+
#log;
|
|
91
109
|
constructor(options) {
|
|
92
110
|
super();
|
|
93
|
-
this
|
|
111
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
94
112
|
this.browserKillerInterval.unref();
|
|
95
|
-
|
|
96
|
-
browserPlugins: ow.array.minLength(1),
|
|
97
|
-
maxOpenPagesPerBrowser: ow.optional.number,
|
|
98
|
-
retireBrowserAfterPageCount: ow.optional.number,
|
|
99
|
-
operationTimeoutSecs: ow.optional.number,
|
|
100
|
-
closeInactiveBrowserAfterSecs: ow.optional.number,
|
|
101
|
-
retireInactiveBrowserAfterSecs: ow.optional.number,
|
|
102
|
-
preLaunchHooks: ow.optional.array,
|
|
103
|
-
postLaunchHooks: ow.optional.array,
|
|
104
|
-
prePageCreateHooks: ow.optional.array,
|
|
105
|
-
postPageCreateHooks: ow.optional.array,
|
|
106
|
-
prePageCloseHooks: ow.optional.array,
|
|
107
|
-
postPageCloseHooks: ow.optional.array,
|
|
108
|
-
useFingerprints: ow.optional.boolean,
|
|
109
|
-
fingerprintOptions: ow.optional.object,
|
|
110
|
-
}));
|
|
111
|
-
const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
|
|
113
|
+
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
112
114
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
113
115
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
114
116
|
const providedPlugin = browserPlugins[i];
|
|
@@ -126,13 +128,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
126
128
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
127
129
|
this.useFingerprints = useFingerprints;
|
|
128
130
|
this.fingerprintOptions = fingerprintOptions;
|
|
129
|
-
this
|
|
131
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
130
132
|
if (controller.activePages === 0 &&
|
|
131
133
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
132
134
|
this.retireBrowserController(controller);
|
|
133
135
|
}
|
|
134
136
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
135
|
-
this
|
|
137
|
+
this.#browserRetireInterval.unref();
|
|
136
138
|
// hooks
|
|
137
139
|
this.preLaunchHooks = preLaunchHooks;
|
|
138
140
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -142,7 +144,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
142
144
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
143
145
|
// fingerprinting
|
|
144
146
|
if (this.useFingerprints) {
|
|
145
|
-
this.
|
|
147
|
+
this.initializeFingerprinting();
|
|
146
148
|
}
|
|
147
149
|
}
|
|
148
150
|
/**
|
|
@@ -163,7 +165,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
163
165
|
* crawler's responsibility.
|
|
164
166
|
*/
|
|
165
167
|
async newPage(options = {}) {
|
|
166
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
168
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
167
169
|
if (this.pages.has(id)) {
|
|
168
170
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
169
171
|
}
|
|
@@ -176,18 +178,21 @@ export class BrowserPool extends TypedEmitter {
|
|
|
176
178
|
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
177
179
|
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
178
180
|
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
181
|
+
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
182
|
+
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
183
|
+
// context and run request B's storage writes inside request A's transaction.
|
|
179
184
|
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
180
185
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
181
|
-
return this
|
|
182
|
-
let browserController = this.
|
|
186
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
187
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
183
188
|
if (!browserController)
|
|
184
|
-
browserController = await this.
|
|
189
|
+
browserController = await this.launchBrowser(id, {
|
|
185
190
|
browserPlugin,
|
|
186
191
|
proxyUrl,
|
|
187
192
|
ignoreTlsErrors,
|
|
188
193
|
});
|
|
189
194
|
tryCancel();
|
|
190
|
-
return await this.
|
|
195
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
191
196
|
}));
|
|
192
197
|
}
|
|
193
198
|
/**
|
|
@@ -196,13 +201,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
196
201
|
* configure the new browser.
|
|
197
202
|
*/
|
|
198
203
|
async newPageInNewBrowser(options = {}) {
|
|
199
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
204
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
200
205
|
if (this.pages.has(id)) {
|
|
201
206
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
202
207
|
}
|
|
203
|
-
const browserController = await this.
|
|
208
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
204
209
|
tryCancel();
|
|
205
|
-
return await this.
|
|
210
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
206
211
|
}
|
|
207
212
|
/**
|
|
208
213
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -268,7 +273,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
268
273
|
getPageId(page) {
|
|
269
274
|
return this.pageIds.get(page);
|
|
270
275
|
}
|
|
271
|
-
async
|
|
276
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
272
277
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
273
278
|
// It's not ideal though, we need to come up with a better API.
|
|
274
279
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -284,7 +289,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
284
289
|
});
|
|
285
290
|
}
|
|
286
291
|
}
|
|
287
|
-
await this.
|
|
292
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
288
293
|
tryCancel();
|
|
289
294
|
let page;
|
|
290
295
|
try {
|
|
@@ -297,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
297
302
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
298
303
|
this.retireBrowserController(browserController);
|
|
299
304
|
}
|
|
300
|
-
this.
|
|
305
|
+
this.overridePageClose(page);
|
|
301
306
|
}
|
|
302
307
|
catch (err) {
|
|
303
308
|
this.retireBrowserController(browserController);
|
|
304
309
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
305
310
|
}
|
|
306
|
-
await this.
|
|
311
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
307
312
|
tryCancel();
|
|
308
|
-
this.emit(
|
|
313
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
309
314
|
return page;
|
|
310
315
|
}
|
|
311
316
|
/**
|
|
@@ -320,7 +325,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
320
325
|
if (hasBeenRetiredOrKilled)
|
|
321
326
|
return;
|
|
322
327
|
this.retiredBrowserControllers.add(browserController);
|
|
323
|
-
this.emit(
|
|
328
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
324
329
|
this.startingBrowserControllers.delete(browserController);
|
|
325
330
|
this.activeBrowserControllers.delete(browserController);
|
|
326
331
|
}
|
|
@@ -400,7 +405,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
400
405
|
* @return {Promise<void>}
|
|
401
406
|
*/
|
|
402
407
|
async closeAllBrowsers() {
|
|
403
|
-
const controllers = this.
|
|
408
|
+
const controllers = this.getAllBrowserControllers();
|
|
404
409
|
const promises = [...controllers]
|
|
405
410
|
.filter((controller) => controller.isActive)
|
|
406
411
|
.map(async (controller) => controller.close());
|
|
@@ -411,26 +416,26 @@ export class BrowserPool extends TypedEmitter {
|
|
|
411
416
|
*/
|
|
412
417
|
async destroy() {
|
|
413
418
|
clearInterval(this.browserKillerInterval);
|
|
414
|
-
clearInterval(this
|
|
419
|
+
clearInterval(this.#browserRetireInterval);
|
|
415
420
|
this.browserKillerInterval = undefined;
|
|
416
|
-
this
|
|
421
|
+
this.#browserRetireInterval = undefined;
|
|
417
422
|
await this.closeAllBrowsers();
|
|
418
|
-
this.
|
|
423
|
+
this.teardown();
|
|
419
424
|
}
|
|
420
|
-
|
|
425
|
+
teardown() {
|
|
421
426
|
this.startingBrowserControllers.clear();
|
|
422
427
|
this.activeBrowserControllers.clear();
|
|
423
428
|
this.retiredBrowserControllers.clear();
|
|
424
429
|
this.removeAllListeners();
|
|
425
430
|
}
|
|
426
|
-
|
|
431
|
+
getAllBrowserControllers() {
|
|
427
432
|
return new Set([
|
|
428
433
|
...this.startingBrowserControllers,
|
|
429
434
|
...this.activeBrowserControllers,
|
|
430
435
|
...this.retiredBrowserControllers,
|
|
431
436
|
]);
|
|
432
437
|
}
|
|
433
|
-
async
|
|
438
|
+
async launchBrowser(pageId, options) {
|
|
434
439
|
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
435
440
|
const browserController = browserPlugin.createController();
|
|
436
441
|
this.startingBrowserControllers.add(browserController);
|
|
@@ -451,7 +456,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
451
456
|
try {
|
|
452
457
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
453
458
|
// because otherwise it would be stuck in limbo without a browser.
|
|
454
|
-
await this.
|
|
459
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
455
460
|
tryCancel();
|
|
456
461
|
const browser = await browserPlugin.launch(launchContext);
|
|
457
462
|
tryCancel();
|
|
@@ -461,17 +466,17 @@ export class BrowserPool extends TypedEmitter {
|
|
|
461
466
|
this.startingBrowserControllers.delete(browserController);
|
|
462
467
|
throw err;
|
|
463
468
|
}
|
|
464
|
-
this
|
|
469
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
465
470
|
browserController.proxyUrl = proxyUrl;
|
|
466
471
|
try {
|
|
467
472
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
468
473
|
// both the controller and the browser before throwing.
|
|
469
|
-
await this.
|
|
474
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
470
475
|
}
|
|
471
476
|
catch (err) {
|
|
472
477
|
this.startingBrowserControllers.delete(browserController);
|
|
473
478
|
browserController.close().catch((closeErr) => {
|
|
474
|
-
this
|
|
479
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
475
480
|
id: browserController.id,
|
|
476
481
|
});
|
|
477
482
|
});
|
|
@@ -481,19 +486,19 @@ export class BrowserPool extends TypedEmitter {
|
|
|
481
486
|
browserController.activate();
|
|
482
487
|
this.startingBrowserControllers.delete(browserController);
|
|
483
488
|
this.activeBrowserControllers.add(browserController);
|
|
484
|
-
this.emit(
|
|
489
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
485
490
|
return browserController;
|
|
486
491
|
}
|
|
487
492
|
/**
|
|
488
493
|
* Picks plugins round robin.
|
|
489
494
|
* @private
|
|
490
495
|
*/
|
|
491
|
-
|
|
496
|
+
pickBrowserPlugin() {
|
|
492
497
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
493
498
|
this.pageCounter++;
|
|
494
499
|
return this.browserPlugins[pluginIndex];
|
|
495
500
|
}
|
|
496
|
-
|
|
501
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
497
502
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
498
503
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
499
504
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
@@ -505,7 +510,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
505
510
|
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
506
511
|
});
|
|
507
512
|
}
|
|
508
|
-
async
|
|
513
|
+
async closeInactiveRetiredBrowsers() {
|
|
509
514
|
const closedBrowserIds = [];
|
|
510
515
|
for (const controller of this.retiredBrowserControllers) {
|
|
511
516
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -513,45 +518,45 @@ export class BrowserPool extends TypedEmitter {
|
|
|
513
518
|
const isBrowserEmpty = controller.activePages === 0;
|
|
514
519
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
515
520
|
const { id } = controller;
|
|
516
|
-
this
|
|
521
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
517
522
|
await controller.close();
|
|
518
523
|
this.retiredBrowserControllers.delete(controller);
|
|
519
524
|
closedBrowserIds.push(id);
|
|
520
525
|
}
|
|
521
526
|
}
|
|
522
527
|
if (closedBrowserIds.length) {
|
|
523
|
-
this
|
|
528
|
+
this.#log.debug('Closed retired browsers.', {
|
|
524
529
|
count: closedBrowserIds.length,
|
|
525
530
|
closedBrowserIds,
|
|
526
531
|
});
|
|
527
532
|
}
|
|
528
533
|
}
|
|
529
|
-
|
|
534
|
+
overridePageClose(page) {
|
|
530
535
|
const originalPageClose = page.close;
|
|
531
536
|
const browserController = this.pageToBrowserController.get(page);
|
|
532
537
|
const pageId = this.getPageId(page);
|
|
533
538
|
page.close = async (...args) => {
|
|
534
|
-
await this.
|
|
539
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
535
540
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
536
|
-
this
|
|
541
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
537
542
|
});
|
|
538
|
-
await this.
|
|
543
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
539
544
|
this.pages.delete(pageId);
|
|
540
|
-
this.
|
|
541
|
-
this.emit(
|
|
545
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
546
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
542
547
|
};
|
|
543
548
|
}
|
|
544
|
-
async
|
|
549
|
+
async executeHooks(hooks, ...args) {
|
|
545
550
|
for (const hook of hooks) {
|
|
546
551
|
await hook(...args);
|
|
547
552
|
}
|
|
548
553
|
}
|
|
549
|
-
|
|
554
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
550
555
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
551
556
|
// Run this with a delay, otherwise page.close()
|
|
552
557
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
553
558
|
setTimeout(() => {
|
|
554
|
-
this
|
|
559
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
555
560
|
void browserController.close().finally(() => {
|
|
556
561
|
this.retiredBrowserControllers.delete(browserController);
|
|
557
562
|
});
|
|
@@ -578,16 +583,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
578
583
|
}
|
|
579
584
|
return false;
|
|
580
585
|
}
|
|
581
|
-
|
|
586
|
+
initializeFingerprinting() {
|
|
582
587
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
583
588
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
584
589
|
this.fingerprintInjector = new FingerprintInjector();
|
|
585
590
|
if (useFingerprintCache) {
|
|
586
591
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
587
592
|
}
|
|
588
|
-
this.
|
|
593
|
+
this.addFingerprintHooks();
|
|
589
594
|
}
|
|
590
|
-
|
|
595
|
+
addFingerprintHooks() {
|
|
591
596
|
this.preLaunchHooks = [
|
|
592
597
|
...this.preLaunchHooks,
|
|
593
598
|
// This is flipped because of the fingerprint cache.
|
package/events.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
4
|
BROWSER_CLOSED = "browserClosed",
|
|
5
5
|
PAGE_CREATED = "pageCreated",
|
|
6
6
|
PAGE_CLOSED = "pageClosed"
|
|
7
7
|
}
|
|
8
|
-
export declare
|
|
8
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
9
|
BROWSER_CLOSED = "browserClosed"
|
|
10
10
|
}
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -35,7 +35,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
35
35
|
if (cacheKey)
|
|
36
36
|
fingerprintCache?.set(cacheKey, fingerprint);
|
|
37
37
|
}
|
|
38
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
39
40
|
if (useIncognitoPages) {
|
|
40
41
|
return;
|
|
41
42
|
}
|