@crawlee/browser-pool 4.0.0-beta.105 → 4.0.0-beta.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstract-classes/browser-controller.d.ts +1 -3
- package/abstract-classes/browser-controller.js +10 -9
- package/abstract-classes/browser-plugin.d.ts +5 -5
- package/abstract-classes/browser-plugin.js +7 -7
- package/browser-pool.d.ts +13 -15
- package/browser-pool.js +48 -47
- package/fingerprinting/hooks.js +2 -1
- package/launch-context.d.ts +2 -3
- package/launch-context.js +11 -8
- package/package.json +4 -4
- package/playwright/playwright-browser.d.ts +2 -5
- package/playwright/playwright-browser.js +16 -16
- package/playwright/playwright-plugin.d.ts +4 -4
- package/playwright/playwright-plugin.js +11 -11
- package/puppeteer/puppeteer-plugin.d.ts +2 -2
- package/puppeteer/puppeteer-plugin.js +4 -4
- package/remote-browser-pool.d.ts +3 -9
- package/remote-browser-pool.js +41 -41
|
@@ -80,6 +80,7 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
|
|
|
80
80
|
* @hideconstructor
|
|
81
81
|
*/
|
|
82
82
|
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
|
|
83
|
+
#private;
|
|
83
84
|
readonly id: string;
|
|
84
85
|
protected readonly log: CrawleeLogger;
|
|
85
86
|
/**
|
|
@@ -103,10 +104,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
|
|
|
103
104
|
activePages: number;
|
|
104
105
|
totalPages: number;
|
|
105
106
|
lastPageOpenedAt: number;
|
|
106
|
-
private _activate;
|
|
107
107
|
private isActivePromise;
|
|
108
|
-
private commitBrowser;
|
|
109
|
-
private hasBrowserPromise;
|
|
110
108
|
constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
111
109
|
/**
|
|
112
110
|
* Activates the BrowserController. If you try to open new pages before
|
|
@@ -35,13 +35,14 @@ export class BrowserController extends TypedEmitter {
|
|
|
35
35
|
activePages = 0;
|
|
36
36
|
totalPages = 0;
|
|
37
37
|
lastPageOpenedAt = Date.now();
|
|
38
|
-
|
|
38
|
+
#activate;
|
|
39
|
+
// kept as TS-private: `BrowserPool` awaits it through cross-object bracket access
|
|
39
40
|
isActivePromise = new Promise((resolve) => {
|
|
40
|
-
this
|
|
41
|
+
this.#activate = resolve;
|
|
41
42
|
});
|
|
42
|
-
commitBrowser;
|
|
43
|
-
hasBrowserPromise = new Promise((resolve) => {
|
|
44
|
-
this
|
|
43
|
+
#commitBrowser;
|
|
44
|
+
#hasBrowserPromise = new Promise((resolve) => {
|
|
45
|
+
this.#commitBrowser = resolve;
|
|
45
46
|
});
|
|
46
47
|
constructor(browserPlugin) {
|
|
47
48
|
super();
|
|
@@ -58,7 +59,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
58
59
|
if (!this.browser) {
|
|
59
60
|
throw new Error('Cannot activate BrowserController without an assigned browser.');
|
|
60
61
|
}
|
|
61
|
-
this
|
|
62
|
+
this.#activate();
|
|
62
63
|
this.isActive = true;
|
|
63
64
|
}
|
|
64
65
|
/**
|
|
@@ -70,7 +71,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
70
71
|
}
|
|
71
72
|
this.browser = browser;
|
|
72
73
|
this.launchContext = launchContext;
|
|
73
|
-
this
|
|
74
|
+
this.#commitBrowser();
|
|
74
75
|
}
|
|
75
76
|
/**
|
|
76
77
|
* Gracefully closes the browser and makes sure
|
|
@@ -79,7 +80,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
79
80
|
* Emits 'browserClosed' event.
|
|
80
81
|
*/
|
|
81
82
|
async close() {
|
|
82
|
-
await this
|
|
83
|
+
await this.#hasBrowserPromise;
|
|
83
84
|
try {
|
|
84
85
|
await this._close();
|
|
85
86
|
// TODO: shouldn't this go in a finally instead?
|
|
@@ -102,7 +103,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
102
103
|
* Emits 'browserClosed' event.
|
|
103
104
|
*/
|
|
104
105
|
async kill() {
|
|
105
|
-
await this
|
|
106
|
+
await this.#hasBrowserPromise;
|
|
106
107
|
await this._kill();
|
|
107
108
|
this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
|
|
108
109
|
}
|
|
@@ -119,7 +119,7 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
119
119
|
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
120
120
|
* lives here so it stays identical across plugins.
|
|
121
121
|
*/
|
|
122
|
-
protected
|
|
122
|
+
protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
|
|
123
123
|
/**
|
|
124
124
|
* Creates a `LaunchContext` with all the information needed
|
|
125
125
|
* to launch a browser. Aside from library specific launch options,
|
|
@@ -132,13 +132,13 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
132
132
|
* Launches the browser using provided launch context.
|
|
133
133
|
*/
|
|
134
134
|
launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
|
135
|
-
private
|
|
136
|
-
protected
|
|
135
|
+
private mergeArgsToHideWebdriver;
|
|
136
|
+
protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
|
|
137
137
|
/**
|
|
138
138
|
* @private
|
|
139
139
|
*/
|
|
140
|
-
protected abstract
|
|
141
|
-
protected abstract
|
|
140
|
+
protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
|
|
141
|
+
protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
|
|
142
142
|
/**
|
|
143
143
|
* @private
|
|
144
144
|
*/
|
|
@@ -69,7 +69,7 @@ export class BrowserPlugin {
|
|
|
69
69
|
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
70
70
|
* lives here so it stays identical across plugins.
|
|
71
71
|
*/
|
|
72
|
-
async
|
|
72
|
+
async connectToRemoteBrowser(launchContext, connect) {
|
|
73
73
|
const connection = this.remoteConnection;
|
|
74
74
|
let url;
|
|
75
75
|
let token;
|
|
@@ -79,7 +79,7 @@ export class BrowserPlugin {
|
|
|
79
79
|
catch (cause) {
|
|
80
80
|
throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
|
|
81
81
|
}
|
|
82
|
-
launchContext.
|
|
82
|
+
launchContext.remoteToken = token;
|
|
83
83
|
try {
|
|
84
84
|
return await connect(url);
|
|
85
85
|
}
|
|
@@ -117,11 +117,11 @@ export class BrowserPlugin {
|
|
|
117
117
|
launchContext.launchOptions ??= {};
|
|
118
118
|
const { proxyUrl, launchOptions } = launchContext;
|
|
119
119
|
if (proxyUrl && !launchContext.isRemote) {
|
|
120
|
-
await this.
|
|
120
|
+
await this.addProxyToLaunchOptions(launchContext);
|
|
121
121
|
}
|
|
122
|
-
if (!launchContext.isRemote && this.
|
|
122
|
+
if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
|
|
123
123
|
// This will set the args for chromium based browsers to hide the webdriver.
|
|
124
|
-
launchOptions.args = this.
|
|
124
|
+
launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
|
|
125
125
|
// When User-Agent is not set, and we're using Chromium in headless mode,
|
|
126
126
|
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
|
|
127
127
|
// as otherwise 'HeadlessChrome' is present in User-Agent string.
|
|
@@ -135,7 +135,7 @@ export class BrowserPlugin {
|
|
|
135
135
|
}
|
|
136
136
|
return this._launch(launchContext);
|
|
137
137
|
}
|
|
138
|
-
|
|
138
|
+
mergeArgsToHideWebdriver(originalArgs) {
|
|
139
139
|
if (!originalArgs?.length) {
|
|
140
140
|
return ['--disable-blink-features=AutomationControlled'];
|
|
141
141
|
}
|
|
@@ -148,7 +148,7 @@ export class BrowserPlugin {
|
|
|
148
148
|
}
|
|
149
149
|
return originalArgs;
|
|
150
150
|
}
|
|
151
|
-
|
|
151
|
+
throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
|
|
152
152
|
const errorMessage = ['Failed to launch browser. Please check the following:'];
|
|
153
153
|
if (executablePath) {
|
|
154
154
|
errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
|
package/browser-pool.d.ts
CHANGED
|
@@ -248,6 +248,7 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
|
|
|
248
248
|
* ```
|
|
249
249
|
*/
|
|
250
250
|
export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
|
|
251
|
+
#private;
|
|
251
252
|
browserPlugins: BrowserPlugins;
|
|
252
253
|
maxOpenPagesPerBrowser: number;
|
|
253
254
|
maxOpenBrowsers: number;
|
|
@@ -273,9 +274,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
273
274
|
fingerprintGenerator?: FingerprintGenerator;
|
|
274
275
|
fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
|
|
275
276
|
private browserKillerInterval?;
|
|
276
|
-
private browserRetireInterval?;
|
|
277
|
-
private limiter;
|
|
278
|
-
private log;
|
|
279
277
|
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
|
280
278
|
/**
|
|
281
279
|
* Opens a new page in one of the running browsers or launches
|
|
@@ -350,7 +348,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
350
348
|
* until it's closed.
|
|
351
349
|
*/
|
|
352
350
|
getPageId(page: PageReturn): string | undefined;
|
|
353
|
-
private
|
|
351
|
+
private createPageForBrowser;
|
|
354
352
|
/**
|
|
355
353
|
* Removes a browser controller from the pool. The underlying
|
|
356
354
|
* browser will be closed after all its pages are closed.
|
|
@@ -413,19 +411,19 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
413
411
|
* Closes all managed browsers and tears down the pool.
|
|
414
412
|
*/
|
|
415
413
|
destroy(): Promise<void>;
|
|
416
|
-
private
|
|
417
|
-
private
|
|
418
|
-
private
|
|
414
|
+
private teardown;
|
|
415
|
+
private getAllBrowserControllers;
|
|
416
|
+
private launchBrowser;
|
|
419
417
|
/**
|
|
420
418
|
* Picks plugins round robin.
|
|
421
419
|
* @private
|
|
422
420
|
*/
|
|
423
|
-
private
|
|
424
|
-
private
|
|
425
|
-
private
|
|
426
|
-
private
|
|
427
|
-
private
|
|
428
|
-
private
|
|
421
|
+
private pickBrowserPlugin;
|
|
422
|
+
private pickBrowserWithFreeCapacity;
|
|
423
|
+
private closeInactiveRetiredBrowsers;
|
|
424
|
+
private overridePageClose;
|
|
425
|
+
private executeHooks;
|
|
426
|
+
private closeRetiredBrowserWithNoPages;
|
|
429
427
|
/**
|
|
430
428
|
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
431
429
|
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
@@ -435,8 +433,8 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
435
433
|
* Returns `true` if any active browser has room for another page.
|
|
436
434
|
*/
|
|
437
435
|
hasActiveBrowserWithFreeCapacity(): boolean;
|
|
438
|
-
private
|
|
439
|
-
private
|
|
436
|
+
private initializeFingerprinting;
|
|
437
|
+
private addFingerprintHooks;
|
|
440
438
|
}
|
|
441
439
|
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
|
|
442
440
|
/**
|
package/browser-pool.js
CHANGED
|
@@ -84,13 +84,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
84
84
|
fingerprintInjector;
|
|
85
85
|
fingerprintGenerator;
|
|
86
86
|
fingerprintCache;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
87
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
88
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
89
|
+
#browserRetireInterval;
|
|
90
|
+
#limiter = pLimit(1);
|
|
91
|
+
#log;
|
|
91
92
|
constructor(options) {
|
|
92
93
|
super();
|
|
93
|
-
this
|
|
94
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
94
95
|
this.browserKillerInterval.unref();
|
|
95
96
|
ow(options, ow.object.exactShape({
|
|
96
97
|
browserPlugins: ow.array.minLength(1),
|
|
@@ -126,13 +127,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
126
127
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
127
128
|
this.useFingerprints = useFingerprints;
|
|
128
129
|
this.fingerprintOptions = fingerprintOptions;
|
|
129
|
-
this
|
|
130
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
130
131
|
if (controller.activePages === 0 &&
|
|
131
132
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
132
133
|
this.retireBrowserController(controller);
|
|
133
134
|
}
|
|
134
135
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
135
|
-
this
|
|
136
|
+
this.#browserRetireInterval.unref();
|
|
136
137
|
// hooks
|
|
137
138
|
this.preLaunchHooks = preLaunchHooks;
|
|
138
139
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -142,7 +143,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
142
143
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
143
144
|
// fingerprinting
|
|
144
145
|
if (this.useFingerprints) {
|
|
145
|
-
this.
|
|
146
|
+
this.initializeFingerprinting();
|
|
146
147
|
}
|
|
147
148
|
}
|
|
148
149
|
/**
|
|
@@ -163,7 +164,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
163
164
|
* crawler's responsibility.
|
|
164
165
|
*/
|
|
165
166
|
async newPage(options = {}) {
|
|
166
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
167
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
167
168
|
if (this.pages.has(id)) {
|
|
168
169
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
169
170
|
}
|
|
@@ -181,16 +182,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
181
182
|
// context and run request B's storage writes inside request A's transaction.
|
|
182
183
|
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
183
184
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
184
|
-
return this
|
|
185
|
-
let browserController = this.
|
|
185
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
186
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
186
187
|
if (!browserController)
|
|
187
|
-
browserController = await this.
|
|
188
|
+
browserController = await this.launchBrowser(id, {
|
|
188
189
|
browserPlugin,
|
|
189
190
|
proxyUrl,
|
|
190
191
|
ignoreTlsErrors,
|
|
191
192
|
});
|
|
192
193
|
tryCancel();
|
|
193
|
-
return await this.
|
|
194
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
194
195
|
}));
|
|
195
196
|
}
|
|
196
197
|
/**
|
|
@@ -199,13 +200,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
199
200
|
* configure the new browser.
|
|
200
201
|
*/
|
|
201
202
|
async newPageInNewBrowser(options = {}) {
|
|
202
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
203
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
203
204
|
if (this.pages.has(id)) {
|
|
204
205
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
205
206
|
}
|
|
206
|
-
const browserController = await this.
|
|
207
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
207
208
|
tryCancel();
|
|
208
|
-
return await this.
|
|
209
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
209
210
|
}
|
|
210
211
|
/**
|
|
211
212
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -271,7 +272,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
271
272
|
getPageId(page) {
|
|
272
273
|
return this.pageIds.get(page);
|
|
273
274
|
}
|
|
274
|
-
async
|
|
275
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
275
276
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
276
277
|
// It's not ideal though, we need to come up with a better API.
|
|
277
278
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -287,7 +288,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
287
288
|
});
|
|
288
289
|
}
|
|
289
290
|
}
|
|
290
|
-
await this.
|
|
291
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
291
292
|
tryCancel();
|
|
292
293
|
let page;
|
|
293
294
|
try {
|
|
@@ -300,13 +301,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
300
301
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
301
302
|
this.retireBrowserController(browserController);
|
|
302
303
|
}
|
|
303
|
-
this.
|
|
304
|
+
this.overridePageClose(page);
|
|
304
305
|
}
|
|
305
306
|
catch (err) {
|
|
306
307
|
this.retireBrowserController(browserController);
|
|
307
308
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
308
309
|
}
|
|
309
|
-
await this.
|
|
310
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
310
311
|
tryCancel();
|
|
311
312
|
this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
|
|
312
313
|
return page;
|
|
@@ -403,7 +404,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
403
404
|
* @return {Promise<void>}
|
|
404
405
|
*/
|
|
405
406
|
async closeAllBrowsers() {
|
|
406
|
-
const controllers = this.
|
|
407
|
+
const controllers = this.getAllBrowserControllers();
|
|
407
408
|
const promises = [...controllers]
|
|
408
409
|
.filter((controller) => controller.isActive)
|
|
409
410
|
.map(async (controller) => controller.close());
|
|
@@ -414,26 +415,26 @@ export class BrowserPool extends TypedEmitter {
|
|
|
414
415
|
*/
|
|
415
416
|
async destroy() {
|
|
416
417
|
clearInterval(this.browserKillerInterval);
|
|
417
|
-
clearInterval(this
|
|
418
|
+
clearInterval(this.#browserRetireInterval);
|
|
418
419
|
this.browserKillerInterval = undefined;
|
|
419
|
-
this
|
|
420
|
+
this.#browserRetireInterval = undefined;
|
|
420
421
|
await this.closeAllBrowsers();
|
|
421
|
-
this.
|
|
422
|
+
this.teardown();
|
|
422
423
|
}
|
|
423
|
-
|
|
424
|
+
teardown() {
|
|
424
425
|
this.startingBrowserControllers.clear();
|
|
425
426
|
this.activeBrowserControllers.clear();
|
|
426
427
|
this.retiredBrowserControllers.clear();
|
|
427
428
|
this.removeAllListeners();
|
|
428
429
|
}
|
|
429
|
-
|
|
430
|
+
getAllBrowserControllers() {
|
|
430
431
|
return new Set([
|
|
431
432
|
...this.startingBrowserControllers,
|
|
432
433
|
...this.activeBrowserControllers,
|
|
433
434
|
...this.retiredBrowserControllers,
|
|
434
435
|
]);
|
|
435
436
|
}
|
|
436
|
-
async
|
|
437
|
+
async launchBrowser(pageId, options) {
|
|
437
438
|
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
438
439
|
const browserController = browserPlugin.createController();
|
|
439
440
|
this.startingBrowserControllers.add(browserController);
|
|
@@ -454,7 +455,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
454
455
|
try {
|
|
455
456
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
456
457
|
// because otherwise it would be stuck in limbo without a browser.
|
|
457
|
-
await this.
|
|
458
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
458
459
|
tryCancel();
|
|
459
460
|
const browser = await browserPlugin.launch(launchContext);
|
|
460
461
|
tryCancel();
|
|
@@ -464,17 +465,17 @@ export class BrowserPool extends TypedEmitter {
|
|
|
464
465
|
this.startingBrowserControllers.delete(browserController);
|
|
465
466
|
throw err;
|
|
466
467
|
}
|
|
467
|
-
this
|
|
468
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
468
469
|
browserController.proxyUrl = proxyUrl;
|
|
469
470
|
try {
|
|
470
471
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
471
472
|
// both the controller and the browser before throwing.
|
|
472
|
-
await this.
|
|
473
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
473
474
|
}
|
|
474
475
|
catch (err) {
|
|
475
476
|
this.startingBrowserControllers.delete(browserController);
|
|
476
477
|
browserController.close().catch((closeErr) => {
|
|
477
|
-
this
|
|
478
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
478
479
|
id: browserController.id,
|
|
479
480
|
});
|
|
480
481
|
});
|
|
@@ -491,12 +492,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
491
492
|
* Picks plugins round robin.
|
|
492
493
|
* @private
|
|
493
494
|
*/
|
|
494
|
-
|
|
495
|
+
pickBrowserPlugin() {
|
|
495
496
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
496
497
|
this.pageCounter++;
|
|
497
498
|
return this.browserPlugins[pluginIndex];
|
|
498
499
|
}
|
|
499
|
-
|
|
500
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
500
501
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
501
502
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
502
503
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
@@ -508,7 +509,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
508
509
|
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
509
510
|
});
|
|
510
511
|
}
|
|
511
|
-
async
|
|
512
|
+
async closeInactiveRetiredBrowsers() {
|
|
512
513
|
const closedBrowserIds = [];
|
|
513
514
|
for (const controller of this.retiredBrowserControllers) {
|
|
514
515
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -516,45 +517,45 @@ export class BrowserPool extends TypedEmitter {
|
|
|
516
517
|
const isBrowserEmpty = controller.activePages === 0;
|
|
517
518
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
518
519
|
const { id } = controller;
|
|
519
|
-
this
|
|
520
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
520
521
|
await controller.close();
|
|
521
522
|
this.retiredBrowserControllers.delete(controller);
|
|
522
523
|
closedBrowserIds.push(id);
|
|
523
524
|
}
|
|
524
525
|
}
|
|
525
526
|
if (closedBrowserIds.length) {
|
|
526
|
-
this
|
|
527
|
+
this.#log.debug('Closed retired browsers.', {
|
|
527
528
|
count: closedBrowserIds.length,
|
|
528
529
|
closedBrowserIds,
|
|
529
530
|
});
|
|
530
531
|
}
|
|
531
532
|
}
|
|
532
|
-
|
|
533
|
+
overridePageClose(page) {
|
|
533
534
|
const originalPageClose = page.close;
|
|
534
535
|
const browserController = this.pageToBrowserController.get(page);
|
|
535
536
|
const pageId = this.getPageId(page);
|
|
536
537
|
page.close = async (...args) => {
|
|
537
|
-
await this.
|
|
538
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
538
539
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
539
|
-
this
|
|
540
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
540
541
|
});
|
|
541
|
-
await this.
|
|
542
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
542
543
|
this.pages.delete(pageId);
|
|
543
|
-
this.
|
|
544
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
544
545
|
this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
|
|
545
546
|
};
|
|
546
547
|
}
|
|
547
|
-
async
|
|
548
|
+
async executeHooks(hooks, ...args) {
|
|
548
549
|
for (const hook of hooks) {
|
|
549
550
|
await hook(...args);
|
|
550
551
|
}
|
|
551
552
|
}
|
|
552
|
-
|
|
553
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
553
554
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
554
555
|
// Run this with a delay, otherwise page.close()
|
|
555
556
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
556
557
|
setTimeout(() => {
|
|
557
|
-
this
|
|
558
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
558
559
|
void browserController.close().finally(() => {
|
|
559
560
|
this.retiredBrowserControllers.delete(browserController);
|
|
560
561
|
});
|
|
@@ -581,16 +582,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
581
582
|
}
|
|
582
583
|
return false;
|
|
583
584
|
}
|
|
584
|
-
|
|
585
|
+
initializeFingerprinting() {
|
|
585
586
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
586
587
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
587
588
|
this.fingerprintInjector = new FingerprintInjector();
|
|
588
589
|
if (useFingerprintCache) {
|
|
589
590
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
590
591
|
}
|
|
591
|
-
this.
|
|
592
|
+
this.addFingerprintHooks();
|
|
592
593
|
}
|
|
593
|
-
|
|
594
|
+
addFingerprintHooks() {
|
|
594
595
|
this.preLaunchHooks = [
|
|
595
596
|
...this.preLaunchHooks,
|
|
596
597
|
// This is flipped because of the fingerprint cache.
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -35,7 +35,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
35
35
|
if (cacheKey)
|
|
36
36
|
fingerprintCache?.set(cacheKey, fingerprint);
|
|
37
37
|
}
|
|
38
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
39
40
|
if (useIncognitoPages) {
|
|
40
41
|
return;
|
|
41
42
|
}
|
package/launch-context.d.ts
CHANGED
|
@@ -56,6 +56,7 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
|
|
|
56
56
|
isRemote?: boolean;
|
|
57
57
|
}
|
|
58
58
|
export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
59
|
+
#private;
|
|
59
60
|
id?: string;
|
|
60
61
|
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
61
62
|
launchOptions: LibraryOptions;
|
|
@@ -64,15 +65,13 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
64
65
|
userDataDir: string;
|
|
65
66
|
readonly isRemote: boolean;
|
|
66
67
|
ignoreProxyCertificate?: boolean;
|
|
67
|
-
private _proxyUrl?;
|
|
68
|
-
private readonly _reservedFieldNames;
|
|
69
68
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
70
69
|
/**
|
|
71
70
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
72
71
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
73
72
|
* @internal
|
|
74
73
|
*/
|
|
75
|
-
|
|
74
|
+
remoteToken?: number;
|
|
76
75
|
[K: PropertyKey]: unknown;
|
|
77
76
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
78
77
|
/**
|
package/launch-context.js
CHANGED
|
@@ -7,15 +7,15 @@ export class LaunchContext {
|
|
|
7
7
|
userDataDir;
|
|
8
8
|
isRemote;
|
|
9
9
|
ignoreProxyCertificate;
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
#proxyUrl;
|
|
11
|
+
#reservedFieldNames;
|
|
12
12
|
fingerprint;
|
|
13
13
|
/**
|
|
14
14
|
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
15
|
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
16
|
* @internal
|
|
17
17
|
*/
|
|
18
|
-
|
|
18
|
+
remoteToken;
|
|
19
19
|
constructor(options) {
|
|
20
20
|
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
21
21
|
this.id = id;
|
|
@@ -26,7 +26,10 @@ export class LaunchContext {
|
|
|
26
26
|
this.userDataDir = userDataDir;
|
|
27
27
|
this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
|
|
28
28
|
this.isRemote = isRemote ?? false;
|
|
29
|
-
this
|
|
29
|
+
this.#proxyUrl = proxyUrl;
|
|
30
|
+
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
|
+
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* Extend the launch context with any extra fields.
|
|
@@ -37,7 +40,7 @@ export class LaunchContext {
|
|
|
37
40
|
*/
|
|
38
41
|
extend(fields) {
|
|
39
42
|
Object.entries(fields).forEach(([key, value]) => {
|
|
40
|
-
if (this.
|
|
43
|
+
if (this.#reservedFieldNames.includes(key)) {
|
|
41
44
|
throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
|
|
42
45
|
}
|
|
43
46
|
else {
|
|
@@ -51,7 +54,7 @@ export class LaunchContext {
|
|
|
51
54
|
*/
|
|
52
55
|
set proxyUrl(url) {
|
|
53
56
|
if (!url) {
|
|
54
|
-
this
|
|
57
|
+
this.#proxyUrl = undefined;
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
60
|
const urlInstance = new URL(url);
|
|
@@ -59,12 +62,12 @@ export class LaunchContext {
|
|
|
59
62
|
urlInstance.search = '';
|
|
60
63
|
urlInstance.hash = '';
|
|
61
64
|
// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
|
|
62
|
-
this
|
|
65
|
+
this.#proxyUrl = urlInstance.href.slice(0, -1);
|
|
63
66
|
}
|
|
64
67
|
/**
|
|
65
68
|
* Returns the proxy URL of the browser.
|
|
66
69
|
*/
|
|
67
70
|
get proxyUrl() {
|
|
68
|
-
return this
|
|
71
|
+
return this.#proxyUrl;
|
|
69
72
|
}
|
|
70
73
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.106",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -31,8 +31,8 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@apify/timeout": "^0.4.4",
|
|
34
|
-
"@crawlee/core": "4.0.0-beta.
|
|
35
|
-
"@crawlee/types": "4.0.0-beta.
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.106",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.106",
|
|
36
36
|
"fingerprint-generator": "^2.1.68",
|
|
37
37
|
"fingerprint-injector": "^2.1.68",
|
|
38
38
|
"lodash.merge": "^4.6.2",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "c622f1fc65e65221ea245817c58ecc0ffb4a5cb0"
|
|
67
67
|
}
|
|
@@ -9,10 +9,7 @@ export interface BrowserOptions {
|
|
|
9
9
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
10
10
|
*/
|
|
11
11
|
export declare class PlaywrightBrowser extends EventEmitter {
|
|
12
|
-
private
|
|
13
|
-
private _version;
|
|
14
|
-
private _isConnected;
|
|
15
|
-
private _browserType?;
|
|
12
|
+
#private;
|
|
16
13
|
constructor(options: BrowserOptions);
|
|
17
14
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
15
|
close(): Promise<void>;
|
|
@@ -20,7 +17,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
|
|
|
20
17
|
isConnected(): boolean;
|
|
21
18
|
version(): string;
|
|
22
19
|
/** @internal */
|
|
23
|
-
|
|
20
|
+
setBrowserType(browserType: BrowserType): void;
|
|
24
21
|
browserType(): BrowserType;
|
|
25
22
|
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
|
26
23
|
newContext(): Promise<never>;
|
|
@@ -3,17 +3,17 @@ import { EventEmitter } from 'node:events';
|
|
|
3
3
|
* Browser wrapper created to have consistent API with persistent and non-persistent contexts.
|
|
4
4
|
*/
|
|
5
5
|
export class PlaywrightBrowser extends EventEmitter {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
#browserContext;
|
|
7
|
+
#version;
|
|
8
|
+
#isConnected = true;
|
|
9
|
+
#browserType;
|
|
10
10
|
constructor(options) {
|
|
11
11
|
super();
|
|
12
12
|
const { browserContext, version } = options;
|
|
13
|
-
this
|
|
14
|
-
this
|
|
15
|
-
this.
|
|
16
|
-
this
|
|
13
|
+
this.#browserContext = browserContext;
|
|
14
|
+
this.#version = version;
|
|
15
|
+
this.#browserContext.once('close', () => {
|
|
16
|
+
this.#isConnected = false;
|
|
17
17
|
this.emit('disconnected');
|
|
18
18
|
});
|
|
19
19
|
}
|
|
@@ -21,26 +21,26 @@ export class PlaywrightBrowser extends EventEmitter {
|
|
|
21
21
|
await this.close();
|
|
22
22
|
}
|
|
23
23
|
async close() {
|
|
24
|
-
await this.
|
|
24
|
+
await this.#browserContext.close();
|
|
25
25
|
}
|
|
26
26
|
contexts() {
|
|
27
|
-
return [this
|
|
27
|
+
return [this.#browserContext];
|
|
28
28
|
}
|
|
29
29
|
isConnected() {
|
|
30
|
-
return this
|
|
30
|
+
return this.#isConnected;
|
|
31
31
|
}
|
|
32
32
|
version() {
|
|
33
|
-
return this
|
|
33
|
+
return this.#version;
|
|
34
34
|
}
|
|
35
35
|
/** @internal */
|
|
36
|
-
|
|
37
|
-
this
|
|
36
|
+
setBrowserType(browserType) {
|
|
37
|
+
this.#browserType = browserType;
|
|
38
38
|
}
|
|
39
39
|
browserType() {
|
|
40
|
-
return this
|
|
40
|
+
return this.#browserType;
|
|
41
41
|
}
|
|
42
42
|
async newPage(...args) {
|
|
43
|
-
return this.
|
|
43
|
+
return this.#browserContext.newPage(...args);
|
|
44
44
|
}
|
|
45
45
|
async newContext() {
|
|
46
46
|
throw new Error('Function `newContext()` is not available in incognito mode');
|
|
@@ -6,15 +6,15 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
|
|
|
6
6
|
import type { SafeParameters } from '../utils.js';
|
|
7
7
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
8
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
|
-
private
|
|
9
|
+
#private;
|
|
10
10
|
/**
|
|
11
11
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
12
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
13
|
*/
|
|
14
14
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
15
15
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
16
|
-
private
|
|
16
|
+
private throwOnFailedLaunch;
|
|
17
17
|
createController(): PlaywrightController;
|
|
18
|
-
protected
|
|
19
|
-
protected
|
|
18
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
19
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
20
20
|
}
|
|
@@ -5,7 +5,7 @@ import { getLocalProxyAddress } from '../proxy-server.js';
|
|
|
5
5
|
import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
|
|
6
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
7
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
8
|
-
|
|
8
|
+
#browserVersion;
|
|
9
9
|
/**
|
|
10
10
|
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
11
11
|
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
@@ -20,7 +20,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
20
20
|
}
|
|
21
21
|
async _launch(launchContext) {
|
|
22
22
|
if (this.remoteConnection) {
|
|
23
|
-
return this.
|
|
23
|
+
return this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
24
24
|
const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
|
|
25
25
|
if (this.remoteConnectionParameters?.protocol === 'playwright') {
|
|
26
26
|
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
@@ -53,7 +53,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
53
53
|
try {
|
|
54
54
|
if (useIncognitoPages) {
|
|
55
55
|
browser = await this.library.launch(launchOptions).catch((error) => {
|
|
56
|
-
return this.
|
|
56
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
57
57
|
});
|
|
58
58
|
if (anonymizedProxyUrl) {
|
|
59
59
|
browser.on('disconnected', async () => {
|
|
@@ -65,7 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
65
65
|
const browserContext = await this.library
|
|
66
66
|
.launchPersistentContext(userDataDir, launchOptions)
|
|
67
67
|
.catch((error) => {
|
|
68
|
-
return this.
|
|
68
|
+
return this.throwOnFailedLaunch(launchContext, error);
|
|
69
69
|
});
|
|
70
70
|
browserContext.once('close', () => {
|
|
71
71
|
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
|
|
@@ -80,17 +80,17 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
80
80
|
await close();
|
|
81
81
|
});
|
|
82
82
|
}
|
|
83
|
-
if (!this
|
|
83
|
+
if (!this.#browserVersion) {
|
|
84
84
|
// Launches unused browser just to get the browser version.
|
|
85
85
|
const inactiveBrowser = await this.library.launch(launchOptions);
|
|
86
|
-
this
|
|
86
|
+
this.#browserVersion = inactiveBrowser.version();
|
|
87
87
|
inactiveBrowser.close().catch((error) => {
|
|
88
88
|
this.log.exception(error, 'Failed to close browser.');
|
|
89
89
|
});
|
|
90
90
|
}
|
|
91
91
|
browser = new PlaywrightBrowserWithPersistentContext({
|
|
92
92
|
browserContext,
|
|
93
|
-
version: this
|
|
93
|
+
version: this.#browserVersion,
|
|
94
94
|
});
|
|
95
95
|
}
|
|
96
96
|
}
|
|
@@ -100,13 +100,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
100
100
|
}
|
|
101
101
|
return browser;
|
|
102
102
|
}
|
|
103
|
-
|
|
104
|
-
this.
|
|
103
|
+
throwOnFailedLaunch(launchContext, cause) {
|
|
104
|
+
this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
|
|
105
105
|
}
|
|
106
106
|
createController() {
|
|
107
107
|
return new PlaywrightController(this);
|
|
108
108
|
}
|
|
109
|
-
async
|
|
109
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
110
110
|
launchContext.launchOptions ??= {};
|
|
111
111
|
const { launchOptions, proxyUrl } = launchContext;
|
|
112
112
|
if (proxyUrl) {
|
|
@@ -118,7 +118,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
|
|
|
118
118
|
};
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
|
|
121
|
+
isChromiumBasedBrowser() {
|
|
122
122
|
const name = this.library.name();
|
|
123
123
|
return name === 'chromium';
|
|
124
124
|
}
|
|
@@ -12,6 +12,6 @@ export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, Pup
|
|
|
12
12
|
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
13
13
|
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
|
14
14
|
createController(): PuppeteerController;
|
|
15
|
-
protected
|
|
16
|
-
protected
|
|
15
|
+
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
16
|
+
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
|
17
17
|
}
|
|
@@ -27,7 +27,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
27
27
|
const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
|
|
28
28
|
let browser;
|
|
29
29
|
if (this.remoteConnection) {
|
|
30
|
-
browser = await this.
|
|
30
|
+
browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
31
31
|
const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
|
|
32
32
|
this.log.info('Connecting to remote browser via connect (CDP).');
|
|
33
33
|
return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
|
|
@@ -73,7 +73,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
73
73
|
}
|
|
74
74
|
catch (error) {
|
|
75
75
|
await close();
|
|
76
|
-
this.
|
|
76
|
+
this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -177,7 +177,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
177
177
|
createController() {
|
|
178
178
|
return new PuppeteerController(this);
|
|
179
179
|
}
|
|
180
|
-
async
|
|
180
|
+
async addProxyToLaunchOptions(_launchContext) {
|
|
181
181
|
/*
|
|
182
182
|
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
|
|
183
183
|
launchContext.launchOptions ??= {};
|
|
@@ -204,7 +204,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
|
|
|
204
204
|
}
|
|
205
205
|
*/
|
|
206
206
|
}
|
|
207
|
-
|
|
207
|
+
isChromiumBasedBrowser(_launchContext) {
|
|
208
208
|
return true;
|
|
209
209
|
}
|
|
210
210
|
}
|
package/remote-browser-pool.d.ts
CHANGED
|
@@ -134,15 +134,9 @@ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browse
|
|
|
134
134
|
* @category Browser management
|
|
135
135
|
*/
|
|
136
136
|
export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
|
|
137
|
+
#private;
|
|
137
138
|
/** The wrapped pool that performs the remote connections and serves pages. */
|
|
138
139
|
readonly browserPool: BrowserPool;
|
|
139
|
-
/** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
|
|
140
|
-
private readonly pool;
|
|
141
|
-
private readonly registry;
|
|
142
|
-
private readonly slotPollIntervalMillis;
|
|
143
|
-
private readonly log;
|
|
144
|
-
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
|
|
145
|
-
private _capacityChange?;
|
|
146
140
|
constructor(options: RemoteBrowserPoolOptions);
|
|
147
141
|
/** Maximum number of remote browsers that may be open at the same time. */
|
|
148
142
|
get maxOpenBrowsers(): number;
|
|
@@ -160,11 +154,11 @@ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<P
|
|
|
160
154
|
/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
|
|
161
155
|
destroy(): Promise<void>;
|
|
162
156
|
/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
|
|
163
|
-
private
|
|
157
|
+
private waitForFreeSlot;
|
|
164
158
|
/**
|
|
165
159
|
* Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
|
|
166
160
|
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
|
|
167
161
|
* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
|
|
168
162
|
*/
|
|
169
|
-
private
|
|
163
|
+
private nextCapacityChange;
|
|
170
164
|
}
|
package/remote-browser-pool.js
CHANGED
|
@@ -7,18 +7,18 @@ import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
|
7
7
|
* {@link RemoteConnection} so it can be injected into a plugin.
|
|
8
8
|
*/
|
|
9
9
|
class RemoteSessionRegistry {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
#sessions = new Map();
|
|
11
|
+
#nextToken = 0;
|
|
12
|
+
#endpoint;
|
|
13
|
+
#onRelease;
|
|
14
|
+
#log;
|
|
15
15
|
constructor(endpoint, onRelease, log) {
|
|
16
|
-
this
|
|
17
|
-
this
|
|
18
|
-
this
|
|
16
|
+
this.#endpoint = endpoint;
|
|
17
|
+
this.#onRelease = onRelease;
|
|
18
|
+
this.#log = log;
|
|
19
19
|
}
|
|
20
20
|
async resolve(options) {
|
|
21
|
-
const resolved = typeof this
|
|
21
|
+
const resolved = typeof this.#endpoint === 'function' ? await this.#endpoint(options) : this.#endpoint;
|
|
22
22
|
let result;
|
|
23
23
|
if (typeof resolved === 'string') {
|
|
24
24
|
if (!resolved)
|
|
@@ -31,30 +31,30 @@ class RemoteSessionRegistry {
|
|
|
31
31
|
else {
|
|
32
32
|
result = resolved;
|
|
33
33
|
}
|
|
34
|
-
const token = this
|
|
35
|
-
this
|
|
34
|
+
const token = this.#nextToken++;
|
|
35
|
+
this.#sessions.set(token, { url: result.url, context: result.context, released: false });
|
|
36
36
|
return { url: result.url, token };
|
|
37
37
|
}
|
|
38
38
|
async release(token) {
|
|
39
|
-
const session = this
|
|
39
|
+
const session = this.#sessions.get(token);
|
|
40
40
|
// Release at most once per session — guards a close()/teardown race (the `released` flag is set
|
|
41
41
|
// synchronously before the awaited onRelease, so releaseAll() can't double-fire an in-flight release).
|
|
42
42
|
if (!session || session.released)
|
|
43
43
|
return;
|
|
44
44
|
session.released = true;
|
|
45
45
|
try {
|
|
46
|
-
await this
|
|
46
|
+
await this.#onRelease?.({ endpoint: session.url, context: session.context });
|
|
47
47
|
}
|
|
48
48
|
catch (err) {
|
|
49
|
-
this
|
|
49
|
+
this.#log.warning('Remote browser release() failed.', { error: err?.message });
|
|
50
50
|
}
|
|
51
51
|
finally {
|
|
52
|
-
this
|
|
52
|
+
this.#sessions.delete(token);
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
/** Releases every session that is still open. Called on pool teardown so no remote session leaks. */
|
|
56
56
|
async releaseAll() {
|
|
57
|
-
await Promise.all([...this
|
|
57
|
+
await Promise.all([...this.#sessions.keys()].map(async (token) => this.release(token)));
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
@@ -92,16 +92,16 @@ export class RemoteBrowserPool {
|
|
|
92
92
|
/** The wrapped pool that performs the remote connections and serves pages. */
|
|
93
93
|
browserPool;
|
|
94
94
|
/** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
|
|
95
|
-
pool;
|
|
96
|
-
registry;
|
|
97
|
-
slotPollIntervalMillis;
|
|
98
|
-
log;
|
|
95
|
+
#pool;
|
|
96
|
+
#registry;
|
|
97
|
+
#slotPollIntervalMillis;
|
|
98
|
+
#log;
|
|
99
99
|
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
|
|
100
|
-
|
|
100
|
+
#capacityChange;
|
|
101
101
|
constructor(options) {
|
|
102
102
|
const { browserPlugins, endpoint, release, maxOpenBrowsers, connection = {}, browserPoolOptions = {}, slotPollIntervalMillis = 500, } = options;
|
|
103
|
-
this
|
|
104
|
-
this
|
|
103
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
|
|
104
|
+
this.#slotPollIntervalMillis = slotPollIntervalMillis;
|
|
105
105
|
// A RemoteBrowserProvider carries its own endpoint, release, and maxOpenBrowsers.
|
|
106
106
|
const provider = endpoint instanceof RemoteBrowserProvider ? endpoint : undefined;
|
|
107
107
|
const resolvedEndpoint = provider
|
|
@@ -111,20 +111,20 @@ export class RemoteBrowserPool {
|
|
|
111
111
|
? ({ context }) => provider.release(context)
|
|
112
112
|
: release;
|
|
113
113
|
const resolvedMax = maxOpenBrowsers ?? provider?.maxOpenBrowsers;
|
|
114
|
-
this
|
|
114
|
+
this.#registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.#log);
|
|
115
115
|
// Wire every plugin for remote connection.
|
|
116
116
|
for (const plugin of browserPlugins) {
|
|
117
|
-
plugin.useRemoteConnection(this
|
|
117
|
+
plugin.useRemoteConnection(this.#registry, connection);
|
|
118
118
|
}
|
|
119
119
|
this.browserPool = new BrowserPool({ ...browserPoolOptions, browserPlugins });
|
|
120
|
-
this
|
|
120
|
+
this.#pool = this.browserPool;
|
|
121
121
|
// Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed
|
|
122
122
|
// kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.
|
|
123
123
|
this.browserPool.on("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, (controller) => {
|
|
124
124
|
controller.once("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
|
|
125
|
-
const token = controller.launchContext.
|
|
125
|
+
const token = controller.launchContext.remoteToken;
|
|
126
126
|
if (token !== undefined)
|
|
127
|
-
void this
|
|
127
|
+
void this.#registry.release(token);
|
|
128
128
|
});
|
|
129
129
|
});
|
|
130
130
|
if (resolvedMax !== undefined) {
|
|
@@ -143,28 +143,28 @@ export class RemoteBrowserPool {
|
|
|
143
143
|
* allows it (either a new browser slot is free, or an active browser still has page capacity).
|
|
144
144
|
*/
|
|
145
145
|
async newPage(options) {
|
|
146
|
-
await this.
|
|
147
|
-
return this
|
|
146
|
+
await this.waitForFreeSlot();
|
|
147
|
+
return this.#pool.newPage(options);
|
|
148
148
|
}
|
|
149
149
|
async closePage(page, options) {
|
|
150
|
-
return this
|
|
150
|
+
return this.#pool.closePage(page, options);
|
|
151
151
|
}
|
|
152
152
|
async extractPageState(page) {
|
|
153
|
-
return this
|
|
153
|
+
return this.#pool.extractPageState(page);
|
|
154
154
|
}
|
|
155
155
|
async injectPageState(page, state) {
|
|
156
|
-
return this
|
|
156
|
+
return this.#pool.injectPageState(page, state);
|
|
157
157
|
}
|
|
158
158
|
/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
|
|
159
159
|
async destroy() {
|
|
160
160
|
await this.browserPool.destroy();
|
|
161
161
|
// Backstop: release any sessions whose browser never emitted a close (e.g. dropped on teardown).
|
|
162
|
-
await this
|
|
162
|
+
await this.#registry.releaseAll();
|
|
163
163
|
}
|
|
164
164
|
/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
|
|
165
|
-
async
|
|
165
|
+
async waitForFreeSlot() {
|
|
166
166
|
while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {
|
|
167
|
-
await this.
|
|
167
|
+
await this.nextCapacityChange();
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
170
|
/**
|
|
@@ -172,20 +172,20 @@ export class RemoteBrowserPool {
|
|
|
172
172
|
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
|
|
173
173
|
* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
|
|
174
174
|
*/
|
|
175
|
-
|
|
176
|
-
this
|
|
175
|
+
nextCapacityChange() {
|
|
176
|
+
this.#capacityChange ??= new Promise((resolve) => {
|
|
177
177
|
const done = () => {
|
|
178
178
|
clearTimeout(timer);
|
|
179
179
|
this.browserPool.off("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
|
|
180
180
|
this.browserPool.off("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
|
|
181
|
-
this
|
|
181
|
+
this.#capacityChange = undefined;
|
|
182
182
|
resolve();
|
|
183
183
|
};
|
|
184
|
-
const timer = setTimeout(done, this
|
|
184
|
+
const timer = setTimeout(done, this.#slotPollIntervalMillis);
|
|
185
185
|
timer.unref?.();
|
|
186
186
|
this.browserPool.once("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
|
|
187
187
|
this.browserPool.once("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
|
|
188
188
|
});
|
|
189
|
-
return this
|
|
189
|
+
return this.#capacityChange;
|
|
190
190
|
}
|
|
191
191
|
}
|