@crawlee/browser-pool 4.0.0-beta.1 → 4.0.0-beta.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/abstract-classes/browser-controller.d.ts +69 -10
- package/abstract-classes/browser-controller.js +8 -11
- package/abstract-classes/browser-plugin.d.ts +44 -11
- package/abstract-classes/browser-plugin.js +65 -7
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +88 -14
- package/browser-pool.js +151 -28
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +0 -1
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +24 -9
- package/fingerprinting/types.d.ts +0 -1
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +0 -1
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +19 -3
- package/launch-context.js +12 -4
- package/package.json +8 -9
- package/playwright/playwright-browser.d.ts +0 -1
- package/playwright/playwright-browser.js +0 -1
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +20 -10
- package/playwright/playwright-plugin.d.ts +6 -3
- package/playwright/playwright-plugin.js +27 -5
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +3 -1
- package/puppeteer/puppeteer-plugin.js +81 -44
- package/remote-browser-pool.d.ts +170 -0
- package/remote-browser-pool.js +191 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/browser-pool.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
|
|
1
2
|
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
|
|
2
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
3
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
@@ -246,9 +247,10 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
|
|
|
246
247
|
* });
|
|
247
248
|
* ```
|
|
248
249
|
*/
|
|
249
|
-
export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
|
|
250
|
+
export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
|
|
250
251
|
browserPlugins: BrowserPlugins;
|
|
251
252
|
maxOpenPagesPerBrowser: number;
|
|
253
|
+
maxOpenBrowsers: number;
|
|
252
254
|
retireBrowserAfterPageCount: number;
|
|
253
255
|
operationTimeoutMillis: number;
|
|
254
256
|
closeInactiveBrowserAfterMillis: number;
|
|
@@ -263,6 +265,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
263
265
|
pageCounter: number;
|
|
264
266
|
pages: Map<string, PageReturn>;
|
|
265
267
|
pageIds: WeakMap<PageReturn, string>;
|
|
268
|
+
startingBrowserControllers: Set<BrowserControllerReturn>;
|
|
266
269
|
activeBrowserControllers: Set<BrowserControllerReturn>;
|
|
267
270
|
retiredBrowserControllers: Set<BrowserControllerReturn>;
|
|
268
271
|
pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
|
|
@@ -272,11 +275,24 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
272
275
|
private browserKillerInterval?;
|
|
273
276
|
private browserRetireInterval?;
|
|
274
277
|
private limiter;
|
|
278
|
+
private log;
|
|
275
279
|
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
|
276
280
|
/**
|
|
277
281
|
* Opens a new page in one of the running browsers or launches
|
|
278
282
|
* a new browser and opens a page there, if no browsers are active,
|
|
279
283
|
* or their page limits have been exceeded.
|
|
284
|
+
*
|
|
285
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
286
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
287
|
+
* fingerprinting is enabled) and reads
|
|
288
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
289
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
290
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
291
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
292
|
+
*
|
|
293
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
294
|
+
* properties are consumed — cookie and header injection remain the
|
|
295
|
+
* crawler's responsibility.
|
|
280
296
|
*/
|
|
281
297
|
newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
|
|
282
298
|
/**
|
|
@@ -346,6 +362,43 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
346
362
|
* closed after all its pages are closed.
|
|
347
363
|
*/
|
|
348
364
|
retireBrowserByPage(page: PageReturn): void;
|
|
365
|
+
/**
|
|
366
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
367
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
368
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
369
|
+
* storage, etc.) cannot leak into future sessions.
|
|
370
|
+
*
|
|
371
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
372
|
+
*
|
|
373
|
+
* @param page The page to release.
|
|
374
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
375
|
+
*/
|
|
376
|
+
closePage(page: PageReturn, options?: {
|
|
377
|
+
error?: Error;
|
|
378
|
+
}): Promise<void>;
|
|
379
|
+
/**
|
|
380
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
381
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
382
|
+
* no longer associated with a controller.
|
|
383
|
+
*
|
|
384
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
385
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
386
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
387
|
+
* include those set by any sibling page sharing the same browser.
|
|
388
|
+
*/
|
|
389
|
+
extractPageState(page: PageReturn): Promise<PageState>;
|
|
390
|
+
/**
|
|
391
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
392
|
+
*
|
|
393
|
+
* No-op when the page is no longer associated with a controller.
|
|
394
|
+
*
|
|
395
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
396
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
397
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
398
|
+
* browser share a single context, so injected cookies are visible to every
|
|
399
|
+
* page served by that browser.
|
|
400
|
+
*/
|
|
401
|
+
injectPageState(page: PageReturn, state: PageState): Promise<void>;
|
|
349
402
|
/**
|
|
350
403
|
* Removes all active browsers from the pool. The browsers will be
|
|
351
404
|
* closed after all their pages are closed.
|
|
@@ -373,15 +426,45 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
373
426
|
private _overridePageClose;
|
|
374
427
|
private _executeHooks;
|
|
375
428
|
private _closeRetiredBrowserWithNoPages;
|
|
429
|
+
/**
|
|
430
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
431
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
432
|
+
*/
|
|
433
|
+
hasFreeBrowserSlot(): boolean;
|
|
434
|
+
/**
|
|
435
|
+
* Returns `true` if any active browser has room for another page.
|
|
436
|
+
*/
|
|
437
|
+
hasActiveBrowserWithFreeCapacity(): boolean;
|
|
376
438
|
private _initializeFingerprinting;
|
|
377
439
|
private _addFingerprintHooks;
|
|
378
440
|
}
|
|
379
|
-
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> {
|
|
441
|
+
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
|
|
442
|
+
/**
|
|
443
|
+
* The proxy URL the pool uses internally to route the page: it keys browser
|
|
444
|
+
* reuse (with `browserPerProxy`, only a browser already on this proxy is
|
|
445
|
+
* reused), configures the launched browser, and is applied to incognito
|
|
446
|
+
* pages. When omitted, it is derived from the
|
|
447
|
+
* {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
|
|
448
|
+
* here takes precedence.
|
|
449
|
+
*
|
|
450
|
+
* This is an implementation detail of the built-in `BrowserPool`'s proxy
|
|
451
|
+
* handling and is intentionally not part of the {@link IBrowserPool}
|
|
452
|
+
* contract — through that interface the proxy is supplied via the session.
|
|
453
|
+
*/
|
|
454
|
+
proxyUrl?: string;
|
|
380
455
|
/**
|
|
381
|
-
*
|
|
382
|
-
*
|
|
456
|
+
* Disable TLS certificate verification for MITM proxies. Applied both when
|
|
457
|
+
* launching a new browser and when creating a page in an existing one. When
|
|
458
|
+
* omitted, it is derived from the
|
|
459
|
+
* {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
|
|
460
|
+
* here takes precedence.
|
|
461
|
+
*
|
|
462
|
+
* This is an implementation detail of the built-in `BrowserPool` and is
|
|
463
|
+
* intentionally not part of the {@link IBrowserPool} contract — through
|
|
464
|
+
* that interface, configure it via the session's `proxyInfo` or through the
|
|
465
|
+
* browser's `launchOptions`.
|
|
383
466
|
*/
|
|
384
|
-
|
|
467
|
+
ignoreTlsErrors?: boolean;
|
|
385
468
|
/**
|
|
386
469
|
* Some libraries (Playwright) allow you to open new pages with specific
|
|
387
470
|
* options. Use this property to set those options.
|
|
@@ -396,14 +479,6 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
|
|
|
396
479
|
* see the `newPageInNewBrowser` function.
|
|
397
480
|
*/
|
|
398
481
|
browserPlugin?: BP;
|
|
399
|
-
/**
|
|
400
|
-
* Proxy URL.
|
|
401
|
-
*/
|
|
402
|
-
proxyUrl?: string;
|
|
403
|
-
/**
|
|
404
|
-
* Proxy tier.
|
|
405
|
-
*/
|
|
406
|
-
proxyTier?: number;
|
|
407
482
|
}
|
|
408
483
|
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
|
|
409
484
|
/**
|
|
@@ -434,4 +509,3 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
|
|
|
434
509
|
*/
|
|
435
510
|
launchOptions?: BP['launchOptions'];
|
|
436
511
|
}
|
|
437
|
-
//# sourceMappingURL=browser-pool.d.ts.map
|
package/browser-pool.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
1
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
5
|
import { nanoid } from 'nanoid';
|
|
@@ -7,7 +9,7 @@ import QuickLRU from 'quick-lru';
|
|
|
7
9
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
8
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
9
11
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
|
|
12
|
+
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
11
13
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
14
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
13
15
|
/**
|
|
@@ -60,6 +62,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
|
60
62
|
export class BrowserPool extends TypedEmitter {
|
|
61
63
|
browserPlugins;
|
|
62
64
|
maxOpenPagesPerBrowser;
|
|
65
|
+
maxOpenBrowsers;
|
|
63
66
|
retireBrowserAfterPageCount;
|
|
64
67
|
operationTimeoutMillis;
|
|
65
68
|
closeInactiveBrowserAfterMillis;
|
|
@@ -74,6 +77,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
74
77
|
pageCounter = 0;
|
|
75
78
|
pages = new Map();
|
|
76
79
|
pageIds = new WeakMap();
|
|
80
|
+
startingBrowserControllers = new Set();
|
|
77
81
|
activeBrowserControllers = new Set();
|
|
78
82
|
retiredBrowserControllers = new Set();
|
|
79
83
|
pageToBrowserController = new WeakMap();
|
|
@@ -83,8 +87,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
83
87
|
browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
84
88
|
browserRetireInterval;
|
|
85
89
|
limiter = pLimit(1);
|
|
90
|
+
log;
|
|
86
91
|
constructor(options) {
|
|
87
92
|
super();
|
|
93
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
88
94
|
this.browserKillerInterval.unref();
|
|
89
95
|
ow(options, ow.object.exactShape({
|
|
90
96
|
browserPlugins: ow.array.minLength(1),
|
|
@@ -114,6 +120,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
114
120
|
}
|
|
115
121
|
this.browserPlugins = browserPlugins;
|
|
116
122
|
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
123
|
+
this.maxOpenBrowsers = Infinity;
|
|
117
124
|
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
118
125
|
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
119
126
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
@@ -142,23 +149,46 @@ export class BrowserPool extends TypedEmitter {
|
|
|
142
149
|
* Opens a new page in one of the running browsers or launches
|
|
143
150
|
* a new browser and opens a page there, if no browsers are active,
|
|
144
151
|
* or their page limits have been exceeded.
|
|
152
|
+
*
|
|
153
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
154
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
155
|
+
* fingerprinting is enabled) and reads
|
|
156
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
157
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
158
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
159
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
160
|
+
*
|
|
161
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
162
|
+
* properties are consumed — cookie and header injection remain the
|
|
163
|
+
* crawler's responsibility.
|
|
145
164
|
*/
|
|
146
165
|
async newPage(options = {}) {
|
|
147
|
-
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl,
|
|
166
|
+
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
148
167
|
if (this.pages.has(id)) {
|
|
149
168
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
150
169
|
}
|
|
151
170
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
152
171
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
153
172
|
}
|
|
173
|
+
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
174
|
+
// otherwise resumes queued callbacks in the previous task's
|
|
175
|
+
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
176
|
+
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
177
|
+
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
178
|
+
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
179
|
+
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
154
180
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
155
|
-
return this.limiter(async () => {
|
|
156
|
-
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, {
|
|
181
|
+
return this.limiter(AsyncResource.bind(async () => {
|
|
182
|
+
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
157
183
|
if (!browserController)
|
|
158
|
-
browserController = await this._launchBrowser(id, {
|
|
184
|
+
browserController = await this._launchBrowser(id, {
|
|
185
|
+
browserPlugin,
|
|
186
|
+
proxyUrl,
|
|
187
|
+
ignoreTlsErrors,
|
|
188
|
+
});
|
|
159
189
|
tryCancel();
|
|
160
|
-
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
|
|
161
|
-
});
|
|
190
|
+
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
191
|
+
}));
|
|
162
192
|
}
|
|
163
193
|
/**
|
|
164
194
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -238,7 +268,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
238
268
|
getPageId(page) {
|
|
239
269
|
return this.pageIds.get(page);
|
|
240
270
|
}
|
|
241
|
-
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
|
|
271
|
+
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
242
272
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
243
273
|
// It's not ideal though, we need to come up with a better API.
|
|
244
274
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -247,6 +277,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
247
277
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
248
278
|
if (finalPageOptions) {
|
|
249
279
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
280
|
+
if (ignoreTlsErrors) {
|
|
281
|
+
Object.assign(finalPageOptions, {
|
|
282
|
+
ignoreHTTPSErrors: true,
|
|
283
|
+
acceptInsecureCerts: true,
|
|
284
|
+
});
|
|
285
|
+
}
|
|
250
286
|
}
|
|
251
287
|
await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
252
288
|
tryCancel();
|
|
@@ -278,11 +314,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
278
314
|
*
|
|
279
315
|
*/
|
|
280
316
|
retireBrowserController(browserController) {
|
|
281
|
-
const
|
|
317
|
+
const isStarting = this.startingBrowserControllers.has(browserController);
|
|
318
|
+
const isActive = this.activeBrowserControllers.has(browserController);
|
|
319
|
+
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
282
320
|
if (hasBeenRetiredOrKilled)
|
|
283
321
|
return;
|
|
284
322
|
this.retiredBrowserControllers.add(browserController);
|
|
285
323
|
this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
|
|
324
|
+
this.startingBrowserControllers.delete(browserController);
|
|
286
325
|
this.activeBrowserControllers.delete(browserController);
|
|
287
326
|
}
|
|
288
327
|
/**
|
|
@@ -294,12 +333,65 @@ export class BrowserPool extends TypedEmitter {
|
|
|
294
333
|
if (browserController)
|
|
295
334
|
this.retireBrowserController(browserController);
|
|
296
335
|
}
|
|
336
|
+
/**
|
|
337
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
338
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
339
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
340
|
+
* storage, etc.) cannot leak into future sessions.
|
|
341
|
+
*
|
|
342
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
343
|
+
*
|
|
344
|
+
* @param page The page to release.
|
|
345
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
346
|
+
*/
|
|
347
|
+
async closePage(page, options) {
|
|
348
|
+
if (options?.error instanceof SessionError) {
|
|
349
|
+
this.retireBrowserByPage(page);
|
|
350
|
+
}
|
|
351
|
+
// Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
|
|
352
|
+
await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
356
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
357
|
+
* no longer associated with a controller.
|
|
358
|
+
*
|
|
359
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
360
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
361
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
362
|
+
* include those set by any sibling page sharing the same browser.
|
|
363
|
+
*/
|
|
364
|
+
async extractPageState(page) {
|
|
365
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
366
|
+
if (!controller) {
|
|
367
|
+
return { cookies: [] };
|
|
368
|
+
}
|
|
369
|
+
return { cookies: await controller.getCookies(page) };
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
373
|
+
*
|
|
374
|
+
* No-op when the page is no longer associated with a controller.
|
|
375
|
+
*
|
|
376
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
377
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
378
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
379
|
+
* browser share a single context, so injected cookies are visible to every
|
|
380
|
+
* page served by that browser.
|
|
381
|
+
*/
|
|
382
|
+
async injectPageState(page, state) {
|
|
383
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
384
|
+
if (!controller) {
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
await controller.setCookies(page, state.cookies);
|
|
388
|
+
}
|
|
297
389
|
/**
|
|
298
390
|
* Removes all active browsers from the pool. The browsers will be
|
|
299
391
|
* closed after all their pages are closed.
|
|
300
392
|
*/
|
|
301
393
|
retireAllBrowsers() {
|
|
302
|
-
this.activeBrowserControllers.forEach((controller) => {
|
|
394
|
+
[...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
303
395
|
this.retireBrowserController(controller);
|
|
304
396
|
});
|
|
305
397
|
}
|
|
@@ -326,23 +418,36 @@ export class BrowserPool extends TypedEmitter {
|
|
|
326
418
|
this._teardown();
|
|
327
419
|
}
|
|
328
420
|
_teardown() {
|
|
421
|
+
this.startingBrowserControllers.clear();
|
|
329
422
|
this.activeBrowserControllers.clear();
|
|
330
423
|
this.retiredBrowserControllers.clear();
|
|
331
424
|
this.removeAllListeners();
|
|
332
425
|
}
|
|
333
426
|
_getAllBrowserControllers() {
|
|
334
|
-
return new Set([
|
|
427
|
+
return new Set([
|
|
428
|
+
...this.startingBrowserControllers,
|
|
429
|
+
...this.activeBrowserControllers,
|
|
430
|
+
...this.retiredBrowserControllers,
|
|
431
|
+
]);
|
|
335
432
|
}
|
|
336
433
|
async _launchBrowser(pageId, options) {
|
|
337
|
-
const { browserPlugin, launchOptions,
|
|
434
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
338
435
|
const browserController = browserPlugin.createController();
|
|
339
|
-
this.
|
|
436
|
+
this.startingBrowserControllers.add(browserController);
|
|
340
437
|
const launchContext = browserPlugin.createLaunchContext({
|
|
341
438
|
id: pageId,
|
|
342
439
|
launchOptions,
|
|
343
|
-
proxyTier,
|
|
344
440
|
proxyUrl,
|
|
345
441
|
});
|
|
442
|
+
// Disable SSL verification for MITM proxies
|
|
443
|
+
if (ignoreTlsErrors) {
|
|
444
|
+
/**
|
|
445
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
446
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
447
|
+
*/
|
|
448
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
449
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
450
|
+
}
|
|
346
451
|
try {
|
|
347
452
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
348
453
|
// because otherwise it would be stuck in limbo without a browser.
|
|
@@ -353,11 +458,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
353
458
|
browserController.assignBrowser(browser, launchContext);
|
|
354
459
|
}
|
|
355
460
|
catch (err) {
|
|
356
|
-
this.
|
|
461
|
+
this.startingBrowserControllers.delete(browserController);
|
|
357
462
|
throw err;
|
|
358
463
|
}
|
|
359
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
360
|
-
browserController.proxyTier = proxyTier;
|
|
464
|
+
this.log.debug('Launched new browser.', { id: browserController.id });
|
|
361
465
|
browserController.proxyUrl = proxyUrl;
|
|
362
466
|
try {
|
|
363
467
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
@@ -365,9 +469,9 @@ export class BrowserPool extends TypedEmitter {
|
|
|
365
469
|
await this._executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
366
470
|
}
|
|
367
471
|
catch (err) {
|
|
368
|
-
this.
|
|
472
|
+
this.startingBrowserControllers.delete(browserController);
|
|
369
473
|
browserController.close().catch((closeErr) => {
|
|
370
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
474
|
+
this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
371
475
|
id: browserController.id,
|
|
372
476
|
});
|
|
373
477
|
});
|
|
@@ -375,6 +479,8 @@ export class BrowserPool extends TypedEmitter {
|
|
|
375
479
|
}
|
|
376
480
|
tryCancel();
|
|
377
481
|
browserController.activate();
|
|
482
|
+
this.startingBrowserControllers.delete(browserController);
|
|
483
|
+
this.activeBrowserControllers.add(browserController);
|
|
378
484
|
this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
|
|
379
485
|
return browserController;
|
|
380
486
|
}
|
|
@@ -392,13 +498,11 @@ export class BrowserPool extends TypedEmitter {
|
|
|
392
498
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
393
499
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
394
500
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
395
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
396
501
|
return (isCorrectPlugin &&
|
|
397
502
|
hasCapacity &&
|
|
398
|
-
(
|
|
399
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
503
|
+
(!controller.launchContext.browserPerProxy ||
|
|
400
504
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
401
|
-
(!options?.proxyUrl && !
|
|
505
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
402
506
|
});
|
|
403
507
|
}
|
|
404
508
|
async _closeInactiveRetiredBrowsers() {
|
|
@@ -409,14 +513,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
409
513
|
const isBrowserEmpty = controller.activePages === 0;
|
|
410
514
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
411
515
|
const { id } = controller;
|
|
412
|
-
log.debug('Closing retired browser.', { id });
|
|
516
|
+
this.log.debug('Closing retired browser.', { id });
|
|
413
517
|
await controller.close();
|
|
414
518
|
this.retiredBrowserControllers.delete(controller);
|
|
415
519
|
closedBrowserIds.push(id);
|
|
416
520
|
}
|
|
417
521
|
}
|
|
418
522
|
if (closedBrowserIds.length) {
|
|
419
|
-
log.debug('Closed retired browsers.', {
|
|
523
|
+
this.log.debug('Closed retired browsers.', {
|
|
420
524
|
count: closedBrowserIds.length,
|
|
421
525
|
closedBrowserIds,
|
|
422
526
|
});
|
|
@@ -429,7 +533,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
429
533
|
page.close = async (...args) => {
|
|
430
534
|
await this._executeHooks(this.prePageCloseHooks, page, browserController);
|
|
431
535
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
432
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
536
|
+
this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
433
537
|
});
|
|
434
538
|
await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
435
539
|
this.pages.delete(pageId);
|
|
@@ -447,13 +551,33 @@ export class BrowserPool extends TypedEmitter {
|
|
|
447
551
|
// Run this with a delay, otherwise page.close()
|
|
448
552
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
449
553
|
setTimeout(() => {
|
|
450
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
554
|
+
this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
451
555
|
void browserController.close().finally(() => {
|
|
452
556
|
this.retiredBrowserControllers.delete(browserController);
|
|
453
557
|
});
|
|
454
558
|
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
|
|
455
559
|
}
|
|
456
560
|
}
|
|
561
|
+
/**
|
|
562
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
563
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
564
|
+
*/
|
|
565
|
+
hasFreeBrowserSlot() {
|
|
566
|
+
const total = this.startingBrowserControllers.size +
|
|
567
|
+
this.activeBrowserControllers.size +
|
|
568
|
+
this.retiredBrowserControllers.size;
|
|
569
|
+
return total < this.maxOpenBrowsers;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Returns `true` if any active browser has room for another page.
|
|
573
|
+
*/
|
|
574
|
+
hasActiveBrowserWithFreeCapacity() {
|
|
575
|
+
for (const controller of this.activeBrowserControllers) {
|
|
576
|
+
if (controller.activePages < this.maxOpenPagesPerBrowser)
|
|
577
|
+
return true;
|
|
578
|
+
}
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
457
581
|
_initializeFingerprinting() {
|
|
458
582
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
459
583
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
@@ -474,4 +598,3 @@ export class BrowserPool extends TypedEmitter {
|
|
|
474
598
|
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
475
599
|
}
|
|
476
600
|
}
|
|
477
|
-
//# sourceMappingURL=browser-pool.js.map
|
package/events.d.ts
CHANGED
package/events.js
CHANGED
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -1,27 +1,39 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
3
|
import { getGeneratorDefaultOptions } from './utils.js';
|
|
4
|
+
function applySessionHints(base, fingerprint) {
|
|
5
|
+
if (!fingerprint)
|
|
6
|
+
return base;
|
|
7
|
+
return {
|
|
8
|
+
...base,
|
|
9
|
+
...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
|
|
10
|
+
...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
|
|
11
|
+
...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
/**
|
|
5
15
|
* @internal
|
|
6
16
|
*/
|
|
7
17
|
export function createFingerprintPreLaunchHook(browserPool) {
|
|
8
18
|
const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
|
|
9
19
|
return (_pageId, launchContext) => {
|
|
20
|
+
// Remote browsers may have their own fingerprinting — skip local fingerprint injection
|
|
21
|
+
if (launchContext.isRemote)
|
|
22
|
+
return;
|
|
10
23
|
const { useIncognitoPages } = launchContext;
|
|
11
|
-
const
|
|
24
|
+
const session = launchContext.session;
|
|
25
|
+
const cacheKey = session?.id ?? launchContext.proxyUrl;
|
|
12
26
|
const { launchOptions } = launchContext;
|
|
13
|
-
// If no options are passed we try to pass best default options as possible to match browser and OS.
|
|
14
|
-
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
15
27
|
let fingerprint;
|
|
16
28
|
if (cacheKey && fingerprintCache?.has(cacheKey)) {
|
|
17
29
|
fingerprint = fingerprintCache.get(cacheKey);
|
|
18
30
|
}
|
|
19
|
-
else if (cacheKey) {
|
|
20
|
-
fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
|
|
21
|
-
fingerprintCache?.set(cacheKey, fingerprint);
|
|
22
|
-
}
|
|
23
31
|
else {
|
|
24
|
-
|
|
32
|
+
const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
33
|
+
const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
|
|
34
|
+
fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
|
|
35
|
+
if (cacheKey)
|
|
36
|
+
fingerprintCache?.set(cacheKey, fingerprint);
|
|
25
37
|
}
|
|
26
38
|
launchContext.extend({ fingerprint });
|
|
27
39
|
if (useIncognitoPages) {
|
|
@@ -41,6 +53,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
41
53
|
export function createPrePageCreateHook() {
|
|
42
54
|
return (_pageId, browserController, pageOptions) => {
|
|
43
55
|
const { launchContext, browserPlugin } = browserController;
|
|
56
|
+
if (launchContext.isRemote)
|
|
57
|
+
return;
|
|
44
58
|
const { fingerprint } = launchContext.fingerprint;
|
|
45
59
|
if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
|
|
46
60
|
pageOptions.userAgent ??= fingerprint.navigator.userAgent;
|
|
@@ -57,6 +71,8 @@ export function createPrePageCreateHook() {
|
|
|
57
71
|
export function createPostPageCreateHook(fingerprintInjector) {
|
|
58
72
|
return async (page, browserController) => {
|
|
59
73
|
const { browserPlugin, launchContext } = browserController;
|
|
74
|
+
if (launchContext.isRemote)
|
|
75
|
+
return;
|
|
60
76
|
const fingerprint = launchContext.fingerprint;
|
|
61
77
|
// TODO this will require refactoring, we should use common API instead of branching based on plugin type,
|
|
62
78
|
// and there should be no public methods specific to some browser.
|
|
@@ -79,4 +95,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
|
|
|
79
95
|
}
|
|
80
96
|
};
|
|
81
97
|
}
|
|
82
|
-
//# sourceMappingURL=hooks.js.map
|
package/fingerprinting/types.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
1
|
import type { LaunchContext } from '../launch-context.js';
|
|
2
2
|
import type { FingerprintGeneratorOptions } from './types.js';
|
|
3
3
|
export declare const getGeneratorDefaultOptions: (launchContext: LaunchContext) => FingerprintGeneratorOptions;
|
|
4
|
-
//# sourceMappingURL=utils.d.ts.map
|
package/fingerprinting/utils.js
CHANGED