@crawlee/browser-pool 4.0.0-beta.16 → 4.0.0-beta.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/abstract-classes/browser-controller.d.ts +74 -14
- package/abstract-classes/browser-controller.js +28 -24
- package/abstract-classes/browser-plugin.d.ts +48 -15
- package/abstract-classes/browser-plugin.js +69 -11
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +100 -28
- package/browser-pool.js +199 -79
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +2 -3
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +3 -4
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +20 -5
- package/launch-context.js +21 -10
- package/package.json +9 -9
- package/playwright/playwright-browser.d.ts +2 -6
- package/playwright/playwright-browser.js +16 -17
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +6 -3
- package/playwright/playwright-plugin.d.ts +10 -7
- package/playwright/playwright-plugin.js +40 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +5 -3
- package/puppeteer/puppeteer-plugin.js +83 -46
- package/remote-browser-pool.d.ts +165 -0
- package/remote-browser-pool.js +195 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/browser-pool.js
CHANGED
|
@@ -1,15 +1,35 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
1
4
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
5
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
6
|
import { nanoid } from 'nanoid';
|
|
4
|
-
import ow from 'ow';
|
|
5
7
|
import pLimit from 'p-limit';
|
|
6
8
|
import QuickLRU from 'quick-lru';
|
|
7
9
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
10
|
+
import { z } from 'zod';
|
|
8
11
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
12
|
+
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
9
13
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
|
|
14
|
+
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
11
15
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
16
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
17
|
+
const browserPoolOptionsSchema = z.strictObject({
|
|
18
|
+
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
19
|
+
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
20
|
+
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
21
|
+
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
22
|
+
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
23
|
+
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
|
|
24
|
+
preLaunchHooks: schemas.anyArray.default(() => []),
|
|
25
|
+
postLaunchHooks: schemas.anyArray.default(() => []),
|
|
26
|
+
prePageCreateHooks: schemas.anyArray.default(() => []),
|
|
27
|
+
postPageCreateHooks: schemas.anyArray.default(() => []),
|
|
28
|
+
prePageCloseHooks: schemas.anyArray.default(() => []),
|
|
29
|
+
postPageCloseHooks: schemas.anyArray.default(() => []),
|
|
30
|
+
useFingerprints: z.boolean().default(true),
|
|
31
|
+
fingerprintOptions: schemas.anyObject.default(() => ({})),
|
|
32
|
+
});
|
|
13
33
|
/**
|
|
14
34
|
* The `BrowserPool` class is the most important class of the `browser-pool` module.
|
|
15
35
|
* It manages opening and closing of browsers and their pages and its constructor
|
|
@@ -60,6 +80,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
|
60
80
|
export class BrowserPool extends TypedEmitter {
|
|
61
81
|
browserPlugins;
|
|
62
82
|
maxOpenPagesPerBrowser;
|
|
83
|
+
maxOpenBrowsers;
|
|
63
84
|
retireBrowserAfterPageCount;
|
|
64
85
|
operationTimeoutMillis;
|
|
65
86
|
closeInactiveBrowserAfterMillis;
|
|
@@ -81,29 +102,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
81
102
|
fingerprintInjector;
|
|
82
103
|
fingerprintGenerator;
|
|
83
104
|
fingerprintCache;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
105
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
106
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
107
|
+
#browserRetireInterval;
|
|
108
|
+
#limiter = pLimit(1);
|
|
109
|
+
#log;
|
|
87
110
|
constructor(options) {
|
|
88
111
|
super();
|
|
112
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
89
113
|
this.browserKillerInterval.unref();
|
|
90
|
-
|
|
91
|
-
browserPlugins: ow.array.minLength(1),
|
|
92
|
-
maxOpenPagesPerBrowser: ow.optional.number,
|
|
93
|
-
retireBrowserAfterPageCount: ow.optional.number,
|
|
94
|
-
operationTimeoutSecs: ow.optional.number,
|
|
95
|
-
closeInactiveBrowserAfterSecs: ow.optional.number,
|
|
96
|
-
retireInactiveBrowserAfterSecs: ow.optional.number,
|
|
97
|
-
preLaunchHooks: ow.optional.array,
|
|
98
|
-
postLaunchHooks: ow.optional.array,
|
|
99
|
-
prePageCreateHooks: ow.optional.array,
|
|
100
|
-
postPageCreateHooks: ow.optional.array,
|
|
101
|
-
prePageCloseHooks: ow.optional.array,
|
|
102
|
-
postPageCloseHooks: ow.optional.array,
|
|
103
|
-
useFingerprints: ow.optional.boolean,
|
|
104
|
-
fingerprintOptions: ow.optional.object,
|
|
105
|
-
}));
|
|
106
|
-
const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
|
|
114
|
+
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
107
115
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
108
116
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
109
117
|
const providedPlugin = browserPlugins[i];
|
|
@@ -115,18 +123,19 @@ export class BrowserPool extends TypedEmitter {
|
|
|
115
123
|
}
|
|
116
124
|
this.browserPlugins = browserPlugins;
|
|
117
125
|
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
126
|
+
this.maxOpenBrowsers = Infinity;
|
|
118
127
|
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
119
128
|
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
120
129
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
121
130
|
this.useFingerprints = useFingerprints;
|
|
122
131
|
this.fingerprintOptions = fingerprintOptions;
|
|
123
|
-
this
|
|
132
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
124
133
|
if (controller.activePages === 0 &&
|
|
125
134
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
126
135
|
this.retireBrowserController(controller);
|
|
127
136
|
}
|
|
128
137
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
129
|
-
this
|
|
138
|
+
this.#browserRetireInterval.unref();
|
|
130
139
|
// hooks
|
|
131
140
|
this.preLaunchHooks = preLaunchHooks;
|
|
132
141
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -136,30 +145,56 @@ export class BrowserPool extends TypedEmitter {
|
|
|
136
145
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
137
146
|
// fingerprinting
|
|
138
147
|
if (this.useFingerprints) {
|
|
139
|
-
this.
|
|
148
|
+
this.initializeFingerprinting();
|
|
140
149
|
}
|
|
141
150
|
}
|
|
142
151
|
/**
|
|
143
152
|
* Opens a new page in one of the running browsers or launches
|
|
144
153
|
* a new browser and opens a page there, if no browsers are active,
|
|
145
154
|
* or their page limits have been exceeded.
|
|
155
|
+
*
|
|
156
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
157
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
158
|
+
* fingerprinting is enabled) and reads
|
|
159
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
160
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
161
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
162
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
163
|
+
*
|
|
164
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
165
|
+
* properties are consumed — cookie and header injection remain the
|
|
166
|
+
* crawler's responsibility.
|
|
146
167
|
*/
|
|
147
168
|
async newPage(options = {}) {
|
|
148
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
169
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
149
170
|
if (this.pages.has(id)) {
|
|
150
171
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
151
172
|
}
|
|
152
173
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
153
174
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
154
175
|
}
|
|
176
|
+
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
177
|
+
// otherwise resumes queued callbacks in the previous task's
|
|
178
|
+
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
179
|
+
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
180
|
+
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
181
|
+
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
182
|
+
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
183
|
+
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
184
|
+
// context and run request B's storage writes inside request A's transaction.
|
|
185
|
+
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
155
186
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
156
|
-
return this
|
|
157
|
-
let browserController = this.
|
|
187
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
188
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
158
189
|
if (!browserController)
|
|
159
|
-
browserController = await this.
|
|
190
|
+
browserController = await this.launchBrowser(id, {
|
|
191
|
+
browserPlugin,
|
|
192
|
+
proxyUrl,
|
|
193
|
+
ignoreTlsErrors,
|
|
194
|
+
});
|
|
160
195
|
tryCancel();
|
|
161
|
-
return await this.
|
|
162
|
-
});
|
|
196
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
197
|
+
}));
|
|
163
198
|
}
|
|
164
199
|
/**
|
|
165
200
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -167,13 +202,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
167
202
|
* configure the new browser.
|
|
168
203
|
*/
|
|
169
204
|
async newPageInNewBrowser(options = {}) {
|
|
170
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
205
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
171
206
|
if (this.pages.has(id)) {
|
|
172
207
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
173
208
|
}
|
|
174
|
-
const browserController = await this.
|
|
209
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
175
210
|
tryCancel();
|
|
176
|
-
return await this.
|
|
211
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
177
212
|
}
|
|
178
213
|
/**
|
|
179
214
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -239,17 +274,22 @@ export class BrowserPool extends TypedEmitter {
|
|
|
239
274
|
getPageId(page) {
|
|
240
275
|
return this.pageIds.get(page);
|
|
241
276
|
}
|
|
242
|
-
async
|
|
277
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
243
278
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
244
279
|
// It's not ideal though, we need to come up with a better API.
|
|
245
|
-
|
|
246
|
-
await browserController['isActivePromise'];
|
|
280
|
+
await browserController.waitForActive();
|
|
247
281
|
tryCancel();
|
|
248
282
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
249
283
|
if (finalPageOptions) {
|
|
250
284
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
285
|
+
if (ignoreTlsErrors) {
|
|
286
|
+
Object.assign(finalPageOptions, {
|
|
287
|
+
ignoreHTTPSErrors: true,
|
|
288
|
+
acceptInsecureCerts: true,
|
|
289
|
+
});
|
|
290
|
+
}
|
|
251
291
|
}
|
|
252
|
-
await this.
|
|
292
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
253
293
|
tryCancel();
|
|
254
294
|
let page;
|
|
255
295
|
try {
|
|
@@ -262,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
262
302
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
263
303
|
this.retireBrowserController(browserController);
|
|
264
304
|
}
|
|
265
|
-
this.
|
|
305
|
+
this.overridePageClose(page);
|
|
266
306
|
}
|
|
267
307
|
catch (err) {
|
|
268
308
|
this.retireBrowserController(browserController);
|
|
269
309
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
270
310
|
}
|
|
271
|
-
await this.
|
|
311
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
272
312
|
tryCancel();
|
|
273
|
-
this.emit(
|
|
313
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
274
314
|
return page;
|
|
275
315
|
}
|
|
276
316
|
/**
|
|
@@ -285,7 +325,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
285
325
|
if (hasBeenRetiredOrKilled)
|
|
286
326
|
return;
|
|
287
327
|
this.retiredBrowserControllers.add(browserController);
|
|
288
|
-
this.emit(
|
|
328
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
289
329
|
this.startingBrowserControllers.delete(browserController);
|
|
290
330
|
this.activeBrowserControllers.delete(browserController);
|
|
291
331
|
}
|
|
@@ -298,6 +338,59 @@ export class BrowserPool extends TypedEmitter {
|
|
|
298
338
|
if (browserController)
|
|
299
339
|
this.retireBrowserController(browserController);
|
|
300
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
343
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
344
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
345
|
+
* storage, etc.) cannot leak into future sessions.
|
|
346
|
+
*
|
|
347
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
348
|
+
*
|
|
349
|
+
* @param page The page to release.
|
|
350
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
351
|
+
*/
|
|
352
|
+
async closePage(page, options) {
|
|
353
|
+
if (options?.error instanceof SessionError) {
|
|
354
|
+
this.retireBrowserByPage(page);
|
|
355
|
+
}
|
|
356
|
+
// Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
|
|
357
|
+
await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
361
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
362
|
+
* no longer associated with a controller.
|
|
363
|
+
*
|
|
364
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
365
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
366
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
367
|
+
* include those set by any sibling page sharing the same browser.
|
|
368
|
+
*/
|
|
369
|
+
async extractPageState(page) {
|
|
370
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
371
|
+
if (!controller) {
|
|
372
|
+
return { cookies: [] };
|
|
373
|
+
}
|
|
374
|
+
return { cookies: await controller.getCookies(page) };
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
378
|
+
*
|
|
379
|
+
* No-op when the page is no longer associated with a controller.
|
|
380
|
+
*
|
|
381
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
382
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
383
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
384
|
+
* browser share a single context, so injected cookies are visible to every
|
|
385
|
+
* page served by that browser.
|
|
386
|
+
*/
|
|
387
|
+
async injectPageState(page, state) {
|
|
388
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
389
|
+
if (!controller) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
await controller.setCookies(page, state.cookies);
|
|
393
|
+
}
|
|
301
394
|
/**
|
|
302
395
|
* Removes all active browsers from the pool. The browsers will be
|
|
303
396
|
* closed after all their pages are closed.
|
|
@@ -312,50 +405,61 @@ export class BrowserPool extends TypedEmitter {
|
|
|
312
405
|
* @return {Promise<void>}
|
|
313
406
|
*/
|
|
314
407
|
async closeAllBrowsers() {
|
|
315
|
-
const controllers = this.
|
|
408
|
+
const controllers = this.getAllBrowserControllers();
|
|
316
409
|
const promises = [...controllers]
|
|
317
410
|
.filter((controller) => controller.isActive)
|
|
318
411
|
.map(async (controller) => controller.close());
|
|
319
412
|
await Promise.all(promises);
|
|
320
413
|
}
|
|
414
|
+
async [Symbol.asyncDispose]() {
|
|
415
|
+
await this.destroy();
|
|
416
|
+
}
|
|
321
417
|
/**
|
|
322
418
|
* Closes all managed browsers and tears down the pool.
|
|
323
419
|
*/
|
|
324
420
|
async destroy() {
|
|
325
421
|
clearInterval(this.browserKillerInterval);
|
|
326
|
-
clearInterval(this
|
|
422
|
+
clearInterval(this.#browserRetireInterval);
|
|
327
423
|
this.browserKillerInterval = undefined;
|
|
328
|
-
this
|
|
424
|
+
this.#browserRetireInterval = undefined;
|
|
329
425
|
await this.closeAllBrowsers();
|
|
330
|
-
this.
|
|
426
|
+
this.teardown();
|
|
331
427
|
}
|
|
332
|
-
|
|
428
|
+
teardown() {
|
|
333
429
|
this.startingBrowserControllers.clear();
|
|
334
430
|
this.activeBrowserControllers.clear();
|
|
335
431
|
this.retiredBrowserControllers.clear();
|
|
336
432
|
this.removeAllListeners();
|
|
337
433
|
}
|
|
338
|
-
|
|
434
|
+
getAllBrowserControllers() {
|
|
339
435
|
return new Set([
|
|
340
436
|
...this.startingBrowserControllers,
|
|
341
437
|
...this.activeBrowserControllers,
|
|
342
438
|
...this.retiredBrowserControllers,
|
|
343
439
|
]);
|
|
344
440
|
}
|
|
345
|
-
async
|
|
346
|
-
const { browserPlugin, launchOptions,
|
|
441
|
+
async launchBrowser(pageId, options) {
|
|
442
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
347
443
|
const browserController = browserPlugin.createController();
|
|
348
444
|
this.startingBrowserControllers.add(browserController);
|
|
349
445
|
const launchContext = browserPlugin.createLaunchContext({
|
|
350
446
|
id: pageId,
|
|
351
447
|
launchOptions,
|
|
352
|
-
proxyTier,
|
|
353
448
|
proxyUrl,
|
|
354
449
|
});
|
|
450
|
+
// Disable SSL verification for MITM proxies
|
|
451
|
+
if (ignoreTlsErrors) {
|
|
452
|
+
/**
|
|
453
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
454
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
455
|
+
*/
|
|
456
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
457
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
458
|
+
}
|
|
355
459
|
try {
|
|
356
460
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
357
461
|
// because otherwise it would be stuck in limbo without a browser.
|
|
358
|
-
await this.
|
|
462
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
359
463
|
tryCancel();
|
|
360
464
|
const browser = await browserPlugin.launch(launchContext);
|
|
361
465
|
tryCancel();
|
|
@@ -365,18 +469,17 @@ export class BrowserPool extends TypedEmitter {
|
|
|
365
469
|
this.startingBrowserControllers.delete(browserController);
|
|
366
470
|
throw err;
|
|
367
471
|
}
|
|
368
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
369
|
-
browserController.proxyTier = proxyTier;
|
|
472
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
370
473
|
browserController.proxyUrl = proxyUrl;
|
|
371
474
|
try {
|
|
372
475
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
373
476
|
// both the controller and the browser before throwing.
|
|
374
|
-
await this.
|
|
477
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
375
478
|
}
|
|
376
479
|
catch (err) {
|
|
377
480
|
this.startingBrowserControllers.delete(browserController);
|
|
378
481
|
browserController.close().catch((closeErr) => {
|
|
379
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
482
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
380
483
|
id: browserController.id,
|
|
381
484
|
});
|
|
382
485
|
});
|
|
@@ -386,33 +489,31 @@ export class BrowserPool extends TypedEmitter {
|
|
|
386
489
|
browserController.activate();
|
|
387
490
|
this.startingBrowserControllers.delete(browserController);
|
|
388
491
|
this.activeBrowserControllers.add(browserController);
|
|
389
|
-
this.emit(
|
|
492
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
390
493
|
return browserController;
|
|
391
494
|
}
|
|
392
495
|
/**
|
|
393
496
|
* Picks plugins round robin.
|
|
394
497
|
* @private
|
|
395
498
|
*/
|
|
396
|
-
|
|
499
|
+
pickBrowserPlugin() {
|
|
397
500
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
398
501
|
this.pageCounter++;
|
|
399
502
|
return this.browserPlugins[pluginIndex];
|
|
400
503
|
}
|
|
401
|
-
|
|
504
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
402
505
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
403
506
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
404
507
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
405
508
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
406
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
407
509
|
return (isCorrectPlugin &&
|
|
408
510
|
hasCapacity &&
|
|
409
|
-
(
|
|
410
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
511
|
+
(!controller.launchContext.browserPerProxy ||
|
|
411
512
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
412
|
-
(!options?.proxyUrl && !
|
|
513
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
413
514
|
});
|
|
414
515
|
}
|
|
415
|
-
async
|
|
516
|
+
async closeInactiveRetiredBrowsers() {
|
|
416
517
|
const closedBrowserIds = [];
|
|
417
518
|
for (const controller of this.retiredBrowserControllers) {
|
|
418
519
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -420,61 +521,81 @@ export class BrowserPool extends TypedEmitter {
|
|
|
420
521
|
const isBrowserEmpty = controller.activePages === 0;
|
|
421
522
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
422
523
|
const { id } = controller;
|
|
423
|
-
log.debug('Closing retired browser.', { id });
|
|
524
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
424
525
|
await controller.close();
|
|
425
526
|
this.retiredBrowserControllers.delete(controller);
|
|
426
527
|
closedBrowserIds.push(id);
|
|
427
528
|
}
|
|
428
529
|
}
|
|
429
530
|
if (closedBrowserIds.length) {
|
|
430
|
-
log.debug('Closed retired browsers.', {
|
|
531
|
+
this.#log.debug('Closed retired browsers.', {
|
|
431
532
|
count: closedBrowserIds.length,
|
|
432
533
|
closedBrowserIds,
|
|
433
534
|
});
|
|
434
535
|
}
|
|
435
536
|
}
|
|
436
|
-
|
|
537
|
+
overridePageClose(page) {
|
|
437
538
|
const originalPageClose = page.close;
|
|
438
539
|
const browserController = this.pageToBrowserController.get(page);
|
|
439
540
|
const pageId = this.getPageId(page);
|
|
440
541
|
page.close = async (...args) => {
|
|
441
|
-
await this.
|
|
542
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
442
543
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
443
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
544
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
444
545
|
});
|
|
445
|
-
await this.
|
|
546
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
446
547
|
this.pages.delete(pageId);
|
|
447
|
-
this.
|
|
448
|
-
this.emit(
|
|
548
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
549
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
449
550
|
};
|
|
450
551
|
}
|
|
451
|
-
async
|
|
552
|
+
async executeHooks(hooks, ...args) {
|
|
452
553
|
for (const hook of hooks) {
|
|
453
554
|
await hook(...args);
|
|
454
555
|
}
|
|
455
556
|
}
|
|
456
|
-
|
|
557
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
457
558
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
458
559
|
// Run this with a delay, otherwise page.close()
|
|
459
560
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
460
561
|
setTimeout(() => {
|
|
461
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
562
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
462
563
|
void browserController.close().finally(() => {
|
|
463
564
|
this.retiredBrowserControllers.delete(browserController);
|
|
464
565
|
});
|
|
465
566
|
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
|
|
466
567
|
}
|
|
467
568
|
}
|
|
468
|
-
|
|
569
|
+
/**
|
|
570
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
571
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
572
|
+
*/
|
|
573
|
+
hasFreeBrowserSlot() {
|
|
574
|
+
const total = this.startingBrowserControllers.size +
|
|
575
|
+
this.activeBrowserControllers.size +
|
|
576
|
+
this.retiredBrowserControllers.size;
|
|
577
|
+
return total < this.maxOpenBrowsers;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Returns `true` if any active browser has room for another page.
|
|
581
|
+
*/
|
|
582
|
+
hasActiveBrowserWithFreeCapacity() {
|
|
583
|
+
for (const controller of this.activeBrowserControllers) {
|
|
584
|
+
if (controller.activePages < this.maxOpenPagesPerBrowser)
|
|
585
|
+
return true;
|
|
586
|
+
}
|
|
587
|
+
return false;
|
|
588
|
+
}
|
|
589
|
+
initializeFingerprinting() {
|
|
469
590
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
470
591
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
471
592
|
this.fingerprintInjector = new FingerprintInjector();
|
|
472
593
|
if (useFingerprintCache) {
|
|
473
594
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
474
595
|
}
|
|
475
|
-
this.
|
|
596
|
+
this.addFingerprintHooks();
|
|
476
597
|
}
|
|
477
|
-
|
|
598
|
+
addFingerprintHooks() {
|
|
478
599
|
this.preLaunchHooks = [
|
|
479
600
|
...this.preLaunchHooks,
|
|
480
601
|
// This is flipped because of the fingerprint cache.
|
|
@@ -485,4 +606,3 @@ export class BrowserPool extends TypedEmitter {
|
|
|
485
606
|
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
486
607
|
}
|
|
487
608
|
}
|
|
488
|
-
//# sourceMappingURL=browser-pool.js.map
|
package/events.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
4
|
BROWSER_CLOSED = "browserClosed",
|
|
5
5
|
PAGE_CREATED = "pageCreated",
|
|
6
6
|
PAGE_CLOSED = "pageClosed"
|
|
7
7
|
}
|
|
8
|
-
export declare
|
|
8
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
9
|
BROWSER_CLOSED = "browserClosed"
|
|
10
10
|
}
|
|
11
|
-
//# sourceMappingURL=events.d.ts.map
|
package/events.js
CHANGED
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -1,29 +1,42 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
3
|
import { getGeneratorDefaultOptions } from './utils.js';
|
|
4
|
+
function applySessionHints(base, fingerprint) {
|
|
5
|
+
if (!fingerprint)
|
|
6
|
+
return base;
|
|
7
|
+
return {
|
|
8
|
+
...base,
|
|
9
|
+
...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
|
|
10
|
+
...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
|
|
11
|
+
...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
/**
|
|
5
15
|
* @internal
|
|
6
16
|
*/
|
|
7
17
|
export function createFingerprintPreLaunchHook(browserPool) {
|
|
8
18
|
const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
|
|
9
19
|
return (_pageId, launchContext) => {
|
|
20
|
+
// Remote browsers may have their own fingerprinting — skip local fingerprint injection
|
|
21
|
+
if (launchContext.isRemote)
|
|
22
|
+
return;
|
|
10
23
|
const { useIncognitoPages } = launchContext;
|
|
11
|
-
const
|
|
24
|
+
const session = launchContext.session;
|
|
25
|
+
const cacheKey = session?.id ?? launchContext.proxyUrl;
|
|
12
26
|
const { launchOptions } = launchContext;
|
|
13
|
-
// If no options are passed we try to pass best default options as possible to match browser and OS.
|
|
14
|
-
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
15
27
|
let fingerprint;
|
|
16
28
|
if (cacheKey && fingerprintCache?.has(cacheKey)) {
|
|
17
29
|
fingerprint = fingerprintCache.get(cacheKey);
|
|
18
30
|
}
|
|
19
|
-
else if (cacheKey) {
|
|
20
|
-
fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
|
|
21
|
-
fingerprintCache?.set(cacheKey, fingerprint);
|
|
22
|
-
}
|
|
23
31
|
else {
|
|
24
|
-
|
|
32
|
+
const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
33
|
+
const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
|
|
34
|
+
fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
|
|
35
|
+
if (cacheKey)
|
|
36
|
+
fingerprintCache?.set(cacheKey, fingerprint);
|
|
25
37
|
}
|
|
26
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
27
40
|
if (useIncognitoPages) {
|
|
28
41
|
return;
|
|
29
42
|
}
|
|
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
41
54
|
export function createPrePageCreateHook() {
|
|
42
55
|
return (_pageId, browserController, pageOptions) => {
|
|
43
56
|
const { launchContext, browserPlugin } = browserController;
|
|
57
|
+
if (launchContext.isRemote)
|
|
58
|
+
return;
|
|
44
59
|
const { fingerprint } = launchContext.fingerprint;
|
|
45
60
|
if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
|
|
46
61
|
pageOptions.userAgent ??= fingerprint.navigator.userAgent;
|
|
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
|
|
|
57
72
|
export function createPostPageCreateHook(fingerprintInjector) {
|
|
58
73
|
return async (page, browserController) => {
|
|
59
74
|
const { browserPlugin, launchContext } = browserController;
|
|
75
|
+
if (launchContext.isRemote)
|
|
76
|
+
return;
|
|
60
77
|
const fingerprint = launchContext.fingerprint;
|
|
61
78
|
// TODO this will require refactoring, we should use common API instead of branching based on plugin type,
|
|
62
79
|
// and there should be no public methods specific to some browser.
|
|
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
|
|
|
79
96
|
}
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
|
-
//# sourceMappingURL=hooks.js.map
|