@crawlee/browser-pool 4.0.0-beta.17 → 4.0.0-beta.170
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/abstract-classes/browser-controller.d.ts +74 -14
- package/abstract-classes/browser-controller.js +28 -24
- package/abstract-classes/browser-plugin.d.ts +48 -15
- package/abstract-classes/browser-plugin.js +69 -11
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +106 -29
- package/browser-pool.js +209 -84
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +2 -3
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +3 -4
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +20 -5
- package/launch-context.js +21 -10
- package/package.json +9 -9
- package/playwright/playwright-browser.d.ts +2 -6
- package/playwright/playwright-browser.js +16 -17
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +6 -3
- package/playwright/playwright-plugin.d.ts +10 -7
- package/playwright/playwright-plugin.js +40 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +5 -3
- package/puppeteer/puppeteer-plugin.js +83 -46
- package/remote-browser-pool.d.ts +170 -0
- package/remote-browser-pool.js +202 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/browser-pool.js
CHANGED
|
@@ -1,15 +1,35 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
1
4
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
5
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
6
|
import { nanoid } from 'nanoid';
|
|
4
|
-
import ow from 'ow';
|
|
5
7
|
import pLimit from 'p-limit';
|
|
6
8
|
import QuickLRU from 'quick-lru';
|
|
7
9
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
10
|
+
import { z } from 'zod';
|
|
8
11
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
12
|
+
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
9
13
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
|
|
14
|
+
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
11
15
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
16
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
17
|
+
const browserPoolOptionsSchema = z.strictObject({
|
|
18
|
+
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
19
|
+
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
20
|
+
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
21
|
+
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
22
|
+
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
23
|
+
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
|
|
24
|
+
preLaunchHooks: schemas.anyArray.default(() => []),
|
|
25
|
+
postLaunchHooks: schemas.anyArray.default(() => []),
|
|
26
|
+
prePageCreateHooks: schemas.anyArray.default(() => []),
|
|
27
|
+
postPageCreateHooks: schemas.anyArray.default(() => []),
|
|
28
|
+
prePageCloseHooks: schemas.anyArray.default(() => []),
|
|
29
|
+
postPageCloseHooks: schemas.anyArray.default(() => []),
|
|
30
|
+
useFingerprints: z.boolean().default(true),
|
|
31
|
+
fingerprintOptions: schemas.anyObject.default(() => ({})),
|
|
32
|
+
});
|
|
13
33
|
/**
|
|
14
34
|
* The `BrowserPool` class is the most important class of the `browser-pool` module.
|
|
15
35
|
* It manages opening and closing of browsers and their pages and its constructor
|
|
@@ -60,6 +80,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
|
60
80
|
export class BrowserPool extends TypedEmitter {
|
|
61
81
|
browserPlugins;
|
|
62
82
|
maxOpenPagesPerBrowser;
|
|
83
|
+
maxOpenBrowsers;
|
|
63
84
|
retireBrowserAfterPageCount;
|
|
64
85
|
operationTimeoutMillis;
|
|
65
86
|
closeInactiveBrowserAfterMillis;
|
|
@@ -81,29 +102,16 @@ export class BrowserPool extends TypedEmitter {
|
|
|
81
102
|
fingerprintInjector;
|
|
82
103
|
fingerprintGenerator;
|
|
83
104
|
fingerprintCache;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
105
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
106
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
107
|
+
#browserRetireInterval;
|
|
108
|
+
#limiter = pLimit(1);
|
|
109
|
+
#log;
|
|
87
110
|
constructor(options) {
|
|
88
111
|
super();
|
|
112
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
89
113
|
this.browserKillerInterval.unref();
|
|
90
|
-
|
|
91
|
-
browserPlugins: ow.array.minLength(1),
|
|
92
|
-
maxOpenPagesPerBrowser: ow.optional.number,
|
|
93
|
-
retireBrowserAfterPageCount: ow.optional.number,
|
|
94
|
-
operationTimeoutSecs: ow.optional.number,
|
|
95
|
-
closeInactiveBrowserAfterSecs: ow.optional.number,
|
|
96
|
-
retireInactiveBrowserAfterSecs: ow.optional.number,
|
|
97
|
-
preLaunchHooks: ow.optional.array,
|
|
98
|
-
postLaunchHooks: ow.optional.array,
|
|
99
|
-
prePageCreateHooks: ow.optional.array,
|
|
100
|
-
postPageCreateHooks: ow.optional.array,
|
|
101
|
-
prePageCloseHooks: ow.optional.array,
|
|
102
|
-
postPageCloseHooks: ow.optional.array,
|
|
103
|
-
useFingerprints: ow.optional.boolean,
|
|
104
|
-
fingerprintOptions: ow.optional.object,
|
|
105
|
-
}));
|
|
106
|
-
const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
|
|
114
|
+
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
107
115
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
108
116
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
109
117
|
const providedPlugin = browserPlugins[i];
|
|
@@ -115,18 +123,19 @@ export class BrowserPool extends TypedEmitter {
|
|
|
115
123
|
}
|
|
116
124
|
this.browserPlugins = browserPlugins;
|
|
117
125
|
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
126
|
+
this.maxOpenBrowsers = Infinity;
|
|
118
127
|
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
119
128
|
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
120
129
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
121
130
|
this.useFingerprints = useFingerprints;
|
|
122
131
|
this.fingerprintOptions = fingerprintOptions;
|
|
123
|
-
this
|
|
132
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
124
133
|
if (controller.activePages === 0 &&
|
|
125
134
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
126
135
|
this.retireBrowserController(controller);
|
|
127
136
|
}
|
|
128
137
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
129
|
-
this
|
|
138
|
+
this.#browserRetireInterval.unref();
|
|
130
139
|
// hooks
|
|
131
140
|
this.preLaunchHooks = preLaunchHooks;
|
|
132
141
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -136,30 +145,56 @@ export class BrowserPool extends TypedEmitter {
|
|
|
136
145
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
137
146
|
// fingerprinting
|
|
138
147
|
if (this.useFingerprints) {
|
|
139
|
-
this.
|
|
148
|
+
this.initializeFingerprinting();
|
|
140
149
|
}
|
|
141
150
|
}
|
|
142
151
|
/**
|
|
143
152
|
* Opens a new page in one of the running browsers or launches
|
|
144
153
|
* a new browser and opens a page there, if no browsers are active,
|
|
145
154
|
* or their page limits have been exceeded.
|
|
155
|
+
*
|
|
156
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
157
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
158
|
+
* fingerprinting is enabled) and reads
|
|
159
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
160
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
161
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
162
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
163
|
+
*
|
|
164
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
165
|
+
* properties are consumed — cookie and header injection remain the
|
|
166
|
+
* crawler's responsibility.
|
|
146
167
|
*/
|
|
147
168
|
async newPage(options = {}) {
|
|
148
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
169
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
149
170
|
if (this.pages.has(id)) {
|
|
150
171
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
151
172
|
}
|
|
152
173
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
153
174
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
154
175
|
}
|
|
176
|
+
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
177
|
+
// otherwise resumes queued callbacks in the previous task's
|
|
178
|
+
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
179
|
+
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
180
|
+
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
181
|
+
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
182
|
+
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
183
|
+
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
184
|
+
// context and run request B's storage writes inside request A's transaction.
|
|
185
|
+
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
155
186
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
156
|
-
return this
|
|
157
|
-
let browserController = this.
|
|
187
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
188
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
158
189
|
if (!browserController)
|
|
159
|
-
browserController = await this.
|
|
190
|
+
browserController = await this.launchBrowser(id, {
|
|
191
|
+
browserPlugin,
|
|
192
|
+
proxyUrl,
|
|
193
|
+
ignoreTlsErrors,
|
|
194
|
+
});
|
|
160
195
|
tryCancel();
|
|
161
|
-
return await this.
|
|
162
|
-
});
|
|
196
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
197
|
+
}));
|
|
163
198
|
}
|
|
164
199
|
/**
|
|
165
200
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -167,13 +202,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
167
202
|
* configure the new browser.
|
|
168
203
|
*/
|
|
169
204
|
async newPageInNewBrowser(options = {}) {
|
|
170
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
205
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
171
206
|
if (this.pages.has(id)) {
|
|
172
207
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
173
208
|
}
|
|
174
|
-
const browserController = await this.
|
|
209
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
175
210
|
tryCancel();
|
|
176
|
-
return await this.
|
|
211
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
177
212
|
}
|
|
178
213
|
/**
|
|
179
214
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -239,17 +274,22 @@ export class BrowserPool extends TypedEmitter {
|
|
|
239
274
|
getPageId(page) {
|
|
240
275
|
return this.pageIds.get(page);
|
|
241
276
|
}
|
|
242
|
-
async
|
|
277
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
243
278
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
244
279
|
// It's not ideal though, we need to come up with a better API.
|
|
245
|
-
|
|
246
|
-
await browserController['isActivePromise'];
|
|
280
|
+
await browserController.waitForActive();
|
|
247
281
|
tryCancel();
|
|
248
282
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
249
283
|
if (finalPageOptions) {
|
|
250
284
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
285
|
+
if (ignoreTlsErrors) {
|
|
286
|
+
Object.assign(finalPageOptions, {
|
|
287
|
+
ignoreHTTPSErrors: true,
|
|
288
|
+
acceptInsecureCerts: true,
|
|
289
|
+
});
|
|
290
|
+
}
|
|
251
291
|
}
|
|
252
|
-
await this.
|
|
292
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
253
293
|
tryCancel();
|
|
254
294
|
let page;
|
|
255
295
|
try {
|
|
@@ -262,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
262
302
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
263
303
|
this.retireBrowserController(browserController);
|
|
264
304
|
}
|
|
265
|
-
this.
|
|
305
|
+
this.overridePageClose(page);
|
|
266
306
|
}
|
|
267
307
|
catch (err) {
|
|
268
308
|
this.retireBrowserController(browserController);
|
|
269
309
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
270
310
|
}
|
|
271
|
-
await this.
|
|
311
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
272
312
|
tryCancel();
|
|
273
|
-
this.emit(
|
|
313
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
274
314
|
return page;
|
|
275
315
|
}
|
|
276
316
|
/**
|
|
@@ -285,7 +325,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
285
325
|
if (hasBeenRetiredOrKilled)
|
|
286
326
|
return;
|
|
287
327
|
this.retiredBrowserControllers.add(browserController);
|
|
288
|
-
this.emit(
|
|
328
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
289
329
|
this.startingBrowserControllers.delete(browserController);
|
|
290
330
|
this.activeBrowserControllers.delete(browserController);
|
|
291
331
|
}
|
|
@@ -298,6 +338,59 @@ export class BrowserPool extends TypedEmitter {
|
|
|
298
338
|
if (browserController)
|
|
299
339
|
this.retireBrowserController(browserController);
|
|
300
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
343
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
344
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
345
|
+
* storage, etc.) cannot leak into future sessions.
|
|
346
|
+
*
|
|
347
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
348
|
+
*
|
|
349
|
+
* @param page The page to release.
|
|
350
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
351
|
+
*/
|
|
352
|
+
async closePage(page, options) {
|
|
353
|
+
if (options?.error instanceof SessionError) {
|
|
354
|
+
this.retireBrowserByPage(page);
|
|
355
|
+
}
|
|
356
|
+
// Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
|
|
357
|
+
await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
361
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
362
|
+
* no longer associated with a controller.
|
|
363
|
+
*
|
|
364
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
365
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
366
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
367
|
+
* include those set by any sibling page sharing the same browser.
|
|
368
|
+
*/
|
|
369
|
+
async extractPageState(page) {
|
|
370
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
371
|
+
if (!controller) {
|
|
372
|
+
return { cookies: [] };
|
|
373
|
+
}
|
|
374
|
+
return { cookies: await controller.getCookies(page) };
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
378
|
+
*
|
|
379
|
+
* No-op when the page is no longer associated with a controller.
|
|
380
|
+
*
|
|
381
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
382
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
383
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
384
|
+
* browser share a single context, so injected cookies are visible to every
|
|
385
|
+
* page served by that browser.
|
|
386
|
+
*/
|
|
387
|
+
async injectPageState(page, state) {
|
|
388
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
389
|
+
if (!controller) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
await controller.setCookies(page, state.cookies);
|
|
393
|
+
}
|
|
301
394
|
/**
|
|
302
395
|
* Removes all active browsers from the pool. The browsers will be
|
|
303
396
|
* closed after all their pages are closed.
|
|
@@ -312,50 +405,66 @@ export class BrowserPool extends TypedEmitter {
|
|
|
312
405
|
* @return {Promise<void>}
|
|
313
406
|
*/
|
|
314
407
|
async closeAllBrowsers() {
|
|
315
|
-
const controllers = this.
|
|
408
|
+
const controllers = this.getAllBrowserControllers();
|
|
316
409
|
const promises = [...controllers]
|
|
317
410
|
.filter((controller) => controller.isActive)
|
|
318
411
|
.map(async (controller) => controller.close());
|
|
319
412
|
await Promise.all(promises);
|
|
320
413
|
}
|
|
414
|
+
async [Symbol.asyncDispose]() {
|
|
415
|
+
await this.destroy();
|
|
416
|
+
}
|
|
321
417
|
/**
|
|
322
|
-
* Closes
|
|
418
|
+
* Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
|
|
419
|
+
* browsers when a run ends and may start another run on the same pool.
|
|
323
420
|
*/
|
|
324
|
-
async
|
|
325
|
-
clearInterval(this.browserKillerInterval);
|
|
326
|
-
clearInterval(this.browserRetireInterval);
|
|
327
|
-
this.browserKillerInterval = undefined;
|
|
328
|
-
this.browserRetireInterval = undefined;
|
|
421
|
+
async releaseAllBrowsers() {
|
|
329
422
|
await this.closeAllBrowsers();
|
|
330
|
-
this._teardown();
|
|
331
|
-
}
|
|
332
|
-
_teardown() {
|
|
333
423
|
this.startingBrowserControllers.clear();
|
|
334
424
|
this.activeBrowserControllers.clear();
|
|
335
425
|
this.retiredBrowserControllers.clear();
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
|
|
429
|
+
* re-armed and its listeners are dropped, so it cannot be used again.
|
|
430
|
+
*/
|
|
431
|
+
async destroy() {
|
|
432
|
+
clearInterval(this.browserKillerInterval);
|
|
433
|
+
clearInterval(this.#browserRetireInterval);
|
|
434
|
+
this.browserKillerInterval = undefined;
|
|
435
|
+
this.#browserRetireInterval = undefined;
|
|
436
|
+
await this.releaseAllBrowsers();
|
|
336
437
|
this.removeAllListeners();
|
|
337
438
|
}
|
|
338
|
-
|
|
439
|
+
getAllBrowserControllers() {
|
|
339
440
|
return new Set([
|
|
340
441
|
...this.startingBrowserControllers,
|
|
341
442
|
...this.activeBrowserControllers,
|
|
342
443
|
...this.retiredBrowserControllers,
|
|
343
444
|
]);
|
|
344
445
|
}
|
|
345
|
-
async
|
|
346
|
-
const { browserPlugin, launchOptions,
|
|
446
|
+
async launchBrowser(pageId, options) {
|
|
447
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
347
448
|
const browserController = browserPlugin.createController();
|
|
348
449
|
this.startingBrowserControllers.add(browserController);
|
|
349
450
|
const launchContext = browserPlugin.createLaunchContext({
|
|
350
451
|
id: pageId,
|
|
351
452
|
launchOptions,
|
|
352
|
-
proxyTier,
|
|
353
453
|
proxyUrl,
|
|
354
454
|
});
|
|
455
|
+
// Disable SSL verification for MITM proxies
|
|
456
|
+
if (ignoreTlsErrors) {
|
|
457
|
+
/**
|
|
458
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
459
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
460
|
+
*/
|
|
461
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
462
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
463
|
+
}
|
|
355
464
|
try {
|
|
356
465
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
357
466
|
// because otherwise it would be stuck in limbo without a browser.
|
|
358
|
-
await this.
|
|
467
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
359
468
|
tryCancel();
|
|
360
469
|
const browser = await browserPlugin.launch(launchContext);
|
|
361
470
|
tryCancel();
|
|
@@ -365,18 +474,17 @@ export class BrowserPool extends TypedEmitter {
|
|
|
365
474
|
this.startingBrowserControllers.delete(browserController);
|
|
366
475
|
throw err;
|
|
367
476
|
}
|
|
368
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
369
|
-
browserController.proxyTier = proxyTier;
|
|
477
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
370
478
|
browserController.proxyUrl = proxyUrl;
|
|
371
479
|
try {
|
|
372
480
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
373
481
|
// both the controller and the browser before throwing.
|
|
374
|
-
await this.
|
|
482
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
375
483
|
}
|
|
376
484
|
catch (err) {
|
|
377
485
|
this.startingBrowserControllers.delete(browserController);
|
|
378
486
|
browserController.close().catch((closeErr) => {
|
|
379
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
487
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
380
488
|
id: browserController.id,
|
|
381
489
|
});
|
|
382
490
|
});
|
|
@@ -386,33 +494,31 @@ export class BrowserPool extends TypedEmitter {
|
|
|
386
494
|
browserController.activate();
|
|
387
495
|
this.startingBrowserControllers.delete(browserController);
|
|
388
496
|
this.activeBrowserControllers.add(browserController);
|
|
389
|
-
this.emit(
|
|
497
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
390
498
|
return browserController;
|
|
391
499
|
}
|
|
392
500
|
/**
|
|
393
501
|
* Picks plugins round robin.
|
|
394
502
|
* @private
|
|
395
503
|
*/
|
|
396
|
-
|
|
504
|
+
pickBrowserPlugin() {
|
|
397
505
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
398
506
|
this.pageCounter++;
|
|
399
507
|
return this.browserPlugins[pluginIndex];
|
|
400
508
|
}
|
|
401
|
-
|
|
509
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
402
510
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
403
511
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
404
512
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
405
513
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
406
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
407
514
|
return (isCorrectPlugin &&
|
|
408
515
|
hasCapacity &&
|
|
409
|
-
(
|
|
410
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
516
|
+
(!controller.launchContext.browserPerProxy ||
|
|
411
517
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
412
|
-
(!options?.proxyUrl && !
|
|
518
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
413
519
|
});
|
|
414
520
|
}
|
|
415
|
-
async
|
|
521
|
+
async closeInactiveRetiredBrowsers() {
|
|
416
522
|
const closedBrowserIds = [];
|
|
417
523
|
for (const controller of this.retiredBrowserControllers) {
|
|
418
524
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -420,61 +526,81 @@ export class BrowserPool extends TypedEmitter {
|
|
|
420
526
|
const isBrowserEmpty = controller.activePages === 0;
|
|
421
527
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
422
528
|
const { id } = controller;
|
|
423
|
-
log.debug('Closing retired browser.', { id });
|
|
529
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
424
530
|
await controller.close();
|
|
425
531
|
this.retiredBrowserControllers.delete(controller);
|
|
426
532
|
closedBrowserIds.push(id);
|
|
427
533
|
}
|
|
428
534
|
}
|
|
429
535
|
if (closedBrowserIds.length) {
|
|
430
|
-
log.debug('Closed retired browsers.', {
|
|
536
|
+
this.#log.debug('Closed retired browsers.', {
|
|
431
537
|
count: closedBrowserIds.length,
|
|
432
538
|
closedBrowserIds,
|
|
433
539
|
});
|
|
434
540
|
}
|
|
435
541
|
}
|
|
436
|
-
|
|
542
|
+
overridePageClose(page) {
|
|
437
543
|
const originalPageClose = page.close;
|
|
438
544
|
const browserController = this.pageToBrowserController.get(page);
|
|
439
545
|
const pageId = this.getPageId(page);
|
|
440
546
|
page.close = async (...args) => {
|
|
441
|
-
await this.
|
|
547
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
442
548
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
443
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
549
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
444
550
|
});
|
|
445
|
-
await this.
|
|
551
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
446
552
|
this.pages.delete(pageId);
|
|
447
|
-
this.
|
|
448
|
-
this.emit(
|
|
553
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
554
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
449
555
|
};
|
|
450
556
|
}
|
|
451
|
-
async
|
|
557
|
+
async executeHooks(hooks, ...args) {
|
|
452
558
|
for (const hook of hooks) {
|
|
453
559
|
await hook(...args);
|
|
454
560
|
}
|
|
455
561
|
}
|
|
456
|
-
|
|
562
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
457
563
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
458
564
|
// Run this with a delay, otherwise page.close()
|
|
459
565
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
460
566
|
setTimeout(() => {
|
|
461
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
567
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
462
568
|
void browserController.close().finally(() => {
|
|
463
569
|
this.retiredBrowserControllers.delete(browserController);
|
|
464
570
|
});
|
|
465
571
|
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
|
|
466
572
|
}
|
|
467
573
|
}
|
|
468
|
-
|
|
574
|
+
/**
|
|
575
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
576
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
577
|
+
*/
|
|
578
|
+
hasFreeBrowserSlot() {
|
|
579
|
+
const total = this.startingBrowserControllers.size +
|
|
580
|
+
this.activeBrowserControllers.size +
|
|
581
|
+
this.retiredBrowserControllers.size;
|
|
582
|
+
return total < this.maxOpenBrowsers;
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Returns `true` if any active browser has room for another page.
|
|
586
|
+
*/
|
|
587
|
+
hasActiveBrowserWithFreeCapacity() {
|
|
588
|
+
for (const controller of this.activeBrowserControllers) {
|
|
589
|
+
if (controller.activePages < this.maxOpenPagesPerBrowser)
|
|
590
|
+
return true;
|
|
591
|
+
}
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
594
|
+
initializeFingerprinting() {
|
|
469
595
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
470
596
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
471
597
|
this.fingerprintInjector = new FingerprintInjector();
|
|
472
598
|
if (useFingerprintCache) {
|
|
473
599
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
474
600
|
}
|
|
475
|
-
this.
|
|
601
|
+
this.addFingerprintHooks();
|
|
476
602
|
}
|
|
477
|
-
|
|
603
|
+
addFingerprintHooks() {
|
|
478
604
|
this.preLaunchHooks = [
|
|
479
605
|
...this.preLaunchHooks,
|
|
480
606
|
// This is flipped because of the fingerprint cache.
|
|
@@ -485,4 +611,3 @@ export class BrowserPool extends TypedEmitter {
|
|
|
485
611
|
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
486
612
|
}
|
|
487
613
|
}
|
|
488
|
-
//# sourceMappingURL=browser-pool.js.map
|
package/events.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
4
|
BROWSER_CLOSED = "browserClosed",
|
|
5
5
|
PAGE_CREATED = "pageCreated",
|
|
6
6
|
PAGE_CLOSED = "pageClosed"
|
|
7
7
|
}
|
|
8
|
-
export declare
|
|
8
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
9
|
BROWSER_CLOSED = "browserClosed"
|
|
10
10
|
}
|
|
11
|
-
//# sourceMappingURL=events.d.ts.map
|
package/events.js
CHANGED
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|
package/fingerprinting/hooks.js
CHANGED
|
@@ -1,29 +1,42 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
3
|
import { getGeneratorDefaultOptions } from './utils.js';
|
|
4
|
+
function applySessionHints(base, fingerprint) {
|
|
5
|
+
if (!fingerprint)
|
|
6
|
+
return base;
|
|
7
|
+
return {
|
|
8
|
+
...base,
|
|
9
|
+
...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
|
|
10
|
+
...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
|
|
11
|
+
...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
4
14
|
/**
|
|
5
15
|
* @internal
|
|
6
16
|
*/
|
|
7
17
|
export function createFingerprintPreLaunchHook(browserPool) {
|
|
8
18
|
const { fingerprintGenerator, fingerprintCache, fingerprintOptions: { fingerprintGeneratorOptions }, } = browserPool;
|
|
9
19
|
return (_pageId, launchContext) => {
|
|
20
|
+
// Remote browsers may have their own fingerprinting — skip local fingerprint injection
|
|
21
|
+
if (launchContext.isRemote)
|
|
22
|
+
return;
|
|
10
23
|
const { useIncognitoPages } = launchContext;
|
|
11
|
-
const
|
|
24
|
+
const session = launchContext.session;
|
|
25
|
+
const cacheKey = session?.id ?? launchContext.proxyUrl;
|
|
12
26
|
const { launchOptions } = launchContext;
|
|
13
|
-
// If no options are passed we try to pass best default options as possible to match browser and OS.
|
|
14
|
-
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
15
27
|
let fingerprint;
|
|
16
28
|
if (cacheKey && fingerprintCache?.has(cacheKey)) {
|
|
17
29
|
fingerprint = fingerprintCache.get(cacheKey);
|
|
18
30
|
}
|
|
19
|
-
else if (cacheKey) {
|
|
20
|
-
fingerprint = fingerprintGenerator.getFingerprint(fingerprintGeneratorFinalOptions);
|
|
21
|
-
fingerprintCache?.set(cacheKey, fingerprint);
|
|
22
|
-
}
|
|
23
31
|
else {
|
|
24
|
-
|
|
32
|
+
const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
|
|
33
|
+
const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
|
|
34
|
+
fingerprint = fingerprintGenerator.getFingerprint(finalOptions);
|
|
35
|
+
if (cacheKey)
|
|
36
|
+
fingerprintCache?.set(cacheKey, fingerprint);
|
|
25
37
|
}
|
|
26
|
-
|
|
38
|
+
// `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)
|
|
39
|
+
launchContext.fingerprint = fingerprint;
|
|
27
40
|
if (useIncognitoPages) {
|
|
28
41
|
return;
|
|
29
42
|
}
|
|
@@ -41,6 +54,8 @@ export function createFingerprintPreLaunchHook(browserPool) {
|
|
|
41
54
|
export function createPrePageCreateHook() {
|
|
42
55
|
return (_pageId, browserController, pageOptions) => {
|
|
43
56
|
const { launchContext, browserPlugin } = browserController;
|
|
57
|
+
if (launchContext.isRemote)
|
|
58
|
+
return;
|
|
44
59
|
const { fingerprint } = launchContext.fingerprint;
|
|
45
60
|
if (launchContext.useIncognitoPages && browserPlugin instanceof PlaywrightPlugin && pageOptions) {
|
|
46
61
|
pageOptions.userAgent ??= fingerprint.navigator.userAgent;
|
|
@@ -57,6 +72,8 @@ export function createPrePageCreateHook() {
|
|
|
57
72
|
export function createPostPageCreateHook(fingerprintInjector) {
|
|
58
73
|
return async (page, browserController) => {
|
|
59
74
|
const { browserPlugin, launchContext } = browserController;
|
|
75
|
+
if (launchContext.isRemote)
|
|
76
|
+
return;
|
|
60
77
|
const fingerprint = launchContext.fingerprint;
|
|
61
78
|
// TODO this will require refactoring, we should use common API instead of branching based on plugin type,
|
|
62
79
|
// and there should be no public methods specific to some browser.
|
|
@@ -79,4 +96,3 @@ export function createPostPageCreateHook(fingerprintInjector) {
|
|
|
79
96
|
}
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
|
-
//# sourceMappingURL=hooks.js.map
|