@crawlee/browser-pool 4.0.0-beta.13 → 4.0.0-beta.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/abstract-classes/browser-controller.d.ts +70 -13
- package/abstract-classes/browser-controller.js +21 -22
- package/abstract-classes/browser-plugin.d.ts +48 -15
- package/abstract-classes/browser-plugin.js +69 -11
- package/anonymize-proxy.d.ts +5 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +100 -28
- package/browser-pool.js +211 -83
- package/container-proxy-server.d.ts +0 -1
- package/container-proxy-server.js +0 -1
- package/events.d.ts +2 -3
- package/events.js +0 -1
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +2 -3
- package/fingerprinting/types.js +0 -1
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +20 -5
- package/launch-context.js +21 -10
- package/package.json +10 -11
- package/playwright/playwright-browser.d.ts +2 -6
- package/playwright/playwright-browser.js +16 -17
- package/playwright/playwright-controller.d.ts +0 -1
- package/playwright/playwright-controller.js +20 -10
- package/playwright/playwright-plugin.d.ts +10 -7
- package/playwright/playwright-plugin.js +40 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +0 -1
- package/puppeteer/puppeteer-controller.js +16 -9
- package/puppeteer/puppeteer-plugin.d.ts +5 -3
- package/puppeteer/puppeteer-plugin.js +83 -46
- package/remote-browser-pool.d.ts +164 -0
- package/remote-browser-pool.js +192 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/browser-pool.js
CHANGED
|
@@ -1,15 +1,34 @@
|
|
|
1
|
+
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
+
import { parseArgument, schemas, SessionError, serviceLocator } from '@crawlee/core';
|
|
1
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
5
|
import { nanoid } from 'nanoid';
|
|
4
|
-
import ow from 'ow';
|
|
5
6
|
import pLimit from 'p-limit';
|
|
6
7
|
import QuickLRU from 'quick-lru';
|
|
7
8
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
9
|
+
import { z } from 'zod';
|
|
8
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
|
+
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
9
12
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
|
|
13
|
+
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
11
14
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
15
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
16
|
+
const browserPoolOptionsSchema = z.strictObject({
|
|
17
|
+
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
18
|
+
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
19
|
+
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
20
|
+
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
21
|
+
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
22
|
+
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
|
|
23
|
+
preLaunchHooks: schemas.anyArray.default(() => []),
|
|
24
|
+
postLaunchHooks: schemas.anyArray.default(() => []),
|
|
25
|
+
prePageCreateHooks: schemas.anyArray.default(() => []),
|
|
26
|
+
postPageCreateHooks: schemas.anyArray.default(() => []),
|
|
27
|
+
prePageCloseHooks: schemas.anyArray.default(() => []),
|
|
28
|
+
postPageCloseHooks: schemas.anyArray.default(() => []),
|
|
29
|
+
useFingerprints: z.boolean().default(true),
|
|
30
|
+
fingerprintOptions: schemas.anyObject.default(() => ({})),
|
|
31
|
+
});
|
|
13
32
|
/**
|
|
14
33
|
* The `BrowserPool` class is the most important class of the `browser-pool` module.
|
|
15
34
|
* It manages opening and closing of browsers and their pages and its constructor
|
|
@@ -60,6 +79,7 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
|
60
79
|
export class BrowserPool extends TypedEmitter {
|
|
61
80
|
browserPlugins;
|
|
62
81
|
maxOpenPagesPerBrowser;
|
|
82
|
+
maxOpenBrowsers;
|
|
63
83
|
retireBrowserAfterPageCount;
|
|
64
84
|
operationTimeoutMillis;
|
|
65
85
|
closeInactiveBrowserAfterMillis;
|
|
@@ -74,35 +94,23 @@ export class BrowserPool extends TypedEmitter {
|
|
|
74
94
|
pageCounter = 0;
|
|
75
95
|
pages = new Map();
|
|
76
96
|
pageIds = new WeakMap();
|
|
97
|
+
startingBrowserControllers = new Set();
|
|
77
98
|
activeBrowserControllers = new Set();
|
|
78
99
|
retiredBrowserControllers = new Set();
|
|
79
100
|
pageToBrowserController = new WeakMap();
|
|
80
101
|
fingerprintInjector;
|
|
81
102
|
fingerprintGenerator;
|
|
82
103
|
fingerprintCache;
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
104
|
+
// kept as TS-private: tests replace this interval through bracket access
|
|
105
|
+
browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
106
|
+
#browserRetireInterval;
|
|
107
|
+
#limiter = pLimit(1);
|
|
108
|
+
#log;
|
|
86
109
|
constructor(options) {
|
|
87
110
|
super();
|
|
111
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
88
112
|
this.browserKillerInterval.unref();
|
|
89
|
-
|
|
90
|
-
browserPlugins: ow.array.minLength(1),
|
|
91
|
-
maxOpenPagesPerBrowser: ow.optional.number,
|
|
92
|
-
retireBrowserAfterPageCount: ow.optional.number,
|
|
93
|
-
operationTimeoutSecs: ow.optional.number,
|
|
94
|
-
closeInactiveBrowserAfterSecs: ow.optional.number,
|
|
95
|
-
retireInactiveBrowserAfterSecs: ow.optional.number,
|
|
96
|
-
preLaunchHooks: ow.optional.array,
|
|
97
|
-
postLaunchHooks: ow.optional.array,
|
|
98
|
-
prePageCreateHooks: ow.optional.array,
|
|
99
|
-
postPageCreateHooks: ow.optional.array,
|
|
100
|
-
prePageCloseHooks: ow.optional.array,
|
|
101
|
-
postPageCloseHooks: ow.optional.array,
|
|
102
|
-
useFingerprints: ow.optional.boolean,
|
|
103
|
-
fingerprintOptions: ow.optional.object,
|
|
104
|
-
}));
|
|
105
|
-
const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
|
|
113
|
+
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
106
114
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
107
115
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
108
116
|
const providedPlugin = browserPlugins[i];
|
|
@@ -114,18 +122,19 @@ export class BrowserPool extends TypedEmitter {
|
|
|
114
122
|
}
|
|
115
123
|
this.browserPlugins = browserPlugins;
|
|
116
124
|
this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
125
|
+
this.maxOpenBrowsers = Infinity;
|
|
117
126
|
this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
118
127
|
this.operationTimeoutMillis = operationTimeoutSecs * 1000;
|
|
119
128
|
this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
|
|
120
129
|
this.useFingerprints = useFingerprints;
|
|
121
130
|
this.fingerprintOptions = fingerprintOptions;
|
|
122
|
-
this
|
|
131
|
+
this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
|
|
123
132
|
if (controller.activePages === 0 &&
|
|
124
133
|
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
|
|
125
134
|
this.retireBrowserController(controller);
|
|
126
135
|
}
|
|
127
136
|
}), retireInactiveBrowserAfterSecs * 1000);
|
|
128
|
-
this
|
|
137
|
+
this.#browserRetireInterval.unref();
|
|
129
138
|
// hooks
|
|
130
139
|
this.preLaunchHooks = preLaunchHooks;
|
|
131
140
|
this.postLaunchHooks = postLaunchHooks;
|
|
@@ -135,30 +144,56 @@ export class BrowserPool extends TypedEmitter {
|
|
|
135
144
|
this.postPageCloseHooks = postPageCloseHooks;
|
|
136
145
|
// fingerprinting
|
|
137
146
|
if (this.useFingerprints) {
|
|
138
|
-
this.
|
|
147
|
+
this.initializeFingerprinting();
|
|
139
148
|
}
|
|
140
149
|
}
|
|
141
150
|
/**
|
|
142
151
|
* Opens a new page in one of the running browsers or launches
|
|
143
152
|
* a new browser and opens a page there, if no browsers are active,
|
|
144
153
|
* or their page limits have been exceeded.
|
|
154
|
+
*
|
|
155
|
+
* **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
|
|
156
|
+
* provided, this implementation uses it as a cache key for browser fingerprints (when
|
|
157
|
+
* fingerprinting is enabled) and reads
|
|
158
|
+
* {@link ProxyInfo.url|session.proxyInfo.url} /
|
|
159
|
+
* {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
|
|
160
|
+
* for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
|
|
161
|
+
* `ignoreTlsErrors` values in the options take precedence.
|
|
162
|
+
*
|
|
163
|
+
* Beyond fingerprint caching and proxy configuration, no other session
|
|
164
|
+
* properties are consumed — cookie and header injection remain the
|
|
165
|
+
* crawler's responsibility.
|
|
145
166
|
*/
|
|
146
167
|
async newPage(options = {}) {
|
|
147
|
-
const { id = nanoid(), pageOptions, browserPlugin = this.
|
|
168
|
+
const { id = nanoid(), pageOptions, browserPlugin = this.pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
|
|
148
169
|
if (this.pages.has(id)) {
|
|
149
170
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
150
171
|
}
|
|
151
172
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
152
173
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
153
174
|
}
|
|
175
|
+
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
176
|
+
// otherwise resumes queued callbacks in the previous task's
|
|
177
|
+
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
178
|
+
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
179
|
+
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
180
|
+
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
181
|
+
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
182
|
+
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
183
|
+
// context and run request B's storage writes inside request A's transaction.
|
|
184
|
+
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
154
185
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
155
|
-
return this
|
|
156
|
-
let browserController = this.
|
|
186
|
+
return this.#limiter(AsyncResource.bind(async () => {
|
|
187
|
+
let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
157
188
|
if (!browserController)
|
|
158
|
-
browserController = await this.
|
|
189
|
+
browserController = await this.launchBrowser(id, {
|
|
190
|
+
browserPlugin,
|
|
191
|
+
proxyUrl,
|
|
192
|
+
ignoreTlsErrors,
|
|
193
|
+
});
|
|
159
194
|
tryCancel();
|
|
160
|
-
return await this.
|
|
161
|
-
});
|
|
195
|
+
return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
196
|
+
}));
|
|
162
197
|
}
|
|
163
198
|
/**
|
|
164
199
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -166,13 +201,13 @@ export class BrowserPool extends TypedEmitter {
|
|
|
166
201
|
* configure the new browser.
|
|
167
202
|
*/
|
|
168
203
|
async newPageInNewBrowser(options = {}) {
|
|
169
|
-
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.
|
|
204
|
+
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;
|
|
170
205
|
if (this.pages.has(id)) {
|
|
171
206
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
172
207
|
}
|
|
173
|
-
const browserController = await this.
|
|
208
|
+
const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });
|
|
174
209
|
tryCancel();
|
|
175
|
-
return await this.
|
|
210
|
+
return await this.createPageForBrowser(id, browserController, pageOptions);
|
|
176
211
|
}
|
|
177
212
|
/**
|
|
178
213
|
* Opens new pages with all available plugins and returns an array
|
|
@@ -238,7 +273,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
238
273
|
getPageId(page) {
|
|
239
274
|
return this.pageIds.get(page);
|
|
240
275
|
}
|
|
241
|
-
async
|
|
276
|
+
async createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
242
277
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
243
278
|
// It's not ideal though, we need to come up with a better API.
|
|
244
279
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -247,8 +282,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
247
282
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
248
283
|
if (finalPageOptions) {
|
|
249
284
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
285
|
+
if (ignoreTlsErrors) {
|
|
286
|
+
Object.assign(finalPageOptions, {
|
|
287
|
+
ignoreHTTPSErrors: true,
|
|
288
|
+
acceptInsecureCerts: true,
|
|
289
|
+
});
|
|
290
|
+
}
|
|
250
291
|
}
|
|
251
|
-
await this.
|
|
292
|
+
await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
252
293
|
tryCancel();
|
|
253
294
|
let page;
|
|
254
295
|
try {
|
|
@@ -261,15 +302,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
261
302
|
if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
|
|
262
303
|
this.retireBrowserController(browserController);
|
|
263
304
|
}
|
|
264
|
-
this.
|
|
305
|
+
this.overridePageClose(page);
|
|
265
306
|
}
|
|
266
307
|
catch (err) {
|
|
267
308
|
this.retireBrowserController(browserController);
|
|
268
309
|
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
|
|
269
310
|
}
|
|
270
|
-
await this.
|
|
311
|
+
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
271
312
|
tryCancel();
|
|
272
|
-
this.emit(
|
|
313
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
273
314
|
return page;
|
|
274
315
|
}
|
|
275
316
|
/**
|
|
@@ -278,11 +319,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
278
319
|
*
|
|
279
320
|
*/
|
|
280
321
|
retireBrowserController(browserController) {
|
|
281
|
-
const
|
|
322
|
+
const isStarting = this.startingBrowserControllers.has(browserController);
|
|
323
|
+
const isActive = this.activeBrowserControllers.has(browserController);
|
|
324
|
+
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
282
325
|
if (hasBeenRetiredOrKilled)
|
|
283
326
|
return;
|
|
284
327
|
this.retiredBrowserControllers.add(browserController);
|
|
285
|
-
this.emit(
|
|
328
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
329
|
+
this.startingBrowserControllers.delete(browserController);
|
|
286
330
|
this.activeBrowserControllers.delete(browserController);
|
|
287
331
|
}
|
|
288
332
|
/**
|
|
@@ -294,12 +338,65 @@ export class BrowserPool extends TypedEmitter {
|
|
|
294
338
|
if (browserController)
|
|
295
339
|
this.retireBrowserController(browserController);
|
|
296
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Releases a page back to the pool. The page is closed and, if the
|
|
343
|
+
* optional `error` is a {@link SessionError}, the browser controller
|
|
344
|
+
* that served the page is retired so that its tainted state (cookies,
|
|
345
|
+
* storage, etc.) cannot leak into future sessions.
|
|
346
|
+
*
|
|
347
|
+
* This is the primary way the crawler should return pages to the pool.
|
|
348
|
+
*
|
|
349
|
+
* @param page The page to release.
|
|
350
|
+
* @param options.error The error that caused the page to be released, if any.
|
|
351
|
+
*/
|
|
352
|
+
async closePage(page, options) {
|
|
353
|
+
if (options?.error instanceof SessionError) {
|
|
354
|
+
this.retireBrowserByPage(page);
|
|
355
|
+
}
|
|
356
|
+
// Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
|
|
357
|
+
await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Extracts the relevant state (currently just cookies) from a page via its
|
|
361
|
+
* owning {@link BrowserController}. Returns empty state when the page is
|
|
362
|
+
* no longer associated with a controller.
|
|
363
|
+
*
|
|
364
|
+
* As with {@link BrowserPool.injectPageState}, cookies are isolated per
|
|
365
|
+
* page only when the pool is configured with `useIncognitoPages: true`.
|
|
366
|
+
* With the default `useIncognitoPages: false`, the extracted cookies
|
|
367
|
+
* include those set by any sibling page sharing the same browser.
|
|
368
|
+
*/
|
|
369
|
+
async extractPageState(page) {
|
|
370
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
371
|
+
if (!controller) {
|
|
372
|
+
return { cookies: [] };
|
|
373
|
+
}
|
|
374
|
+
return { cookies: await controller.getCookies(page) };
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Injects state into a page via its owning {@link BrowserController}.
|
|
378
|
+
*
|
|
379
|
+
* No-op when the page is no longer associated with a controller.
|
|
380
|
+
*
|
|
381
|
+
* Note that cookies are isolated per page only when the pool is configured
|
|
382
|
+
* with `useIncognitoPages: true` — each page then gets its own browser
|
|
383
|
+
* context. With the default `useIncognitoPages: false`, all pages in a
|
|
384
|
+
* browser share a single context, so injected cookies are visible to every
|
|
385
|
+
* page served by that browser.
|
|
386
|
+
*/
|
|
387
|
+
async injectPageState(page, state) {
|
|
388
|
+
const controller = this.getBrowserControllerByPage(page);
|
|
389
|
+
if (!controller) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
await controller.setCookies(page, state.cookies);
|
|
393
|
+
}
|
|
297
394
|
/**
|
|
298
395
|
* Removes all active browsers from the pool. The browsers will be
|
|
299
396
|
* closed after all their pages are closed.
|
|
300
397
|
*/
|
|
301
398
|
retireAllBrowsers() {
|
|
302
|
-
this.activeBrowserControllers.forEach((controller) => {
|
|
399
|
+
[...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
303
400
|
this.retireBrowserController(controller);
|
|
304
401
|
});
|
|
305
402
|
}
|
|
@@ -308,7 +405,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
308
405
|
* @return {Promise<void>}
|
|
309
406
|
*/
|
|
310
407
|
async closeAllBrowsers() {
|
|
311
|
-
const controllers = this.
|
|
408
|
+
const controllers = this.getAllBrowserControllers();
|
|
312
409
|
const promises = [...controllers]
|
|
313
410
|
.filter((controller) => controller.isActive)
|
|
314
411
|
.map(async (controller) => controller.close());
|
|
@@ -319,55 +416,67 @@ export class BrowserPool extends TypedEmitter {
|
|
|
319
416
|
*/
|
|
320
417
|
async destroy() {
|
|
321
418
|
clearInterval(this.browserKillerInterval);
|
|
322
|
-
clearInterval(this
|
|
419
|
+
clearInterval(this.#browserRetireInterval);
|
|
323
420
|
this.browserKillerInterval = undefined;
|
|
324
|
-
this
|
|
421
|
+
this.#browserRetireInterval = undefined;
|
|
325
422
|
await this.closeAllBrowsers();
|
|
326
|
-
this.
|
|
423
|
+
this.teardown();
|
|
327
424
|
}
|
|
328
|
-
|
|
425
|
+
teardown() {
|
|
426
|
+
this.startingBrowserControllers.clear();
|
|
329
427
|
this.activeBrowserControllers.clear();
|
|
330
428
|
this.retiredBrowserControllers.clear();
|
|
331
429
|
this.removeAllListeners();
|
|
332
430
|
}
|
|
333
|
-
|
|
334
|
-
return new Set([
|
|
431
|
+
getAllBrowserControllers() {
|
|
432
|
+
return new Set([
|
|
433
|
+
...this.startingBrowserControllers,
|
|
434
|
+
...this.activeBrowserControllers,
|
|
435
|
+
...this.retiredBrowserControllers,
|
|
436
|
+
]);
|
|
335
437
|
}
|
|
336
|
-
async
|
|
337
|
-
const { browserPlugin, launchOptions,
|
|
438
|
+
async launchBrowser(pageId, options) {
|
|
439
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
338
440
|
const browserController = browserPlugin.createController();
|
|
339
|
-
this.
|
|
441
|
+
this.startingBrowserControllers.add(browserController);
|
|
340
442
|
const launchContext = browserPlugin.createLaunchContext({
|
|
341
443
|
id: pageId,
|
|
342
444
|
launchOptions,
|
|
343
|
-
proxyTier,
|
|
344
445
|
proxyUrl,
|
|
345
446
|
});
|
|
447
|
+
// Disable SSL verification for MITM proxies
|
|
448
|
+
if (ignoreTlsErrors) {
|
|
449
|
+
/**
|
|
450
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
451
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
452
|
+
*/
|
|
453
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
454
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
455
|
+
}
|
|
346
456
|
try {
|
|
347
457
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
348
458
|
// because otherwise it would be stuck in limbo without a browser.
|
|
349
|
-
await this.
|
|
459
|
+
await this.executeHooks(this.preLaunchHooks, pageId, launchContext);
|
|
350
460
|
tryCancel();
|
|
351
461
|
const browser = await browserPlugin.launch(launchContext);
|
|
352
462
|
tryCancel();
|
|
353
463
|
browserController.assignBrowser(browser, launchContext);
|
|
354
464
|
}
|
|
355
465
|
catch (err) {
|
|
356
|
-
this.
|
|
466
|
+
this.startingBrowserControllers.delete(browserController);
|
|
357
467
|
throw err;
|
|
358
468
|
}
|
|
359
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
360
|
-
browserController.proxyTier = proxyTier;
|
|
469
|
+
this.#log.debug('Launched new browser.', { id: browserController.id });
|
|
361
470
|
browserController.proxyUrl = proxyUrl;
|
|
362
471
|
try {
|
|
363
472
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
364
473
|
// both the controller and the browser before throwing.
|
|
365
|
-
await this.
|
|
474
|
+
await this.executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
366
475
|
}
|
|
367
476
|
catch (err) {
|
|
368
|
-
this.
|
|
477
|
+
this.startingBrowserControllers.delete(browserController);
|
|
369
478
|
browserController.close().catch((closeErr) => {
|
|
370
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
479
|
+
this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
371
480
|
id: browserController.id,
|
|
372
481
|
});
|
|
373
482
|
});
|
|
@@ -375,33 +484,33 @@ export class BrowserPool extends TypedEmitter {
|
|
|
375
484
|
}
|
|
376
485
|
tryCancel();
|
|
377
486
|
browserController.activate();
|
|
378
|
-
this.
|
|
487
|
+
this.startingBrowserControllers.delete(browserController);
|
|
488
|
+
this.activeBrowserControllers.add(browserController);
|
|
489
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
379
490
|
return browserController;
|
|
380
491
|
}
|
|
381
492
|
/**
|
|
382
493
|
* Picks plugins round robin.
|
|
383
494
|
* @private
|
|
384
495
|
*/
|
|
385
|
-
|
|
496
|
+
pickBrowserPlugin() {
|
|
386
497
|
const pluginIndex = this.pageCounter % this.browserPlugins.length;
|
|
387
498
|
this.pageCounter++;
|
|
388
499
|
return this.browserPlugins[pluginIndex];
|
|
389
500
|
}
|
|
390
|
-
|
|
501
|
+
pickBrowserWithFreeCapacity(browserPlugin, options) {
|
|
391
502
|
return [...this.activeBrowserControllers].find((controller) => {
|
|
392
503
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
393
504
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
394
505
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
395
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
396
506
|
return (isCorrectPlugin &&
|
|
397
507
|
hasCapacity &&
|
|
398
|
-
(
|
|
399
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
508
|
+
(!controller.launchContext.browserPerProxy ||
|
|
400
509
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
401
|
-
(!options?.proxyUrl && !
|
|
510
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
402
511
|
});
|
|
403
512
|
}
|
|
404
|
-
async
|
|
513
|
+
async closeInactiveRetiredBrowsers() {
|
|
405
514
|
const closedBrowserIds = [];
|
|
406
515
|
for (const controller of this.retiredBrowserControllers) {
|
|
407
516
|
const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
|
|
@@ -409,61 +518,81 @@ export class BrowserPool extends TypedEmitter {
|
|
|
409
518
|
const isBrowserEmpty = controller.activePages === 0;
|
|
410
519
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
411
520
|
const { id } = controller;
|
|
412
|
-
log.debug('Closing retired browser.', { id });
|
|
521
|
+
this.#log.debug('Closing retired browser.', { id });
|
|
413
522
|
await controller.close();
|
|
414
523
|
this.retiredBrowserControllers.delete(controller);
|
|
415
524
|
closedBrowserIds.push(id);
|
|
416
525
|
}
|
|
417
526
|
}
|
|
418
527
|
if (closedBrowserIds.length) {
|
|
419
|
-
log.debug('Closed retired browsers.', {
|
|
528
|
+
this.#log.debug('Closed retired browsers.', {
|
|
420
529
|
count: closedBrowserIds.length,
|
|
421
530
|
closedBrowserIds,
|
|
422
531
|
});
|
|
423
532
|
}
|
|
424
533
|
}
|
|
425
|
-
|
|
534
|
+
overridePageClose(page) {
|
|
426
535
|
const originalPageClose = page.close;
|
|
427
536
|
const browserController = this.pageToBrowserController.get(page);
|
|
428
537
|
const pageId = this.getPageId(page);
|
|
429
538
|
page.close = async (...args) => {
|
|
430
|
-
await this.
|
|
539
|
+
await this.executeHooks(this.prePageCloseHooks, page, browserController);
|
|
431
540
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
432
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
541
|
+
this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
433
542
|
});
|
|
434
|
-
await this.
|
|
543
|
+
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
435
544
|
this.pages.delete(pageId);
|
|
436
|
-
this.
|
|
437
|
-
this.emit(
|
|
545
|
+
this.closeRetiredBrowserWithNoPages(browserController);
|
|
546
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
438
547
|
};
|
|
439
548
|
}
|
|
440
|
-
async
|
|
549
|
+
async executeHooks(hooks, ...args) {
|
|
441
550
|
for (const hook of hooks) {
|
|
442
551
|
await hook(...args);
|
|
443
552
|
}
|
|
444
553
|
}
|
|
445
|
-
|
|
554
|
+
closeRetiredBrowserWithNoPages(browserController) {
|
|
446
555
|
if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
|
|
447
556
|
// Run this with a delay, otherwise page.close()
|
|
448
557
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
449
558
|
setTimeout(() => {
|
|
450
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
559
|
+
this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
451
560
|
void browserController.close().finally(() => {
|
|
452
561
|
this.retiredBrowserControllers.delete(browserController);
|
|
453
562
|
});
|
|
454
563
|
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
|
|
455
564
|
}
|
|
456
565
|
}
|
|
457
|
-
|
|
566
|
+
/**
|
|
567
|
+
* Returns `true` if the pool can accept a new browser launch without exceeding
|
|
568
|
+
* {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
|
|
569
|
+
*/
|
|
570
|
+
hasFreeBrowserSlot() {
|
|
571
|
+
const total = this.startingBrowserControllers.size +
|
|
572
|
+
this.activeBrowserControllers.size +
|
|
573
|
+
this.retiredBrowserControllers.size;
|
|
574
|
+
return total < this.maxOpenBrowsers;
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Returns `true` if any active browser has room for another page.
|
|
578
|
+
*/
|
|
579
|
+
hasActiveBrowserWithFreeCapacity() {
|
|
580
|
+
for (const controller of this.activeBrowserControllers) {
|
|
581
|
+
if (controller.activePages < this.maxOpenPagesPerBrowser)
|
|
582
|
+
return true;
|
|
583
|
+
}
|
|
584
|
+
return false;
|
|
585
|
+
}
|
|
586
|
+
initializeFingerprinting() {
|
|
458
587
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
|
459
588
|
this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
|
|
460
589
|
this.fingerprintInjector = new FingerprintInjector();
|
|
461
590
|
if (useFingerprintCache) {
|
|
462
591
|
this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
|
|
463
592
|
}
|
|
464
|
-
this.
|
|
593
|
+
this.addFingerprintHooks();
|
|
465
594
|
}
|
|
466
|
-
|
|
595
|
+
addFingerprintHooks() {
|
|
467
596
|
this.preLaunchHooks = [
|
|
468
597
|
...this.preLaunchHooks,
|
|
469
598
|
// This is flipped because of the fingerprint cache.
|
|
@@ -474,4 +603,3 @@ export class BrowserPool extends TypedEmitter {
|
|
|
474
603
|
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
|
|
475
604
|
}
|
|
476
605
|
}
|
|
477
|
-
//# sourceMappingURL=browser-pool.js.map
|
package/events.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
4
|
BROWSER_CLOSED = "browserClosed",
|
|
5
5
|
PAGE_CREATED = "pageCreated",
|
|
6
6
|
PAGE_CLOSED = "pageClosed"
|
|
7
7
|
}
|
|
8
|
-
export declare
|
|
8
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
9
|
BROWSER_CLOSED = "browserClosed"
|
|
10
10
|
}
|
|
11
|
-
//# sourceMappingURL=events.d.ts.map
|
package/events.js
CHANGED
|
@@ -14,4 +14,3 @@ export declare function createPrePageCreateHook(): (_pageId: string, browserCont
|
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
16
16
|
export declare function createPostPageCreateHook(fingerprintInjector: FingerprintInjector): (page: any, browserController: BrowserController) => Promise<void>;
|
|
17
|
-
//# sourceMappingURL=hooks.d.ts.map
|