@crawlee/browser-pool 4.0.0-beta.223 → 4.0.0-beta.224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstract-classes/browser-plugin.d.ts +1 -29
- package/abstract-classes/browser-plugin.js +3 -57
- package/browser-pool.d.ts +10 -13
- package/browser-pool.js +33 -38
- package/index.d.ts +3 -2
- package/index.js +2 -1
- package/launch-context.d.ts +0 -6
- package/launch-context.js +1 -7
- package/package.json +5 -5
- package/playwright/playwright-plugin.d.ts +0 -8
- package/playwright/playwright-plugin.js +0 -25
- package/playwright/remote-playwright-plugin.d.ts +29 -0
- package/playwright/remote-playwright-plugin.js +38 -0
- package/puppeteer/puppeteer-plugin.d.ts +7 -5
- package/puppeteer/puppeteer-plugin.js +57 -71
- package/puppeteer/remote-puppeteer-plugin.d.ts +22 -0
- package/puppeteer/remote-puppeteer-plugin.js +31 -0
- package/remote-browser-plugin.d.ts +49 -0
- package/remote-browser-plugin.js +63 -0
- package/remote-browser-provider.d.ts +8 -10
- package/remote-browser-provider.js +8 -10
- package/remote-browser-pool.d.ts +0 -170
- package/remote-browser-pool.js +0 -206
|
@@ -2,8 +2,7 @@ import { type CrawleeLogger, CriticalError } from '@crawlee/core';
|
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { LaunchContextOptions } from '../launch-context.js';
|
|
4
4
|
import { LaunchContext } from '../launch-context.js';
|
|
5
|
-
import type {
|
|
6
|
-
import { type UnwrapPromise } from '../utils.js';
|
|
5
|
+
import type { UnwrapPromise } from '../utils.js';
|
|
7
6
|
import type { BrowserController } from './browser-controller.js';
|
|
8
7
|
/**
|
|
9
8
|
* The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
|
|
@@ -97,34 +96,7 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
97
96
|
useIncognitoPages: boolean;
|
|
98
97
|
readonly browserPerProxy?: boolean;
|
|
99
98
|
readonly ignoreProxyCertificate?: boolean;
|
|
100
|
-
/**
|
|
101
|
-
* Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
|
|
102
|
-
* launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
|
|
103
|
-
* remote-session policy lives in the pool, not here.
|
|
104
|
-
*
|
|
105
|
-
* @internal
|
|
106
|
-
*/
|
|
107
|
-
remoteConnection?: RemoteConnection;
|
|
108
|
-
/** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
|
|
109
|
-
remoteConnectionParameters?: RemoteConnectionParameters;
|
|
110
99
|
constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
|
|
111
|
-
/**
|
|
112
|
-
* Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
|
|
113
|
-
* Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
|
|
114
|
-
* (e.g. forcing incognito pages).
|
|
115
|
-
*
|
|
116
|
-
* @internal
|
|
117
|
-
*/
|
|
118
|
-
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
119
|
-
/**
|
|
120
|
-
* Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
|
|
121
|
-
* the launch context (so the controller can release it on close), and runs the library-specific `connect`.
|
|
122
|
-
* On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
|
|
123
|
-
*
|
|
124
|
-
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
125
|
-
* lives here so it stays identical across plugins.
|
|
126
|
-
*/
|
|
127
|
-
protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
|
|
128
100
|
/**
|
|
129
101
|
* Creates a `LaunchContext` with all the information needed
|
|
130
102
|
* to launch a browser. Aside from library specific launch options,
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { CriticalError, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import merge from 'lodash.merge';
|
|
3
3
|
import { LaunchContext } from '../launch-context.js';
|
|
4
|
-
import { sanitizeEndpointForLog } from '../utils.js';
|
|
5
4
|
/**
|
|
6
5
|
* The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
|
|
7
6
|
* when Chromium/Chrome browser is launched:
|
|
@@ -33,16 +32,6 @@ export class BrowserPlugin {
|
|
|
33
32
|
useIncognitoPages;
|
|
34
33
|
browserPerProxy;
|
|
35
34
|
ignoreProxyCertificate;
|
|
36
|
-
/**
|
|
37
|
-
* Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
|
|
38
|
-
* launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
|
|
39
|
-
* remote-session policy lives in the pool, not here.
|
|
40
|
-
*
|
|
41
|
-
* @internal
|
|
42
|
-
*/
|
|
43
|
-
remoteConnection;
|
|
44
|
-
/** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
|
|
45
|
-
remoteConnectionParameters;
|
|
46
35
|
constructor(library, options = {}) {
|
|
47
36
|
const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
|
|
48
37
|
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
@@ -54,45 +43,6 @@ export class BrowserPlugin {
|
|
|
54
43
|
this.browserPerProxy = browserPerProxy;
|
|
55
44
|
this.ignoreProxyCertificate = ignoreProxyCertificate;
|
|
56
45
|
}
|
|
57
|
-
/**
|
|
58
|
-
* Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
|
|
59
|
-
* Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
|
|
60
|
-
* (e.g. forcing incognito pages).
|
|
61
|
-
*
|
|
62
|
-
* @internal
|
|
63
|
-
*/
|
|
64
|
-
useRemoteConnection(connection, parameters = {}) {
|
|
65
|
-
this.remoteConnection = connection;
|
|
66
|
-
this.remoteConnectionParameters = parameters;
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
|
|
70
|
-
* the launch context (so the controller can release it on close), and runs the library-specific `connect`.
|
|
71
|
-
* On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
|
|
72
|
-
*
|
|
73
|
-
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
74
|
-
* lives here so it stays identical across plugins.
|
|
75
|
-
*/
|
|
76
|
-
async connectToRemoteBrowser(launchContext, connect) {
|
|
77
|
-
const connection = this.remoteConnection;
|
|
78
|
-
let url;
|
|
79
|
-
let token;
|
|
80
|
-
try {
|
|
81
|
-
({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
|
|
82
|
-
}
|
|
83
|
-
catch (cause) {
|
|
84
|
-
throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
|
|
85
|
-
}
|
|
86
|
-
launchContext.remoteToken = token;
|
|
87
|
-
try {
|
|
88
|
-
return await connect(url);
|
|
89
|
-
}
|
|
90
|
-
catch (cause) {
|
|
91
|
-
await connection.release(token);
|
|
92
|
-
throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
|
|
93
|
-
'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
46
|
/**
|
|
97
47
|
* Creates a `LaunchContext` with all the information needed
|
|
98
48
|
* to launch a browser. Aside from library specific launch options,
|
|
@@ -100,7 +50,7 @@ export class BrowserPlugin {
|
|
|
100
50
|
* management of the pool and extra features.
|
|
101
51
|
*/
|
|
102
52
|
createLaunchContext(options = {}) {
|
|
103
|
-
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote
|
|
53
|
+
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote, } = options;
|
|
104
54
|
return new LaunchContext({
|
|
105
55
|
id,
|
|
106
56
|
launchOptions: merge({}, this.launchOptions, launchOptions),
|
|
@@ -117,13 +67,12 @@ export class BrowserPlugin {
|
|
|
117
67
|
* Launches the browser using provided launch context.
|
|
118
68
|
*/
|
|
119
69
|
async launch(launchContext = this.createLaunchContext()) {
|
|
120
|
-
// launchOptions is only used by the local launch path below — remote connections ignore it.
|
|
121
70
|
launchContext.launchOptions ??= {};
|
|
122
71
|
const { proxyUrl, launchOptions } = launchContext;
|
|
123
|
-
if (proxyUrl
|
|
72
|
+
if (proxyUrl) {
|
|
124
73
|
await this.addProxyToLaunchOptions(launchContext);
|
|
125
74
|
}
|
|
126
|
-
if (
|
|
75
|
+
if (this.isChromiumBasedBrowser(launchContext)) {
|
|
127
76
|
// This will set the args for chromium based browsers to hide the webdriver.
|
|
128
77
|
launchOptions.args = this.#mergeArgsToHideWebdriver(launchOptions.args);
|
|
129
78
|
// When User-Agent is not set, and we're using Chromium in headless mode,
|
|
@@ -134,9 +83,6 @@ export class BrowserPlugin {
|
|
|
134
83
|
launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
|
|
135
84
|
}
|
|
136
85
|
}
|
|
137
|
-
if (launchContext.isRemote) {
|
|
138
|
-
this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
|
|
139
|
-
}
|
|
140
86
|
return this._launch(launchContext);
|
|
141
87
|
}
|
|
142
88
|
#mergeArgsToHideWebdriver(originalArgs) {
|
package/browser-pool.d.ts
CHANGED
|
@@ -54,6 +54,16 @@ export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin
|
|
|
54
54
|
* @default 20
|
|
55
55
|
*/
|
|
56
56
|
maxOpenPagesPerBrowser?: number;
|
|
57
|
+
/**
|
|
58
|
+
* Maximum number of browser processes alive at once. A browser counts from the moment its launch starts until it
|
|
59
|
+
* actually closes — a retired browser that is still finishing its last pages still occupies a slot. Once the
|
|
60
|
+
* limit is reached, {@link BrowserPool.newPage|newPage} waits for a slot (a browser closing, or an open
|
|
61
|
+
* browser with page capacity to spare) instead of launching another. Set it to a remote browser service's
|
|
62
|
+
* concurrent-session limit to avoid `429` errors.
|
|
63
|
+
*
|
|
64
|
+
* @default Infinity
|
|
65
|
+
*/
|
|
66
|
+
maxOpenBrowsers?: number;
|
|
57
67
|
/**
|
|
58
68
|
* Browsers tend to get bloated after processing a lot of pages. This option
|
|
59
69
|
* configures the maximum number of processed pages after which the browser will
|
|
@@ -251,8 +261,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
251
261
|
#private;
|
|
252
262
|
browserPlugins: BrowserPlugins;
|
|
253
263
|
/** @internal */
|
|
254
|
-
maxOpenBrowsers: number;
|
|
255
|
-
/** @internal */
|
|
256
264
|
fingerprintOptions: FingerprintOptions;
|
|
257
265
|
/** @internal */
|
|
258
266
|
fingerprintInjector?: FingerprintInjector;
|
|
@@ -412,20 +420,9 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
412
420
|
* Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
|
|
413
421
|
* Counts starting, active, and retired browsers.
|
|
414
422
|
*
|
|
415
|
-
* A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
|
|
416
|
-
* has set a cap — {@link RemoteBrowserPool} does, from its own
|
|
417
|
-
* {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
|
|
418
|
-
* `BrowserPoolOptions` key for it.
|
|
419
|
-
*
|
|
420
423
|
* @internal
|
|
421
424
|
*/
|
|
422
425
|
hasFreeBrowserSlot(): boolean;
|
|
423
|
-
/**
|
|
424
|
-
* Returns `true` if any active browser has room for another page.
|
|
425
|
-
*
|
|
426
|
-
* @internal
|
|
427
|
-
*/
|
|
428
|
-
hasActiveBrowserWithFreeCapacity(): boolean;
|
|
429
426
|
}
|
|
430
427
|
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
|
|
431
428
|
/**
|
package/browser-pool.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { AsyncResource } from 'node:async_hooks';
|
|
2
1
|
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
3
2
|
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
@@ -14,9 +13,11 @@ import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePage
|
|
|
14
13
|
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
15
14
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
16
15
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
16
|
+
const SLOT_POLL_INTERVAL_MILLIS = 500;
|
|
17
17
|
const browserPoolOptionsSchema = z.strictObject({
|
|
18
18
|
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
19
19
|
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
20
|
+
maxOpenBrowsers: schemas.anyNumber.default(Infinity),
|
|
20
21
|
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
21
22
|
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
22
23
|
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
@@ -80,8 +81,6 @@ const browserPoolOptionsSchema = z.strictObject({
|
|
|
80
81
|
export class BrowserPool extends TypedEmitter {
|
|
81
82
|
browserPlugins;
|
|
82
83
|
/** @internal */
|
|
83
|
-
maxOpenBrowsers;
|
|
84
|
-
/** @internal */
|
|
85
84
|
fingerprintOptions;
|
|
86
85
|
/** @internal */
|
|
87
86
|
fingerprintInjector;
|
|
@@ -89,6 +88,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
89
88
|
/** @internal */
|
|
90
89
|
fingerprintCache;
|
|
91
90
|
#maxOpenPagesPerBrowser;
|
|
91
|
+
#maxOpenBrowsers;
|
|
92
92
|
#retireBrowserAfterPageCount;
|
|
93
93
|
#operationTimeoutMillis;
|
|
94
94
|
#closeInactiveBrowserAfterMillis;
|
|
@@ -113,10 +113,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
113
113
|
#browserRetireInterval;
|
|
114
114
|
#limiter = pLimit(1);
|
|
115
115
|
#log;
|
|
116
|
+
/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
|
|
117
|
+
#capacityChange;
|
|
116
118
|
constructor(options) {
|
|
117
119
|
super();
|
|
118
120
|
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
119
|
-
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
121
|
+
const { browserPlugins, maxOpenPagesPerBrowser, maxOpenBrowsers, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
120
122
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
121
123
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
122
124
|
const providedPlugin = browserPlugins[i];
|
|
@@ -127,7 +129,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
127
129
|
}
|
|
128
130
|
}
|
|
129
131
|
this.browserPlugins = browserPlugins;
|
|
130
|
-
this
|
|
132
|
+
this.#maxOpenBrowsers = maxOpenBrowsers;
|
|
131
133
|
this.fingerprintOptions = fingerprintOptions;
|
|
132
134
|
this.#maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
|
|
133
135
|
this.#retireBrowserAfterPageCount = retireBrowserAfterPageCount;
|
|
@@ -189,28 +191,18 @@ export class BrowserPool extends TypedEmitter {
|
|
|
189
191
|
if (browserPlugin && !this.browserPlugins.includes(browserPlugin)) {
|
|
190
192
|
throw new Error('Provided browserPlugin is not one of the plugins used by BrowserPool.');
|
|
191
193
|
}
|
|
192
|
-
// Bind the limiter callback to the current async-hooks context. p-limit
|
|
193
|
-
// otherwise resumes queued callbacks in the previous task's
|
|
194
|
-
// AsyncLocalStorage context, leaking aborted cancelTasks across unrelated
|
|
195
|
-
// requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
|
|
196
|
-
// fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
|
|
197
|
-
// ESM-only rewrite, so we can't bump it in Crawlee v3.
|
|
198
|
-
// Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
|
|
199
|
-
// ALS-scoped: without it, a queued callback would resume in the previous request's async
|
|
200
|
-
// context and run request B's storage writes inside request A's transaction.
|
|
201
|
-
// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
|
|
202
194
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
203
|
-
return this.#limiter(
|
|
195
|
+
return this.#limiter(async () => {
|
|
204
196
|
let browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
197
|
+
// Checked under the limiter so concurrent waiters can't all pass on the same freed slot.
|
|
198
|
+
while (!browserController && !this.hasFreeBrowserSlot()) {
|
|
199
|
+
await this.#nextCapacityChange();
|
|
200
|
+
browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
201
|
+
}
|
|
202
|
+
browserController ??= await this.#launchBrowser(id, { browserPlugin, proxyUrl, ignoreTlsErrors });
|
|
211
203
|
tryCancel();
|
|
212
204
|
return await this.#createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
213
|
-
})
|
|
205
|
+
});
|
|
214
206
|
}
|
|
215
207
|
/**
|
|
216
208
|
* Unlike {@link newPage}, `newPageInNewBrowser` always launches a new
|
|
@@ -628,30 +620,33 @@ export class BrowserPool extends TypedEmitter {
|
|
|
628
620
|
* Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
|
|
629
621
|
* Counts starting, active, and retired browsers.
|
|
630
622
|
*
|
|
631
|
-
* A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
|
|
632
|
-
* has set a cap — {@link RemoteBrowserPool} does, from its own
|
|
633
|
-
* {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
|
|
634
|
-
* `BrowserPoolOptions` key for it.
|
|
635
|
-
*
|
|
636
623
|
* @internal
|
|
637
624
|
*/
|
|
638
625
|
hasFreeBrowserSlot() {
|
|
639
626
|
const total = this.#startingBrowserControllers.size +
|
|
640
627
|
this.activeBrowserControllers.size +
|
|
641
628
|
this.retiredBrowserControllers.size;
|
|
642
|
-
return total < this
|
|
629
|
+
return total < this.#maxOpenBrowsers;
|
|
643
630
|
}
|
|
644
631
|
/**
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
* @internal
|
|
632
|
+
* Resolves on the next browser-retired / page-closed event, or after a short fallback interval. All
|
|
633
|
+
* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners) per tick.
|
|
648
634
|
*/
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
635
|
+
#nextCapacityChange() {
|
|
636
|
+
this.#capacityChange ??= new Promise((resolve) => {
|
|
637
|
+
const done = () => {
|
|
638
|
+
clearTimeout(timer);
|
|
639
|
+
this.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
640
|
+
this.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
641
|
+
this.#capacityChange = undefined;
|
|
642
|
+
resolve();
|
|
643
|
+
};
|
|
644
|
+
const timer = setTimeout(done, SLOT_POLL_INTERVAL_MILLIS);
|
|
645
|
+
timer.unref?.();
|
|
646
|
+
this.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
647
|
+
this.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
648
|
+
});
|
|
649
|
+
return this.#capacityChange;
|
|
655
650
|
}
|
|
656
651
|
#initializeFingerprinting() {
|
|
657
652
|
const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
|
package/index.d.ts
CHANGED
|
@@ -26,7 +26,9 @@
|
|
|
26
26
|
*/
|
|
27
27
|
export * from './browser-pool.js';
|
|
28
28
|
export * from './playwright/playwright-plugin.js';
|
|
29
|
+
export * from './playwright/remote-playwright-plugin.js';
|
|
29
30
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
31
|
+
export * from './puppeteer/remote-puppeteer-plugin.js';
|
|
30
32
|
export * from './events.js';
|
|
31
33
|
export type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
|
|
32
34
|
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
@@ -40,8 +42,7 @@ export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstrac
|
|
|
40
42
|
export type { LaunchContextOptions } from './launch-context.js';
|
|
41
43
|
export { LaunchContext } from './launch-context.js';
|
|
42
44
|
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
43
|
-
export {
|
|
44
|
-
export type { RemoteBrowserPoolOptions, CrawlerRemoteBrowserOptions, RemoteBrowserEndpoint, ResolvedRemoteEndpoint, RemoteConnection, RemoteConnectionParameters, } from './remote-browser-pool.js';
|
|
45
|
+
export type { RemoteBrowserEndpoint, RemoteBrowserOptions, ResolvedRemoteEndpoint } from './remote-browser-plugin.js';
|
|
45
46
|
export type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
|
|
46
47
|
export { anonymizeProxySugar, type AnonymizeProxySugarOptions } from './anonymize-proxy.js';
|
|
47
48
|
export type { IBrowserPool, NewPageOptions } from '@crawlee/types';
|
package/index.js
CHANGED
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
*/
|
|
25
25
|
export * from './browser-pool.js';
|
|
26
26
|
export * from './playwright/playwright-plugin.js';
|
|
27
|
+
export * from './playwright/remote-playwright-plugin.js';
|
|
27
28
|
export * from './puppeteer/puppeteer-plugin.js';
|
|
29
|
+
export * from './puppeteer/remote-puppeteer-plugin.js';
|
|
28
30
|
export * from './events.js';
|
|
29
31
|
export { BrowserName, DeviceCategory, OperatingSystemsName } from './fingerprinting/types.js';
|
|
30
32
|
export { BrowserController } from './abstract-classes/browser-controller.js';
|
|
@@ -34,5 +36,4 @@ export { PlaywrightBrowser } from './playwright/playwright-browser.js';
|
|
|
34
36
|
export { BrowserPlugin, BrowserLaunchError, DEFAULT_USER_AGENT } from './abstract-classes/browser-plugin.js';
|
|
35
37
|
export { LaunchContext } from './launch-context.js';
|
|
36
38
|
export { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
37
|
-
export { RemoteBrowserPool } from './remote-browser-pool.js';
|
|
38
39
|
export { anonymizeProxySugar } from './anonymize-proxy.js';
|
package/launch-context.d.ts
CHANGED
|
@@ -67,12 +67,6 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
|
|
|
67
67
|
readonly isRemote: boolean;
|
|
68
68
|
ignoreProxyCertificate?: boolean;
|
|
69
69
|
fingerprint?: BrowserFingerprintWithHeaders;
|
|
70
|
-
/**
|
|
71
|
-
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
72
|
-
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
73
|
-
* @internal
|
|
74
|
-
*/
|
|
75
|
-
remoteToken?: number;
|
|
76
70
|
[K: PropertyKey]: unknown;
|
|
77
71
|
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
78
72
|
/**
|
package/launch-context.js
CHANGED
|
@@ -10,12 +10,6 @@ export class LaunchContext {
|
|
|
10
10
|
#proxyUrl;
|
|
11
11
|
#reservedFieldNames;
|
|
12
12
|
fingerprint;
|
|
13
|
-
/**
|
|
14
|
-
* Token identifying the remote browser session this context connected to, set by the plugin and read by
|
|
15
|
-
* the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
|
|
16
|
-
* @internal
|
|
17
|
-
*/
|
|
18
|
-
remoteToken;
|
|
19
13
|
constructor(options) {
|
|
20
14
|
const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
|
|
21
15
|
this.id = id;
|
|
@@ -29,7 +23,7 @@ export class LaunchContext {
|
|
|
29
23
|
this.#proxyUrl = proxyUrl;
|
|
30
24
|
// Computed here (not in a field initializer) so that all fields already exist; the accessors live on
|
|
31
25
|
// the prototype, so they are never own keys and have to be listed explicitly.
|
|
32
|
-
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', '
|
|
26
|
+
this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'extend'];
|
|
33
27
|
}
|
|
34
28
|
/**
|
|
35
29
|
* Extend the launch context with any extra fields.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.224",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -31,9 +31,9 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@apify/timeout": "^1.0.1",
|
|
34
|
-
"@crawlee/core": "4.0.0-beta.
|
|
35
|
-
"@crawlee/types": "4.0.0-beta.
|
|
36
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.224",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.224",
|
|
36
|
+
"@crawlee/utils": "4.0.0-beta.224",
|
|
37
37
|
"fingerprint-generator": "^2.1.68",
|
|
38
38
|
"fingerprint-injector": "^2.1.68",
|
|
39
39
|
"lodash.merge": "^4.6.2",
|
|
@@ -64,5 +64,5 @@
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "669f726126327dee3b1d6b950663fd4fceace651"
|
|
68
68
|
}
|
|
@@ -2,18 +2,10 @@
|
|
|
2
2
|
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
3
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
4
4
|
import type { LaunchContext } from '../launch-context.js';
|
|
5
|
-
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
6
5
|
import type { SafeParameters } from '../utils.js';
|
|
7
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
8
7
|
export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
|
|
9
8
|
#private;
|
|
10
|
-
/**
|
|
11
|
-
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
12
|
-
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
13
|
-
*
|
|
14
|
-
* @internal
|
|
15
|
-
*/
|
|
16
|
-
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
17
9
|
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
18
10
|
createController(): PlaywrightController;
|
|
19
11
|
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
@@ -6,32 +6,7 @@ import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './p
|
|
|
6
6
|
import { PlaywrightController } from './playwright-controller.js';
|
|
7
7
|
export class PlaywrightPlugin extends BrowserPlugin {
|
|
8
8
|
#browserVersion;
|
|
9
|
-
/**
|
|
10
|
-
* Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
|
|
11
|
-
* accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
|
|
12
|
-
*
|
|
13
|
-
* @internal
|
|
14
|
-
*/
|
|
15
|
-
useRemoteConnection(connection, parameters = {}) {
|
|
16
|
-
super.useRemoteConnection(connection, parameters);
|
|
17
|
-
if (!this.useIncognitoPages) {
|
|
18
|
-
this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
|
|
19
|
-
'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
|
|
20
|
-
}
|
|
21
|
-
this.useIncognitoPages = true;
|
|
22
|
-
}
|
|
23
9
|
async _launch(launchContext) {
|
|
24
|
-
if (this.remoteConnection) {
|
|
25
|
-
return this.connectToRemoteBrowser(launchContext, async (url) => {
|
|
26
|
-
const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
|
|
27
|
-
if (this.remoteConnectionParameters?.protocol === 'playwright') {
|
|
28
|
-
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
29
|
-
return this.library.connect(url, connectOptions);
|
|
30
|
-
}
|
|
31
|
-
this.log.info('Connecting to remote browser via connectOverCDP.');
|
|
32
|
-
return this.library.connectOverCDP(url, connectOptions);
|
|
33
|
-
});
|
|
34
|
-
}
|
|
35
10
|
const { launchOptions, useIncognitoPages, userDataDir, proxyUrl } = launchContext;
|
|
36
11
|
let browser;
|
|
37
12
|
// Required for the `proxy` context option to work.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
+
import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
|
|
3
|
+
import type { BrowserPluginOptions, CreateLaunchContextOptions } from '../abstract-classes/browser-plugin.js';
|
|
4
|
+
import type { LaunchContext } from '../launch-context.js';
|
|
5
|
+
import { type RemoteBrowserOptions } from '../remote-browser-plugin.js';
|
|
6
|
+
import type { SafeParameters } from '../utils.js';
|
|
7
|
+
import { PlaywrightPlugin } from './playwright-plugin.js';
|
|
8
|
+
export interface RemotePlaywrightPluginOptions extends BrowserPluginOptions<SafeParameters<BrowserType['launch']>[0]>, RemoteBrowserOptions {
|
|
9
|
+
/**
|
|
10
|
+
* Which protocol to connect with. `'cdp'` uses `connectOverCDP()` (the default), `'playwright'` uses
|
|
11
|
+
* `connect()` (Playwright's own WebSocket protocol, for endpoints started with `launchServer()`).
|
|
12
|
+
*/
|
|
13
|
+
protocol?: 'cdp' | 'playwright';
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
17
|
+
* instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
18
|
+
*
|
|
19
|
+
* Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
|
|
20
|
+
* contexts), so `useIncognitoPages` is always `true`.
|
|
21
|
+
*/
|
|
22
|
+
export declare class RemotePlaywrightPlugin extends PlaywrightPlugin {
|
|
23
|
+
#private;
|
|
24
|
+
constructor(library: BrowserType, options: RemotePlaywrightPluginOptions);
|
|
25
|
+
createLaunchContext(options?: CreateLaunchContextOptions<BrowserType>): LaunchContext<BrowserType>;
|
|
26
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
27
|
+
launch(launchContext?: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
28
|
+
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { connectRemoteBrowser } from '../remote-browser-plugin.js';
|
|
2
|
+
import { PlaywrightPlugin } from './playwright-plugin.js';
|
|
3
|
+
/**
|
|
4
|
+
* A {@link PlaywrightPlugin} that connects to a remote browser service (Browserbase, Browserless, Steel, …)
|
|
5
|
+
* instead of launching locally. `launchOptions` are ignored — the remote service owns the browser.
|
|
6
|
+
*
|
|
7
|
+
* Remote connections only support incognito pages (`connect()` / `connectOverCDP()` don't accept persistent
|
|
8
|
+
* contexts), so `useIncognitoPages` is always `true`.
|
|
9
|
+
*/
|
|
10
|
+
export class RemotePlaywrightPlugin extends PlaywrightPlugin {
|
|
11
|
+
#remote;
|
|
12
|
+
constructor(library, options) {
|
|
13
|
+
super(library, { ...options, useIncognitoPages: true });
|
|
14
|
+
this.#remote = options;
|
|
15
|
+
if (options.useIncognitoPages === false) {
|
|
16
|
+
this.log.info('Remote Playwright connection — useIncognitoPages forced to true. ' +
|
|
17
|
+
'Pages will not share cookies/storage between each other; use the SessionPool for shared state.');
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
createLaunchContext(options = {}) {
|
|
21
|
+
return super.createLaunchContext({ ...options, isRemote: true });
|
|
22
|
+
}
|
|
23
|
+
/** Skips the local proxy and webdriver-stealth preparation; the remote service owns the browser. */
|
|
24
|
+
async launch(launchContext = this.createLaunchContext()) {
|
|
25
|
+
return this._launch(launchContext);
|
|
26
|
+
}
|
|
27
|
+
async _launch(launchContext) {
|
|
28
|
+
const { protocol, connectOptions = {} } = this.#remote;
|
|
29
|
+
return connectRemoteBrowser(this.#remote, launchContext.proxyUrl, this.log, async (url) => {
|
|
30
|
+
if (protocol === 'playwright') {
|
|
31
|
+
this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
|
|
32
|
+
return this.library.connect(url, connectOptions);
|
|
33
|
+
}
|
|
34
|
+
this.log.info('Connecting to remote browser via connectOverCDP.');
|
|
35
|
+
return this.library.connectOverCDP(url, connectOptions);
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -4,17 +4,19 @@ import type Puppeteer from 'puppeteer';
|
|
|
4
4
|
import type * as PuppeteerTypes from 'puppeteer';
|
|
5
5
|
import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
|
|
6
6
|
import type { LaunchContext } from '../launch-context.js';
|
|
7
|
-
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
8
7
|
import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
|
|
9
8
|
import { PuppeteerController } from './puppeteer-controller.js';
|
|
9
|
+
export type PuppeteerLaunchContext = LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>;
|
|
10
10
|
export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
|
11
|
+
protected _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser>;
|
|
12
|
+
/** @internal */
|
|
13
|
+
protected isOldPuppeteerVersion(): Promise<boolean>;
|
|
11
14
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
15
|
+
* Attaches page crash handling and the incognito-context-per-page `newPage` proxy. Shared by the local launch
|
|
16
|
+
* and {@link RemotePuppeteerPlugin}.
|
|
14
17
|
* @internal
|
|
15
18
|
*/
|
|
16
|
-
|
|
17
|
-
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
|
19
|
+
protected wrapBrowser(browser: PuppeteerTypes.Browser, launchContext: PuppeteerLaunchContext, oldPuppeteerVersion: boolean): PuppeteerTypes.Browser;
|
|
18
20
|
createController(): PuppeteerController;
|
|
19
21
|
protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
|
20
22
|
protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|