@crawlee/browser-pool 4.0.0-beta.21 → 4.0.0-beta.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/abstract-classes/browser-controller.d.ts +100 -29
- package/abstract-classes/browser-controller.js +70 -24
- package/abstract-classes/browser-plugin.d.ts +54 -23
- package/abstract-classes/browser-plugin.js +75 -13
- package/anonymize-proxy.d.ts +6 -2
- package/anonymize-proxy.js +9 -5
- package/browser-pool.d.ts +123 -49
- package/browser-pool.js +320 -147
- package/events.d.ts +2 -4
- package/events.js +0 -2
- package/fingerprinting/hooks.d.ts +0 -1
- package/fingerprinting/hooks.js +26 -10
- package/fingerprinting/types.d.ts +3 -34
- package/fingerprinting/types.js +0 -2
- package/fingerprinting/utils.d.ts +0 -1
- package/fingerprinting/utils.js +5 -6
- package/index.d.ts +14 -7
- package/index.js +5 -3
- package/launch-context.d.ts +21 -5
- package/launch-context.js +21 -10
- package/package.json +9 -9
- package/playwright/playwright-browser.d.ts +3 -6
- package/playwright/playwright-browser.js +17 -17
- package/playwright/playwright-controller.d.ts +1 -1
- package/playwright/playwright-controller.js +10 -6
- package/playwright/playwright-plugin.d.ts +12 -7
- package/playwright/playwright-plugin.js +42 -16
- package/proxy-server.d.ts +0 -1
- package/proxy-server.js +0 -1
- package/puppeteer/puppeteer-controller.d.ts +1 -1
- package/puppeteer/puppeteer-controller.js +29 -20
- package/puppeteer/puppeteer-plugin.d.ts +9 -3
- package/puppeteer/puppeteer-plugin.js +87 -46
- package/remote-browser-pool.d.ts +178 -0
- package/remote-browser-pool.js +206 -0
- package/remote-browser-provider.d.ts +83 -0
- package/remote-browser-provider.js +67 -0
- package/utils.d.ts +7 -1
- package/utils.js +19 -1
- package/abstract-classes/browser-controller.d.ts.map +0 -1
- package/abstract-classes/browser-controller.js.map +0 -1
- package/abstract-classes/browser-plugin.d.ts.map +0 -1
- package/abstract-classes/browser-plugin.js.map +0 -1
- package/anonymize-proxy.d.ts.map +0 -1
- package/anonymize-proxy.js.map +0 -1
- package/browser-pool.d.ts.map +0 -1
- package/browser-pool.js.map +0 -1
- package/container-proxy-server.d.ts +0 -12
- package/container-proxy-server.d.ts.map +0 -1
- package/container-proxy-server.js +0 -43
- package/container-proxy-server.js.map +0 -1
- package/events.d.ts.map +0 -1
- package/events.js.map +0 -1
- package/fingerprinting/hooks.d.ts.map +0 -1
- package/fingerprinting/hooks.js.map +0 -1
- package/fingerprinting/types.d.ts.map +0 -1
- package/fingerprinting/types.js.map +0 -1
- package/fingerprinting/utils.d.ts.map +0 -1
- package/fingerprinting/utils.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/launch-context.d.ts.map +0 -1
- package/launch-context.js.map +0 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/playwright/playwright-browser.d.ts.map +0 -1
- package/playwright/playwright-browser.js.map +0 -1
- package/playwright/playwright-controller.d.ts.map +0 -1
- package/playwright/playwright-controller.js.map +0 -1
- package/playwright/playwright-plugin.d.ts.map +0 -1
- package/playwright/playwright-plugin.js.map +0 -1
- package/proxy-server.d.ts.map +0 -1
- package/proxy-server.js.map +0 -1
- package/puppeteer/puppeteer-controller.d.ts.map +0 -1
- package/puppeteer/puppeteer-controller.js.map +0 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +0 -1
- package/puppeteer/puppeteer-plugin.js.map +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
|
@@ -1,9 +1,73 @@
|
|
|
1
|
+
import { type CrawleeLogger } from '@crawlee/core';
|
|
1
2
|
import type { Cookie, Dictionary } from '@crawlee/types';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
4
5
|
import type { LaunchContext } from '../launch-context.js';
|
|
5
6
|
import type { UnwrapPromise } from '../utils.js';
|
|
6
7
|
import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
|
|
8
|
+
/**
|
|
9
|
+
* The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
|
|
10
|
+
* Other fields are only available on the concrete `LaunchContext` class.
|
|
11
|
+
*/
|
|
12
|
+
export interface IBrowserLaunchContext {
|
|
13
|
+
/**
|
|
14
|
+
* The proxy URL the browser was launched with, if any.
|
|
15
|
+
*/
|
|
16
|
+
proxyUrl?: string;
|
|
17
|
+
/**
|
|
18
|
+
* The fingerprint applied to the browser, if fingerprinting is enabled.
|
|
19
|
+
* Typed as `unknown` here; cast to the concrete `LaunchContext` if you
|
|
20
|
+
* need the structured shape.
|
|
21
|
+
*/
|
|
22
|
+
fingerprint?: unknown;
|
|
23
|
+
/**
|
|
24
|
+
* `true` if each page in this browser uses its own context.
|
|
25
|
+
*/
|
|
26
|
+
useIncognitoPages?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* The actual options the browser was launched with, after pre-launch hooks.
|
|
29
|
+
*/
|
|
30
|
+
launchOptions?: Dictionary | undefined;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The minimal public contract of a browser controller.
|
|
34
|
+
*
|
|
35
|
+
* Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
|
|
36
|
+
* promises, …) is intentionally **not** part of this contract.
|
|
37
|
+
*
|
|
38
|
+
* @category Browser management
|
|
39
|
+
*/
|
|
40
|
+
export interface IBrowserController<Page = unknown> {
|
|
41
|
+
/**
|
|
42
|
+
* A stable identifier for this controller instance. Useful for tracking
|
|
43
|
+
* which browser served which request.
|
|
44
|
+
*/
|
|
45
|
+
readonly id: string;
|
|
46
|
+
/**
|
|
47
|
+
* The configuration the underlying browser was launched with — proxy URL,
|
|
48
|
+
* fingerprint, session, launcher-specific options, etc.
|
|
49
|
+
*/
|
|
50
|
+
readonly launchContext: IBrowserLaunchContext;
|
|
51
|
+
/**
|
|
52
|
+
* The raw browser handle from the underlying automation library
|
|
53
|
+
* (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
|
|
54
|
+
* Escape hatch for things the controller does not expose directly.
|
|
55
|
+
*/
|
|
56
|
+
readonly browser: unknown;
|
|
57
|
+
/**
|
|
58
|
+
* Reads cookies for the given page.
|
|
59
|
+
*/
|
|
60
|
+
getCookies(page: Page): Promise<Cookie[]>;
|
|
61
|
+
/**
|
|
62
|
+
* Writes cookies for the given page.
|
|
63
|
+
*/
|
|
64
|
+
setCookies(page: Page, cookies: Cookie[]): Promise<void>;
|
|
65
|
+
/**
|
|
66
|
+
* Gracefully closes the browser this controller owns. After this resolves,
|
|
67
|
+
* the controller is no longer usable.
|
|
68
|
+
*/
|
|
69
|
+
close(): Promise<void>;
|
|
70
|
+
}
|
|
7
71
|
export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
8
72
|
[BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
|
|
9
73
|
}
|
|
@@ -15,12 +79,20 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
|
|
|
15
79
|
* for the specialized classes, because it's the same for all of them.
|
|
16
80
|
* @hideconstructor
|
|
17
81
|
*/
|
|
18
|
-
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
|
|
19
|
-
|
|
82
|
+
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
|
|
83
|
+
#private;
|
|
84
|
+
readonly id: string;
|
|
85
|
+
/**
|
|
86
|
+
* Kept `protected` rather than `#private` because the concrete controllers in this package are
|
|
87
|
+
* separate classes — an ES private field would not be visible to them.
|
|
88
|
+
*
|
|
89
|
+
* @internal
|
|
90
|
+
*/
|
|
91
|
+
protected readonly log: CrawleeLogger;
|
|
20
92
|
/**
|
|
21
93
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
22
94
|
*/
|
|
23
|
-
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
95
|
+
readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
24
96
|
/**
|
|
25
97
|
* Browser representation of the underlying automation library.
|
|
26
98
|
*/
|
|
@@ -30,23 +102,21 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
|
|
|
30
102
|
*/
|
|
31
103
|
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
32
104
|
/**
|
|
33
|
-
* The proxy
|
|
34
|
-
* `undefined` if no tiered proxy is used.
|
|
35
|
-
*/
|
|
36
|
-
proxyTier?: number;
|
|
37
|
-
/**
|
|
38
|
-
* The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
|
|
105
|
+
* The proxy URL used by the browser controller.
|
|
39
106
|
* `undefined` if no proxy is used
|
|
40
107
|
*/
|
|
41
108
|
proxyUrl?: string;
|
|
109
|
+
/** @internal */
|
|
42
110
|
isActive: boolean;
|
|
43
111
|
activePages: number;
|
|
112
|
+
/** @internal */
|
|
44
113
|
totalPages: number;
|
|
114
|
+
/** @internal */
|
|
45
115
|
lastPageOpenedAt: number;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
116
|
+
/**
|
|
117
|
+
* Returns a promise that resolves once the browser controller has been activated.
|
|
118
|
+
*/
|
|
119
|
+
waitForActive(): Promise<void>;
|
|
50
120
|
constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
|
51
121
|
/**
|
|
52
122
|
* Activates the BrowserController. If you try to open new pages before
|
|
@@ -77,31 +147,32 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
|
|
|
77
147
|
* @ignore
|
|
78
148
|
*/
|
|
79
149
|
newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
|
|
80
|
-
setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
|
|
81
|
-
getCookies(page: NewPageResult): Promise<Cookie[]>;
|
|
82
150
|
/**
|
|
83
|
-
*
|
|
151
|
+
* Accounts for a page no longer being open. Callable from either side and safe to call twice:
|
|
152
|
+
* the page's own `close` event, or the pool when it gives up on a close that never settles.
|
|
153
|
+
* Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
|
|
154
|
+
* @internal
|
|
84
155
|
*/
|
|
85
|
-
|
|
156
|
+
registerPageClosed(page: NewPageResult): void;
|
|
86
157
|
/**
|
|
87
|
-
*
|
|
158
|
+
* Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
|
|
159
|
+
* context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
|
|
160
|
+
* or the pool gave up on a close that never settled, and it runs at most once.
|
|
161
|
+
*
|
|
162
|
+
* Hanging it off the page's `close` event instead leaks it in exactly the case this controller
|
|
163
|
+
* now handles, because that event does not arrive for a page the browser never destroyed.
|
|
164
|
+
* @ignore
|
|
88
165
|
*/
|
|
166
|
+
protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
|
|
167
|
+
setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
|
|
168
|
+
getCookies(page: NewPageResult): Promise<Cookie[]>;
|
|
169
|
+
protected abstract _close(): Promise<void>;
|
|
89
170
|
protected abstract _kill(): Promise<void>;
|
|
90
|
-
/**
|
|
91
|
-
* @private
|
|
92
|
-
*/
|
|
93
171
|
protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
|
|
94
|
-
/**
|
|
95
|
-
* @private
|
|
96
|
-
*/
|
|
97
172
|
protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
|
|
98
|
-
/**
|
|
99
|
-
* @private
|
|
100
|
-
*/
|
|
101
173
|
protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
|
|
102
174
|
/**
|
|
103
|
-
* @
|
|
175
|
+
* @internal
|
|
104
176
|
*/
|
|
105
177
|
abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
|
106
178
|
}
|
|
107
|
-
//# sourceMappingURL=browser-controller.d.ts.map
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { serviceLocator } from '@crawlee/core';
|
|
1
2
|
import { nanoid } from 'nanoid';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { tryCancel } from '@apify/timeout';
|
|
4
|
-
import {
|
|
5
|
+
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
5
6
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
7
|
/**
|
|
7
8
|
* The `BrowserController` serves two purposes. First, it is the base class that
|
|
@@ -13,6 +14,13 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
|
13
14
|
*/
|
|
14
15
|
export class BrowserController extends TypedEmitter {
|
|
15
16
|
id = nanoid();
|
|
17
|
+
/**
|
|
18
|
+
* Kept `protected` rather than `#private` because the concrete controllers in this package are
|
|
19
|
+
* separate classes — an ES private field would not be visible to them.
|
|
20
|
+
*
|
|
21
|
+
* @internal
|
|
22
|
+
*/
|
|
23
|
+
log;
|
|
16
24
|
/**
|
|
17
25
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
18
26
|
*/
|
|
@@ -26,29 +34,36 @@ export class BrowserController extends TypedEmitter {
|
|
|
26
34
|
*/
|
|
27
35
|
launchContext = undefined;
|
|
28
36
|
/**
|
|
29
|
-
* The proxy
|
|
30
|
-
* `undefined` if no tiered proxy is used.
|
|
31
|
-
*/
|
|
32
|
-
proxyTier;
|
|
33
|
-
/**
|
|
34
|
-
* The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
|
|
37
|
+
* The proxy URL used by the browser controller.
|
|
35
38
|
* `undefined` if no proxy is used
|
|
36
39
|
*/
|
|
37
40
|
proxyUrl;
|
|
41
|
+
/** @internal */
|
|
38
42
|
isActive = false;
|
|
39
43
|
activePages = 0;
|
|
44
|
+
#closedPages = new WeakSet();
|
|
45
|
+
#pageTeardowns = new WeakMap();
|
|
46
|
+
/** @internal */
|
|
40
47
|
totalPages = 0;
|
|
48
|
+
/** @internal */
|
|
41
49
|
lastPageOpenedAt = Date.now();
|
|
42
|
-
|
|
43
|
-
isActivePromise = new Promise((resolve) => {
|
|
44
|
-
this
|
|
50
|
+
#activate;
|
|
51
|
+
#isActivePromise = new Promise((resolve) => {
|
|
52
|
+
this.#activate = resolve;
|
|
45
53
|
});
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Returns a promise that resolves once the browser controller has been activated.
|
|
56
|
+
*/
|
|
57
|
+
async waitForActive() {
|
|
58
|
+
await this.#isActivePromise;
|
|
59
|
+
}
|
|
60
|
+
#commitBrowser;
|
|
61
|
+
#hasBrowserPromise = new Promise((resolve) => {
|
|
62
|
+
this.#commitBrowser = resolve;
|
|
49
63
|
});
|
|
50
64
|
constructor(browserPlugin) {
|
|
51
65
|
super();
|
|
66
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
52
67
|
this.browserPlugin = browserPlugin;
|
|
53
68
|
}
|
|
54
69
|
/**
|
|
@@ -61,7 +76,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
61
76
|
if (!this.browser) {
|
|
62
77
|
throw new Error('Cannot activate BrowserController without an assigned browser.');
|
|
63
78
|
}
|
|
64
|
-
this
|
|
79
|
+
this.#activate();
|
|
65
80
|
this.isActive = true;
|
|
66
81
|
}
|
|
67
82
|
/**
|
|
@@ -73,7 +88,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
73
88
|
}
|
|
74
89
|
this.browser = browser;
|
|
75
90
|
this.launchContext = launchContext;
|
|
76
|
-
this
|
|
91
|
+
this.#commitBrowser();
|
|
77
92
|
}
|
|
78
93
|
/**
|
|
79
94
|
* Gracefully closes the browser and makes sure
|
|
@@ -82,21 +97,22 @@ export class BrowserController extends TypedEmitter {
|
|
|
82
97
|
* Emits 'browserClosed' event.
|
|
83
98
|
*/
|
|
84
99
|
async close() {
|
|
85
|
-
await this
|
|
100
|
+
await this.#hasBrowserPromise;
|
|
86
101
|
try {
|
|
87
102
|
await this._close();
|
|
88
103
|
// TODO: shouldn't this go in a finally instead?
|
|
89
104
|
this.isActive = false;
|
|
90
105
|
}
|
|
91
106
|
catch (error) {
|
|
92
|
-
log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
107
|
+
this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
93
108
|
}
|
|
94
|
-
this.emit(
|
|
95
|
-
setTimeout(() => {
|
|
109
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
110
|
+
const killTimer = setTimeout(() => {
|
|
96
111
|
this._kill().catch((err) => {
|
|
97
|
-
log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
112
|
+
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
98
113
|
});
|
|
99
114
|
}, PROCESS_KILL_TIMEOUT_MILLIS);
|
|
115
|
+
killTimer.unref();
|
|
100
116
|
}
|
|
101
117
|
/**
|
|
102
118
|
* Immediately kills the browser process.
|
|
@@ -104,9 +120,9 @@ export class BrowserController extends TypedEmitter {
|
|
|
104
120
|
* Emits 'browserClosed' event.
|
|
105
121
|
*/
|
|
106
122
|
async kill() {
|
|
107
|
-
await this
|
|
123
|
+
await this.#hasBrowserPromise;
|
|
108
124
|
await this._kill();
|
|
109
|
-
this.emit(
|
|
125
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
110
126
|
}
|
|
111
127
|
/**
|
|
112
128
|
* Opens new browser page.
|
|
@@ -115,12 +131,43 @@ export class BrowserController extends TypedEmitter {
|
|
|
115
131
|
async newPage(pageOptions) {
|
|
116
132
|
this.activePages++;
|
|
117
133
|
this.totalPages++;
|
|
118
|
-
await this
|
|
134
|
+
await this.#isActivePromise;
|
|
119
135
|
const page = await this._newPage(pageOptions);
|
|
120
136
|
tryCancel();
|
|
121
137
|
this.lastPageOpenedAt = Date.now();
|
|
122
138
|
return page;
|
|
123
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* Accounts for a page no longer being open. Callable from either side and safe to call twice:
|
|
142
|
+
* the page's own `close` event, or the pool when it gives up on a close that never settles.
|
|
143
|
+
* Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
|
|
144
|
+
* @internal
|
|
145
|
+
*/
|
|
146
|
+
registerPageClosed(page) {
|
|
147
|
+
if (this.#closedPages.has(page))
|
|
148
|
+
return;
|
|
149
|
+
this.#closedPages.add(page);
|
|
150
|
+
this.activePages--;
|
|
151
|
+
const teardown = this.#pageTeardowns.get(page);
|
|
152
|
+
if (!teardown)
|
|
153
|
+
return;
|
|
154
|
+
this.#pageTeardowns.delete(page);
|
|
155
|
+
teardown().catch((error) => {
|
|
156
|
+
this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
|
|
161
|
+
* context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
|
|
162
|
+
* or the pool gave up on a close that never settled, and it runs at most once.
|
|
163
|
+
*
|
|
164
|
+
* Hanging it off the page's `close` event instead leaks it in exactly the case this controller
|
|
165
|
+
* now handles, because that event does not arrive for a page the browser never destroyed.
|
|
166
|
+
* @ignore
|
|
167
|
+
*/
|
|
168
|
+
registerPageTeardown(page, teardown) {
|
|
169
|
+
this.#pageTeardowns.set(page, teardown);
|
|
170
|
+
}
|
|
124
171
|
async setCookies(page, cookies) {
|
|
125
172
|
return this._setCookies(page, cookies);
|
|
126
173
|
}
|
|
@@ -128,4 +175,3 @@ export class BrowserController extends TypedEmitter {
|
|
|
128
175
|
return this._getCookies(page);
|
|
129
176
|
}
|
|
130
177
|
}
|
|
131
|
-
//# sourceMappingURL=browser-controller.js.map
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { CriticalError } from '@crawlee/core';
|
|
1
|
+
import { type CrawleeLogger, CriticalError } from '@crawlee/core';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { LaunchContextOptions } from '../launch-context.js';
|
|
4
4
|
import { LaunchContext } from '../launch-context.js';
|
|
5
|
-
import type {
|
|
5
|
+
import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
|
|
6
|
+
import { type UnwrapPromise } from '../utils.js';
|
|
6
7
|
import type { BrowserController } from './browser-controller.js';
|
|
7
8
|
/**
|
|
8
9
|
* The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
|
|
@@ -11,8 +12,6 @@ import type { BrowserController } from './browser-controller.js';
|
|
|
11
12
|
* - without using a fingerprint,
|
|
12
13
|
* - without specifying a user agent.
|
|
13
14
|
* Last updated on 2022-05-05.
|
|
14
|
-
*
|
|
15
|
-
* After you update it here, please update it also in jsdom-crawler.ts
|
|
16
15
|
*/
|
|
17
16
|
export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
|
|
18
17
|
/**
|
|
@@ -27,14 +26,13 @@ export interface CommonLibrary {
|
|
|
27
26
|
launch(opts?: Dictionary): Promise<CommonBrowser>;
|
|
28
27
|
name?: () => string;
|
|
29
28
|
}
|
|
30
|
-
/** @internal */
|
|
31
29
|
export interface CommonBrowser {
|
|
32
30
|
newPage(...args: unknown[]): Promise<CommonPage>;
|
|
33
31
|
}
|
|
34
|
-
/** @internal */
|
|
35
32
|
export interface CommonPage {
|
|
36
33
|
close(...args: unknown[]): Promise<unknown>;
|
|
37
34
|
url(): string | Promise<string>;
|
|
35
|
+
evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
|
|
38
36
|
}
|
|
39
37
|
export interface BrowserPluginOptions<LibraryOptions> {
|
|
40
38
|
/**
|
|
@@ -68,6 +66,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
|
|
|
68
66
|
* Might cause performance issues, as Crawlee might launch too many browser instances.
|
|
69
67
|
*/
|
|
70
68
|
browserPerProxy?: boolean;
|
|
69
|
+
/**
|
|
70
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
71
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
72
|
+
*/
|
|
73
|
+
ignoreProxyCertificate?: boolean;
|
|
71
74
|
}
|
|
72
75
|
export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
|
|
73
76
|
}
|
|
@@ -78,14 +81,49 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
|
|
|
78
81
|
* feed them to {@link BrowserPool} for use.
|
|
79
82
|
*/
|
|
80
83
|
export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
81
|
-
name: string;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
readonly name: string;
|
|
85
|
+
/**
|
|
86
|
+
* Kept `protected` rather than `#private` because the concrete plugins in this package are separate
|
|
87
|
+
* classes — an ES private field would not be visible to them.
|
|
88
|
+
*
|
|
89
|
+
* @internal
|
|
90
|
+
*/
|
|
91
|
+
protected readonly log: CrawleeLogger;
|
|
92
|
+
readonly library: Library;
|
|
93
|
+
readonly launchOptions: LibraryOptions;
|
|
94
|
+
readonly proxyUrl?: string;
|
|
95
|
+
readonly userDataDir?: string;
|
|
86
96
|
useIncognitoPages: boolean;
|
|
87
|
-
browserPerProxy?: boolean;
|
|
97
|
+
readonly browserPerProxy?: boolean;
|
|
98
|
+
readonly ignoreProxyCertificate?: boolean;
|
|
99
|
+
/**
|
|
100
|
+
* Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
|
|
101
|
+
* launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
|
|
102
|
+
* remote-session policy lives in the pool, not here.
|
|
103
|
+
*
|
|
104
|
+
* @internal
|
|
105
|
+
*/
|
|
106
|
+
remoteConnection?: RemoteConnection;
|
|
107
|
+
/** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
|
|
108
|
+
remoteConnectionParameters?: RemoteConnectionParameters;
|
|
88
109
|
constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
|
|
110
|
+
/**
|
|
111
|
+
* Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
|
|
112
|
+
* Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
|
|
113
|
+
* (e.g. forcing incognito pages).
|
|
114
|
+
*
|
|
115
|
+
* @internal
|
|
116
|
+
*/
|
|
117
|
+
useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
|
|
118
|
+
/**
|
|
119
|
+
* Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
|
|
120
|
+
* the launch context (so the controller can release it on close), and runs the library-specific `connect`.
|
|
121
|
+
* On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
|
|
122
|
+
*
|
|
123
|
+
* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
|
|
124
|
+
* lives here so it stays identical across plugins.
|
|
125
|
+
*/
|
|
126
|
+
protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
|
|
89
127
|
/**
|
|
90
128
|
* Creates a `LaunchContext` with all the information needed
|
|
91
129
|
* to launch a browser. Aside from library specific launch options,
|
|
@@ -98,19 +136,12 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
|
|
|
98
136
|
* Launches the browser using provided launch context.
|
|
99
137
|
*/
|
|
100
138
|
launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
|
101
|
-
private
|
|
102
|
-
protected
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
*/
|
|
106
|
-
protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
|
|
107
|
-
protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
|
|
108
|
-
/**
|
|
109
|
-
* @private
|
|
110
|
-
*/
|
|
139
|
+
private mergeArgsToHideWebdriver;
|
|
140
|
+
protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
|
|
141
|
+
protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
|
|
142
|
+
protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
|
|
111
143
|
protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
|
112
144
|
}
|
|
113
145
|
export declare class BrowserLaunchError extends CriticalError {
|
|
114
146
|
constructor(...args: ConstructorParameters<typeof CriticalError>);
|
|
115
147
|
}
|
|
116
|
-
//# sourceMappingURL=browser-plugin.d.ts.map
|