@crawlee/browser-pool 4.0.0-beta.5 → 4.0.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/abstract-classes/browser-controller.d.ts +3 -6
- package/abstract-classes/browser-controller.d.ts.map +1 -1
- package/abstract-classes/browser-controller.js +6 -9
- package/abstract-classes/browser-controller.js.map +1 -1
- package/abstract-classes/browser-plugin.d.ts +8 -1
- package/abstract-classes/browser-plugin.d.ts.map +1 -1
- package/abstract-classes/browser-plugin.js +8 -4
- package/abstract-classes/browser-plugin.js.map +1 -1
- package/anonymize-proxy.d.ts +5 -1
- package/anonymize-proxy.d.ts.map +1 -1
- package/anonymize-proxy.js +8 -3
- package/anonymize-proxy.js.map +1 -1
- package/browser-pool.d.ts +5 -2
- package/browser-pool.d.ts.map +1 -1
- package/browser-pool.js +53 -25
- package/browser-pool.js.map +1 -1
- package/index.d.ts +10 -6
- package/index.d.ts.map +1 -1
- package/index.js +3 -2
- package/index.js.map +1 -1
- package/launch-context.d.ts +6 -2
- package/launch-context.d.ts.map +1 -1
- package/launch-context.js +3 -3
- package/launch-context.js.map +1 -1
- package/package.json +7 -8
- package/playwright/playwright-controller.d.ts.map +1 -1
- package/playwright/playwright-controller.js +16 -9
- package/playwright/playwright-controller.js.map +1 -1
- package/playwright/playwright-plugin.d.ts.map +1 -1
- package/playwright/playwright-plugin.js +4 -3
- package/playwright/playwright-plugin.js.map +1 -1
- package/puppeteer/puppeteer-controller.d.ts.map +1 -1
- package/puppeteer/puppeteer-controller.js +11 -8
- package/puppeteer/puppeteer-controller.js.map +1 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +1 -1
- package/puppeteer/puppeteer-plugin.js +11 -6
- package/puppeteer/puppeteer-plugin.js.map +1 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
+
<p align=center>
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
12
16
|
<p align=center>
|
|
13
17
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
18
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
85
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
86
90
|
|
|
87
91
|
```bash
|
|
88
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
89
93
|
```
|
|
90
94
|
|
|
91
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
94
98
|
{
|
|
95
99
|
"overrides": {
|
|
96
100
|
"apify": {
|
|
97
|
-
"@crawlee/core": "
|
|
98
|
-
"@crawlee/types": "
|
|
99
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
100
104
|
}
|
|
101
105
|
}
|
|
102
106
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type CrawleeLogger } from '@crawlee/core';
|
|
1
2
|
import type { Cookie, Dictionary } from '@crawlee/types';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
@@ -17,6 +18,7 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
|
|
|
17
18
|
*/
|
|
18
19
|
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
|
|
19
20
|
id: string;
|
|
21
|
+
protected log: CrawleeLogger;
|
|
20
22
|
/**
|
|
21
23
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
22
24
|
*/
|
|
@@ -30,12 +32,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
|
|
|
30
32
|
*/
|
|
31
33
|
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
|
32
34
|
/**
|
|
33
|
-
* The proxy
|
|
34
|
-
* `undefined` if no tiered proxy is used.
|
|
35
|
-
*/
|
|
36
|
-
proxyTier?: number;
|
|
37
|
-
/**
|
|
38
|
-
* The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
|
|
35
|
+
* The proxy URL used by the browser controller.
|
|
39
36
|
* `undefined` if no proxy is used
|
|
40
37
|
*/
|
|
41
38
|
proxyUrl?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-controller.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-controller.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAkB,MAAM,eAAe,CAAC;AACnE,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIvF,MAAM,WAAW,uBAAuB,CACpC,OAAO,SAAS,aAAa,EAC7B,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,CAAC,yBAAyB,CAAC,cAAc,CAAC,EAAE,CACxC,UAAU,EAAE,iBAAiB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,KAClG,IAAI,CAAC;CACb;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB,CACnC,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CACpE,SAAQ,YAAY,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IACjH,EAAE,SAAY;IACd,SAAS,CAAC,GAAG,EAAG,aAAa,CAAC;IAE9B;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IAEnG;;OAEG;IACH,OAAO,EAAE,YAAY,CAAc;IAEnC;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAc;IAEhH;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,QAAQ,UAAS;IAEjB,WAAW,SAAK;IAEhB,UAAU,SAAK;IAEf,gBAAgB,SAAc;IAE9B,OAAO,CAAC,SAAS,CAAc;IAE/B,OAAO,CAAC,eAAe,CAEpB;IAEH,OAAO,CAAC,aAAa,CAAc;IAEnC,OAAO,CAAC,iBAAiB,CAEtB;gBAES,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAM9G;;;;;OAKG;IACH,QAAQ,IAAI,IAAI;IAQhB;;OAEG;IACH,aAAa,CACT,OAAO,EAAE,YAAY,EACrB,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,IAAI;IASP;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB5B;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACG,OAAO,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7D,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjE,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIxD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAC1C;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IACzC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAEjF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAEtE;;OAEG;IACH,QAAQ,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1G"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { serviceLocator } from '@crawlee/core';
|
|
1
2
|
import { nanoid } from 'nanoid';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { tryCancel } from '@apify/timeout';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
5
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
6
|
/**
|
|
7
7
|
* The `BrowserController` serves two purposes. First, it is the base class that
|
|
@@ -13,6 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
|
13
13
|
*/
|
|
14
14
|
export class BrowserController extends TypedEmitter {
|
|
15
15
|
id = nanoid();
|
|
16
|
+
log;
|
|
16
17
|
/**
|
|
17
18
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
18
19
|
*/
|
|
@@ -26,12 +27,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
26
27
|
*/
|
|
27
28
|
launchContext = undefined;
|
|
28
29
|
/**
|
|
29
|
-
* The proxy
|
|
30
|
-
* `undefined` if no tiered proxy is used.
|
|
31
|
-
*/
|
|
32
|
-
proxyTier;
|
|
33
|
-
/**
|
|
34
|
-
* The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
|
|
30
|
+
* The proxy URL used by the browser controller.
|
|
35
31
|
* `undefined` if no proxy is used
|
|
36
32
|
*/
|
|
37
33
|
proxyUrl;
|
|
@@ -49,6 +45,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
49
45
|
});
|
|
50
46
|
constructor(browserPlugin) {
|
|
51
47
|
super();
|
|
48
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
52
49
|
this.browserPlugin = browserPlugin;
|
|
53
50
|
}
|
|
54
51
|
/**
|
|
@@ -89,12 +86,12 @@ export class BrowserController extends TypedEmitter {
|
|
|
89
86
|
this.isActive = false;
|
|
90
87
|
}
|
|
91
88
|
catch (error) {
|
|
92
|
-
log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
89
|
+
this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
93
90
|
}
|
|
94
91
|
this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
|
|
95
92
|
setTimeout(() => {
|
|
96
93
|
this._kill().catch((err) => {
|
|
97
|
-
log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
94
|
+
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
98
95
|
});
|
|
99
96
|
}, PROCESS_KILL_TIMEOUT_MILLIS);
|
|
100
97
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-controller.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"browser-controller.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO3C,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAczC;;;;;;;GAOG;AACH,MAAM,OAAgB,iBAMpB,SAAQ,YAA2G;IACjH,EAAE,GAAG,MAAM,EAAE,CAAC;IACJ,GAAG,CAAiB;IAE9B;;OAEG;IACH,aAAa,CAAsF;IAEnG;;OAEG;IACH,OAAO,GAAiB,SAAU,CAAC;IAEnC;;OAEG;IACH,aAAa,GAAwF,SAAU,CAAC;IAEhH;;;OAGG;IACH,QAAQ,CAAU;IAElB,QAAQ,GAAG,KAAK,CAAC;IAEjB,WAAW,GAAG,CAAC,CAAC;IAEhB,UAAU,GAAG,CAAC,CAAC;IAEf,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEtB,SAAS,CAAc;IAEvB,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACpD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEK,aAAa,CAAc;IAE3B,iBAAiB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACtD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,YAAY,aAAkG;QAC1G,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,QAAQ;QACJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,aAAa,CACT,OAAqB,EACrB,aAAkG;QAElG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAE7B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,gDAAgD;YAChD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAqC,KAAe,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACpG,CAAC;QAED,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;QAE1D,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,GAAG,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC7B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,WAA4B;QACtC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,eAAe,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB,EAAE,OAAiB;QACnD,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CA6BJ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CriticalError } from '@crawlee/core';
|
|
1
|
+
import { type CrawleeLogger, CriticalError } from '@crawlee/core';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { LaunchContextOptions } from '../launch-context.js';
|
|
4
4
|
import { LaunchContext } from '../launch-context.js';
|
|
@@ -68,6 +68,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
|
|
|
68
68
|
* Might cause performance issues, as Crawlee might launch too many browser instances.
|
|
69
69
|
*/
|
|
70
70
|
browserPerProxy?: boolean;
|
|
71
|
+
/**
|
|
72
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
73
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
74
|
+
*/
|
|
75
|
+
ignoreProxyCertificate?: boolean;
|
|
71
76
|
}
|
|
72
77
|
export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
|
|
73
78
|
}
|
|
@@ -79,12 +84,14 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
|
|
|
79
84
|
*/
|
|
80
85
|
export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
81
86
|
name: string;
|
|
87
|
+
protected log: CrawleeLogger;
|
|
82
88
|
library: Library;
|
|
83
89
|
launchOptions: LibraryOptions;
|
|
84
90
|
proxyUrl?: string;
|
|
85
91
|
userDataDir?: string;
|
|
86
92
|
useIncognitoPages: boolean;
|
|
87
93
|
browserPerProxy?: boolean;
|
|
94
|
+
ignoreProxyCertificate?: boolean;
|
|
88
95
|
constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
|
|
89
96
|
/**
|
|
90
97
|
* Creates a `LaunchContext` with all the information needed
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-plugin.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-plugin.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,aAAa,EAAkB,MAAM,eAAe,CAAC;AAClF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,0HAC4F,CAAC;AAE5H;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,MAAM,MAAM,CAAC;CACvB;AAED,gBAAgB;AAChB,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACpD;AAED,gBAAgB;AAChB,MAAM,WAAW,UAAU;IACvB,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,GAAG,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,cAAc,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B,CACvC,OAAO,SAAS,aAAa,EAC7B,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CACpE,SAAQ,OAAO,CACb,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,eAAe,CAAC,CACpH;CAAG;AAEJ;;;;;GAKG;AACH,8BAAsB,aAAa,CAC/B,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,IAAI,SAAyB;IAC7B,SAAS,CAAC,GAAG,EAAG,aAAa,CAAC;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,sBAAsB,CAAC,EAAE,OAAO,CAAC;gBAErB,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,oBAAoB,CAAC,cAAc,CAAM;IAoBhF;;;;;OAKG;IACH,mBAAmB,CACf,OAAO,GAAE,0BAA0B,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAM,GAC/G,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAuBtF,QAAQ,CAAC,gBAAgB,IAAI,iBAAiB,CAC1C,OAAO,EACP,cAAc,EACd,YAAY,EACZ,cAAc,EACd,aAAa,CAChB;IAED;;OAEG;IACG,MAAM,CACR,aAAa,GAAE,aAAa,CACxB,OAAO,EACP,cAAc,EACd,YAAY,EACZ,cAAc,EACd,aAAa,CACa,GAC/B,OAAO,CAAC,YAAY,CAAC;IAwBxB,OAAO,CAAC,yBAAyB;IAgBjC,SAAS,CAAC,0BAA0B,CAChC,KAAK,EAAE,OAAO,EACd,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,WAAW,EAAE,MAAM,EACnB,oBAAoB,EAAE,MAAM,GAC7B,KAAK;IAuBR;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,wBAAwB,CACvC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CACtC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO;IAEV;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,OAAO,CACtB,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO,CAAC,YAAY,CAAC;CAC3B;AAED,qBAAa,kBAAmB,SAAQ,aAAa;gBAC9B,GAAG,IAAI,EAAE,qBAAqB,CAAC,OAAO,aAAa,CAAC;CAgB1E"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CriticalError } from '@crawlee/core';
|
|
1
|
+
import { CriticalError, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import merge from 'lodash.merge';
|
|
3
3
|
import { LaunchContext } from '../launch-context.js';
|
|
4
4
|
/**
|
|
@@ -20,20 +20,24 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
|
|
|
20
20
|
*/
|
|
21
21
|
export class BrowserPlugin {
|
|
22
22
|
name = this.constructor.name;
|
|
23
|
+
log;
|
|
23
24
|
library;
|
|
24
25
|
launchOptions;
|
|
25
26
|
proxyUrl;
|
|
26
27
|
userDataDir;
|
|
27
28
|
useIncognitoPages;
|
|
28
29
|
browserPerProxy;
|
|
30
|
+
ignoreProxyCertificate;
|
|
29
31
|
constructor(library, options = {}) {
|
|
30
|
-
const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
|
|
32
|
+
const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
|
|
33
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
31
34
|
this.library = library;
|
|
32
35
|
this.launchOptions = launchOptions;
|
|
33
36
|
this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
|
|
34
37
|
this.userDataDir = userDataDir;
|
|
35
38
|
this.useIncognitoPages = useIncognitoPages;
|
|
36
39
|
this.browserPerProxy = browserPerProxy;
|
|
40
|
+
this.ignoreProxyCertificate = ignoreProxyCertificate;
|
|
37
41
|
}
|
|
38
42
|
/**
|
|
39
43
|
* Creates a `LaunchContext` with all the information needed
|
|
@@ -42,7 +46,7 @@ export class BrowserPlugin {
|
|
|
42
46
|
* management of the pool and extra features.
|
|
43
47
|
*/
|
|
44
48
|
createLaunchContext(options = {}) {
|
|
45
|
-
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy,
|
|
49
|
+
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, } = options;
|
|
46
50
|
return new LaunchContext({
|
|
47
51
|
id,
|
|
48
52
|
launchOptions: merge({}, this.launchOptions, launchOptions),
|
|
@@ -51,7 +55,7 @@ export class BrowserPlugin {
|
|
|
51
55
|
useIncognitoPages,
|
|
52
56
|
userDataDir,
|
|
53
57
|
browserPerProxy,
|
|
54
|
-
|
|
58
|
+
ignoreProxyCertificate,
|
|
55
59
|
});
|
|
56
60
|
}
|
|
57
61
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-plugin.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"browser-plugin.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElF,OAAO,KAAK,MAAM,cAAc,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIrD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAC3B,uHAAuH,CAAC;AA2E5H;;;;;GAKG;AACH,MAAM,OAAgB,aAAa;IAO/B,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;IACnB,GAAG,CAAiB;IAC9B,OAAO,CAAU;IACjB,aAAa,CAAiB;IAC9B,QAAQ,CAAU;IAClB,WAAW,CAAU;IACrB,iBAAiB,CAAU;IAC3B,eAAe,CAAW;IAE1B,sBAAsB,CAAW;IAEjC,YAAY,OAAgB,EAAE,UAAgD,EAAE;QAC5E,MAAM,EACF,aAAa,GAAG,EAAoB,EACpC,QAAQ,EACR,WAAW,EACX,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,KAAK,EACvB,sBAAsB,GAAG,KAAK,GACjC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACzD,CAAC;IAED;;;;;OAKG;IACH,mBAAmB,CACf,UAA4G,EAAE;QAE9G,MAAM,EACF,EAAE,EACF,aAAa,GAAG,EAAE,EAClB,QAAQ,GAAG,IAAI,CAAC,QAAQ,EACxB,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAC1C,WAAW,GAAG,IAAI,CAAC,WAAW,EAC9B,eAAe,GAAG,IAAI,CAAC,eAAe,EACtC,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,GACvD,GAAG,OAAO,CAAC;QAEZ,OAAO,IAAI,aAAa,CAAC;YACrB,EAAE;YACF,aAAa,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC;YAC3D,aAAa,EAAE,IAAI;YACnB,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,eAAe;YACf,sBAAsB;SACzB,CAAC,CAAC;IACP,CAAC;IAUD;;OAEG;IACH,KAAK,CAAC,MAAM,CACR,gBAMI,IAAI,CAAC,mBAAmB,EAAE;QAE9B,aAAa,CAAC,aAAa,KAAK,EAAoB,CAAC;QAErD,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC;QAElD,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC;YAC9C,4EAA4E;YAC3E,aAA4B,CAAC,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,aAAc,CAAC,IAAI,CAAC,CAAC;YACzF,yEAAyE;YACzE,wEAAwE;YACxE,iEAAiE;YACjE,MAAM,SAAS,GAAG,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC;YAC5F,IAAI,aAAc,CAAC,QAAQ,IAAI,CAAC,aAAa,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtE,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,kBAAkB,EAAE,CAAC,CAAC;YACnE,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACvC,CAAC;IAEO,yBAAyB,CAAC,YAAuB;QACrD,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;YACxB,OAAO,CAAC,+CAA+C,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,aAAa,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;QAE3G,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,YAAY,CAAC,aAAa,CAAC,IAAI,uBAAuB,CAAC;QAC3D,CAAC;aAAM,CAAC;YACJ,YAAY,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAES,0BAA0B,CAChC,KAAc,EACd,cAAkC,EAClC,WAAmB,EACnB,oBAA4B;QAE5B,MAAM,YAAY,GAAG,CAAC,uDAAuD,CAAC,CAAC;QAE/E,IAAI,cAAc,EAAE,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC,iDAAiD,cAAc,eAAe,CAAC,CAAC;QACtG,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC/B,YAAY,CAAC,IAAI,CAAC,uCAAuC,WAAW,GAAG,CAAC,CAAC;QAC7E,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,KAAK,oBAAoB,EAAE,CAAC,CAAC;QAE/C,YAAY,CAAC,IAAI,CACb,EAAE,EACF,uHAAuH,EACvH,EAAE,CACL,CAAC;QAEF,oFAAoF;QACpF,MAAM,IAAI,kBAAkB,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAChF,CAAC;CAmBJ;AAED,MAAM,OAAO,kBAAmB,SAAQ,aAAa;IACjD,YAAmB,GAAG,IAAiD;QACnE,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QAEjC,MAAM,CAAC,EAAE,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE;YACjC,GAAG,EAAE,GAAG,EAAE;gBACN,IAAI,IAAI,CAAC,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC9B,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,uBAAuB,QAAQ,EAAE,CAAC;gBACjF,CAAC;gBAED,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC1C,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/anonymize-proxy.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
type PromiseVoid = () => Promise<void>;
|
|
2
|
-
export
|
|
2
|
+
export interface AnonymizeProxySugarOptions {
|
|
3
|
+
ignoreProxyCertificate?: boolean;
|
|
4
|
+
}
|
|
5
|
+
/** @internal */
|
|
6
|
+
export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
|
|
3
7
|
export {};
|
|
4
8
|
//# sourceMappingURL=anonymize-proxy.d.ts.map
|
package/anonymize-proxy.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize-proxy.d.ts","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAEA,KAAK,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,eAAO,MAAM,mBAAmB,GAC5B,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,WAAW,MAAM,
|
|
1
|
+
{"version":3,"file":"anonymize-proxy.d.ts","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAEA,KAAK,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,MAAM,WAAW,0BAA0B;IACvC,sBAAsB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,gBAAgB;AAChB,eAAO,MAAM,mBAAmB,GAC5B,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,UAAU,0BAA0B,KACrC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,EAAE,WAAW,CAAC,CA8B3C,CAAC"}
|
package/anonymize-proxy.js
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
|
|
2
|
-
|
|
2
|
+
/** @internal */
|
|
3
|
+
export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
|
|
3
4
|
if (proxyUrl) {
|
|
4
5
|
const url = new URL(proxyUrl);
|
|
5
6
|
if (username || password) {
|
|
6
7
|
url.username = username ?? '';
|
|
7
8
|
url.password = password ?? '';
|
|
8
9
|
}
|
|
9
|
-
if (url.username || url.password) {
|
|
10
|
+
if (url.username || url.password || options?.ignoreProxyCertificate) {
|
|
10
11
|
// trim off trailing slash if it's present
|
|
11
12
|
const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
|
|
12
|
-
const anonymized = await anonymizeProxy(
|
|
13
|
+
const anonymized = await anonymizeProxy({
|
|
14
|
+
url: proxyUrlString,
|
|
15
|
+
port: 0,
|
|
16
|
+
ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
|
|
17
|
+
});
|
|
13
18
|
return [
|
|
14
19
|
anonymized,
|
|
15
20
|
async () => {
|
package/anonymize-proxy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize-proxy.js","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"anonymize-proxy.js","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAQnE,gBAAgB;AAChB,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACpC,QAAiB,EACjB,QAAiB,EACjB,QAAiB,EACjB,OAAoC,EACM,EAAE;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;YAC9B,GAAG,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,IAAI,OAAO,EAAE,sBAAsB,EAAE,CAAC;YAClE,0CAA0C;YAC1C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YACjF,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC;gBACpC,GAAG,EAAE,cAAc;gBACnB,IAAI,EAAE,CAAC;gBACP,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,KAAK;aACnE,CAAC,CAAC;YAEH,OAAO;gBACH,UAAU;gBACV,KAAK,IAAI,EAAE;oBACP,MAAM,oBAAoB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACjD,CAAC;aACJ,CAAC;QACN,CAAC;QAED,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC,CAAC;AACvC,CAAC,CAAC"}
|
package/browser-pool.d.ts
CHANGED
|
@@ -263,6 +263,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
263
263
|
pageCounter: number;
|
|
264
264
|
pages: Map<string, PageReturn>;
|
|
265
265
|
pageIds: WeakMap<PageReturn, string>;
|
|
266
|
+
startingBrowserControllers: Set<BrowserControllerReturn>;
|
|
266
267
|
activeBrowserControllers: Set<BrowserControllerReturn>;
|
|
267
268
|
retiredBrowserControllers: Set<BrowserControllerReturn>;
|
|
268
269
|
pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
|
|
@@ -272,6 +273,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
272
273
|
private browserKillerInterval?;
|
|
273
274
|
private browserRetireInterval?;
|
|
274
275
|
private limiter;
|
|
276
|
+
private log;
|
|
275
277
|
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
|
276
278
|
/**
|
|
277
279
|
* Opens a new page in one of the running browsers or launches
|
|
@@ -401,9 +403,10 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
|
|
|
401
403
|
*/
|
|
402
404
|
proxyUrl?: string;
|
|
403
405
|
/**
|
|
404
|
-
*
|
|
406
|
+
* Disable TLS certificate verification for MITM proxies.
|
|
407
|
+
* Applied both when launching a new browser and when creating a page in an existing one.
|
|
405
408
|
*/
|
|
406
|
-
|
|
409
|
+
ignoreTlsErrors?: boolean;
|
|
407
410
|
}
|
|
408
411
|
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
|
|
409
412
|
/**
|
package/browser-pool.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAMlD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAMlD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAKzE,MAAM,WAAW,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI;IACjE,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE,CAAC,iBAAiB,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvF,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAAE,CAAC,iBAAiB,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3F;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAC;IAC1D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB,CAAC,MAAM,SAAS,aAAa,GAAG,aAAa;IAC5E;;;;OAIG;IACH,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;;OAQG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;OAMG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC;;;;;;;;OAQG;IACH,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;GAKG;AACH,MAAM,MAAM,aAAa,CAAC,EAAE,SAAS,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAElH;;;;;;;;GAQG;AACH,MAAM,MAAM,cAAc,CAAC,EAAE,SAAS,iBAAiB,IAAI,CACvD,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;;;GAQG;AACH,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,EAAE,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC7F,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,EACrB,WAAW,CAAC,EAAE,EAAE,KACf,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC5G,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;GAMG;AACH,MAAM,MAAM,gBAAgB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC1G,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,IAAI,CAC1D,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B,MAAM,WAAW,gBAAgB,CAC7B,EAAE,SAAS,iBAAiB,EAC5B,EAAE,SAAS,aAAa,EACxB,EAAE,SAAS,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAE9F;;;;;OAKG;IACH,cAAc,CAAC,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;IACrC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,cAAc,CAAC,EAAE,CAAC,EAAE,CAAC;IACvC;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,CAAC,EAAE,CAAC,EAAE,CAAC;IAC7C;;;;;;;OAOG;IACH,mBAAmB,CAAC,EAAE,kBAAkB,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IACnD;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,gBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IAC/C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,CAAC,EAAE,CAAC,EAAE,CAAC;CAChD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,kBAAkB,GAAG,kBAAkB,EACvD,cAAc,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAC3F,uBAAuB,SAAS,iBAAiB,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC1G,mBAAmB,SAAS,aAAa,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,EACrG,WAAW,GAAG,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/D,UAAU,SAAS,aAAa,CAAC,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,aAAa,CAC5F,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CACjD,CACH,SAAQ,YAAY,CAAC,iBAAiB,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;IAC1E,cAAc,EAAE,cAAc,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,2BAA2B,EAAE,MAAM,CAAC;IACpC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,+BAA+B,EAAE,MAAM,CAAC;IACxC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,EAAE,CAAC;IACrD,eAAe,EAAE,cAAc,CAAC,uBAAuB,CAAC,EAAE,CAAC;IAC3D,kBAAkB,EAAE,iBAAiB,CAAC,uBAAuB,EAAE,WAAW,CAAC,EAAE,CAAC;IAC9E,mBAAmB,EAAE,kBAAkB,CAAC,uBAAuB,EAAE,UAAU,CAAC,EAAE,CAAC;IAC/E,iBAAiB,EAAE,gBAAgB,CAAC,uBAAuB,EAAE,UAAU,CAAC,EAAE,CAAC;IAC3E,kBAAkB,EAAE,iBAAiB,CAAC,uBAAuB,CAAC,EAAE,CAAC;IACjE,WAAW,SAAK;IAChB,KAAK,0BAAiC;IACtC,OAAO,8BAAqC;IAC5C,0BAA0B,+BAAsC;IAChE,wBAAwB,+BAAsC;IAC9D,yBAAyB,+BAAsC;IAC/D,uBAAuB,+CAAsD;IAC7E,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IAEnE,OAAO,CAAC,qBAAqB,CAAC,CAG5B;IAEF,OAAO,CAAC,qBAAqB,CAAC,CAAiB;IAE/C,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,GAAG,CAAiB;gBAEhB,OAAO,EAAE,OAAO,GAAG,gBAAgB,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,UAAU,CAAC;IA+FzG;;;;OAIG;IACG,OAAO,CAAC,OAAO,GAAE,yBAAyB,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAiChH;;;;OAIG;IACG,mBAAmB,CACrB,OAAO,GAAE,qCAAqC,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAM,GACzF,OAAO,CAAC,UAAU,CAAC;IAYtB;;;;;;;;;;;;;;;;;;;OAmBG;IACG,qBAAqB,CACvB,WAAW,GAAE,IAAI,CAAC,yBAAyB,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAAC,EAAO,GAC1G,OAAO,CAAC,UAAU,EAAE,CAAC;IAWxB;;;;;;;;;;;;OAYG;IACH,0BAA0B,CAAC,IAAI,EAAE,UAAU,GAAG,uBAAuB,GAAG,SAAS;IAIjF;;;;;OAKG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAI3C;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS;YAIjC,qBAAqB;IAgEnC;;;;OAIG;IACH,uBAAuB,CAAC,iBAAiB,EAAE,uBAAuB,GAAG,IAAI;IAazE;;;OAGG;IACH,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAK3C;;;OAGG;IACH,iBAAiB,IAAI,IAAI;IAMzB;;;OAGG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAW9B,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,yBAAyB;YAQnB,cAAc;IA6D5B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,4BAA4B;YAgBtB,6BAA6B;IAyB3C,OAAO,CAAC,kBAAkB;YAqBZ,aAAa;IAM3B,OAAO,CAAC,+BAA+B;IAavC,OAAO,CAAC,yBAAyB;IAYjC,OAAO,CAAC,oBAAoB;CAU/B;AAED,MAAM,WAAW,yBAAyB,CAAC,WAAW,EAAE,EAAE,SAAS,aAAa;IAC5E;;;OAGG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,qCAAqC,CAAC,WAAW,EAAE,EAAE,SAAS,aAAa;IACxF;;;OAGG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC;IACnB;;OAEG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC;CACvC"}
|
package/browser-pool.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { serviceLocator } from '@crawlee/core';
|
|
1
2
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
3
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
4
|
import { nanoid } from 'nanoid';
|
|
@@ -7,7 +8,6 @@ import QuickLRU from 'quick-lru';
|
|
|
7
8
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
8
9
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
9
10
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
import { log } from './logger.js';
|
|
11
11
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
12
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
13
13
|
/**
|
|
@@ -74,6 +74,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
74
74
|
pageCounter = 0;
|
|
75
75
|
pages = new Map();
|
|
76
76
|
pageIds = new WeakMap();
|
|
77
|
+
startingBrowserControllers = new Set();
|
|
77
78
|
activeBrowserControllers = new Set();
|
|
78
79
|
retiredBrowserControllers = new Set();
|
|
79
80
|
pageToBrowserController = new WeakMap();
|
|
@@ -83,8 +84,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
83
84
|
browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
84
85
|
browserRetireInterval;
|
|
85
86
|
limiter = pLimit(1);
|
|
87
|
+
log;
|
|
86
88
|
constructor(options) {
|
|
87
89
|
super();
|
|
90
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
88
91
|
this.browserKillerInterval.unref();
|
|
89
92
|
ow(options, ow.object.exactShape({
|
|
90
93
|
browserPlugins: ow.array.minLength(1),
|
|
@@ -144,7 +147,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
144
147
|
* or their page limits have been exceeded.
|
|
145
148
|
*/
|
|
146
149
|
async newPage(options = {}) {
|
|
147
|
-
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl,
|
|
150
|
+
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, ignoreTlsErrors, } = options;
|
|
148
151
|
if (this.pages.has(id)) {
|
|
149
152
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
150
153
|
}
|
|
@@ -153,11 +156,15 @@ export class BrowserPool extends TypedEmitter {
|
|
|
153
156
|
}
|
|
154
157
|
// Limiter is necessary - https://github.com/apify/crawlee/issues/1126
|
|
155
158
|
return this.limiter(async () => {
|
|
156
|
-
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, {
|
|
159
|
+
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
|
|
157
160
|
if (!browserController)
|
|
158
|
-
browserController = await this._launchBrowser(id, {
|
|
161
|
+
browserController = await this._launchBrowser(id, {
|
|
162
|
+
browserPlugin,
|
|
163
|
+
proxyUrl,
|
|
164
|
+
ignoreTlsErrors,
|
|
165
|
+
});
|
|
159
166
|
tryCancel();
|
|
160
|
-
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
|
|
167
|
+
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
161
168
|
});
|
|
162
169
|
}
|
|
163
170
|
/**
|
|
@@ -238,7 +245,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
238
245
|
getPageId(page) {
|
|
239
246
|
return this.pageIds.get(page);
|
|
240
247
|
}
|
|
241
|
-
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
|
|
248
|
+
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
242
249
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
243
250
|
// It's not ideal though, we need to come up with a better API.
|
|
244
251
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -247,6 +254,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
247
254
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
248
255
|
if (finalPageOptions) {
|
|
249
256
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
257
|
+
if (ignoreTlsErrors) {
|
|
258
|
+
Object.assign(finalPageOptions, {
|
|
259
|
+
ignoreHTTPSErrors: true,
|
|
260
|
+
acceptInsecureCerts: true,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
250
263
|
}
|
|
251
264
|
await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
252
265
|
tryCancel();
|
|
@@ -278,11 +291,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
278
291
|
*
|
|
279
292
|
*/
|
|
280
293
|
retireBrowserController(browserController) {
|
|
281
|
-
const
|
|
294
|
+
const isStarting = this.startingBrowserControllers.has(browserController);
|
|
295
|
+
const isActive = this.activeBrowserControllers.has(browserController);
|
|
296
|
+
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
282
297
|
if (hasBeenRetiredOrKilled)
|
|
283
298
|
return;
|
|
284
299
|
this.retiredBrowserControllers.add(browserController);
|
|
285
300
|
this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
|
|
301
|
+
this.startingBrowserControllers.delete(browserController);
|
|
286
302
|
this.activeBrowserControllers.delete(browserController);
|
|
287
303
|
}
|
|
288
304
|
/**
|
|
@@ -299,7 +315,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
299
315
|
* closed after all their pages are closed.
|
|
300
316
|
*/
|
|
301
317
|
retireAllBrowsers() {
|
|
302
|
-
this.activeBrowserControllers.forEach((controller) => {
|
|
318
|
+
[...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
303
319
|
this.retireBrowserController(controller);
|
|
304
320
|
});
|
|
305
321
|
}
|
|
@@ -326,23 +342,36 @@ export class BrowserPool extends TypedEmitter {
|
|
|
326
342
|
this._teardown();
|
|
327
343
|
}
|
|
328
344
|
_teardown() {
|
|
345
|
+
this.startingBrowserControllers.clear();
|
|
329
346
|
this.activeBrowserControllers.clear();
|
|
330
347
|
this.retiredBrowserControllers.clear();
|
|
331
348
|
this.removeAllListeners();
|
|
332
349
|
}
|
|
333
350
|
_getAllBrowserControllers() {
|
|
334
|
-
return new Set([
|
|
351
|
+
return new Set([
|
|
352
|
+
...this.startingBrowserControllers,
|
|
353
|
+
...this.activeBrowserControllers,
|
|
354
|
+
...this.retiredBrowserControllers,
|
|
355
|
+
]);
|
|
335
356
|
}
|
|
336
357
|
async _launchBrowser(pageId, options) {
|
|
337
|
-
const { browserPlugin, launchOptions,
|
|
358
|
+
const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
|
|
338
359
|
const browserController = browserPlugin.createController();
|
|
339
|
-
this.
|
|
360
|
+
this.startingBrowserControllers.add(browserController);
|
|
340
361
|
const launchContext = browserPlugin.createLaunchContext({
|
|
341
362
|
id: pageId,
|
|
342
363
|
launchOptions,
|
|
343
|
-
proxyTier,
|
|
344
364
|
proxyUrl,
|
|
345
365
|
});
|
|
366
|
+
// Disable SSL verification for MITM proxies
|
|
367
|
+
if (ignoreTlsErrors) {
|
|
368
|
+
/**
|
|
369
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
370
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
371
|
+
*/
|
|
372
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
373
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
374
|
+
}
|
|
346
375
|
try {
|
|
347
376
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
348
377
|
// because otherwise it would be stuck in limbo without a browser.
|
|
@@ -353,11 +382,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
353
382
|
browserController.assignBrowser(browser, launchContext);
|
|
354
383
|
}
|
|
355
384
|
catch (err) {
|
|
356
|
-
this.
|
|
385
|
+
this.startingBrowserControllers.delete(browserController);
|
|
357
386
|
throw err;
|
|
358
387
|
}
|
|
359
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
360
|
-
browserController.proxyTier = proxyTier;
|
|
388
|
+
this.log.debug('Launched new browser.', { id: browserController.id });
|
|
361
389
|
browserController.proxyUrl = proxyUrl;
|
|
362
390
|
try {
|
|
363
391
|
// If the launch fails on the post-launch hooks, we need to clean up
|
|
@@ -365,9 +393,9 @@ export class BrowserPool extends TypedEmitter {
|
|
|
365
393
|
await this._executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
366
394
|
}
|
|
367
395
|
catch (err) {
|
|
368
|
-
this.
|
|
396
|
+
this.startingBrowserControllers.delete(browserController);
|
|
369
397
|
browserController.close().catch((closeErr) => {
|
|
370
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
398
|
+
this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
371
399
|
id: browserController.id,
|
|
372
400
|
});
|
|
373
401
|
});
|
|
@@ -375,6 +403,8 @@ export class BrowserPool extends TypedEmitter {
|
|
|
375
403
|
}
|
|
376
404
|
tryCancel();
|
|
377
405
|
browserController.activate();
|
|
406
|
+
this.startingBrowserControllers.delete(browserController);
|
|
407
|
+
this.activeBrowserControllers.add(browserController);
|
|
378
408
|
this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
|
|
379
409
|
return browserController;
|
|
380
410
|
}
|
|
@@ -392,13 +422,11 @@ export class BrowserPool extends TypedEmitter {
|
|
|
392
422
|
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
|
|
393
423
|
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
|
|
394
424
|
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
|
|
395
|
-
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
|
|
396
425
|
return (isCorrectPlugin &&
|
|
397
426
|
hasCapacity &&
|
|
398
|
-
(
|
|
399
|
-
(options?.proxyTier && isCorrectProxyTier) ||
|
|
427
|
+
(!controller.launchContext.browserPerProxy ||
|
|
400
428
|
(options?.proxyUrl && isSameProxyUrl) ||
|
|
401
|
-
(!options?.proxyUrl && !
|
|
429
|
+
(!options?.proxyUrl && !controller.proxyUrl)));
|
|
402
430
|
});
|
|
403
431
|
}
|
|
404
432
|
async _closeInactiveRetiredBrowsers() {
|
|
@@ -409,14 +437,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
409
437
|
const isBrowserEmpty = controller.activePages === 0;
|
|
410
438
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
411
439
|
const { id } = controller;
|
|
412
|
-
log.debug('Closing retired browser.', { id });
|
|
440
|
+
this.log.debug('Closing retired browser.', { id });
|
|
413
441
|
await controller.close();
|
|
414
442
|
this.retiredBrowserControllers.delete(controller);
|
|
415
443
|
closedBrowserIds.push(id);
|
|
416
444
|
}
|
|
417
445
|
}
|
|
418
446
|
if (closedBrowserIds.length) {
|
|
419
|
-
log.debug('Closed retired browsers.', {
|
|
447
|
+
this.log.debug('Closed retired browsers.', {
|
|
420
448
|
count: closedBrowserIds.length,
|
|
421
449
|
closedBrowserIds,
|
|
422
450
|
});
|
|
@@ -429,7 +457,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
429
457
|
page.close = async (...args) => {
|
|
430
458
|
await this._executeHooks(this.prePageCloseHooks, page, browserController);
|
|
431
459
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
432
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
460
|
+
this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
433
461
|
});
|
|
434
462
|
await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
435
463
|
this.pages.delete(pageId);
|
|
@@ -447,7 +475,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
447
475
|
// Run this with a delay, otherwise page.close()
|
|
448
476
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
449
477
|
setTimeout(() => {
|
|
450
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
478
|
+
this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
451
479
|
void browserController.close().finally(() => {
|
|
452
480
|
this.retiredBrowserControllers.delete(browserController);
|
|
453
481
|
});
|