@crawlee/browser-pool 4.0.0-beta.4 → 4.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/abstract-classes/browser-controller.d.ts +2 -0
- package/abstract-classes/browser-controller.d.ts.map +1 -1
- package/abstract-classes/browser-controller.js +5 -3
- package/abstract-classes/browser-controller.js.map +1 -1
- package/abstract-classes/browser-plugin.d.ts +8 -1
- package/abstract-classes/browser-plugin.d.ts.map +1 -1
- package/abstract-classes/browser-plugin.js +8 -3
- package/abstract-classes/browser-plugin.js.map +1 -1
- package/anonymize-proxy.d.ts +5 -1
- package/anonymize-proxy.d.ts.map +1 -1
- package/anonymize-proxy.js +8 -3
- package/anonymize-proxy.js.map +1 -1
- package/browser-pool.d.ts +7 -0
- package/browser-pool.d.ts.map +1 -1
- package/browser-pool.js +51 -18
- package/browser-pool.js.map +1 -1
- package/index.d.ts +2 -2
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/launch-context.d.ts +6 -0
- package/launch-context.d.ts.map +1 -1
- package/launch-context.js +3 -1
- package/launch-context.js.map +1 -1
- package/package.json +6 -7
- package/playwright/playwright-controller.d.ts.map +1 -1
- package/playwright/playwright-controller.js +16 -9
- package/playwright/playwright-controller.js.map +1 -1
- package/playwright/playwright-plugin.d.ts.map +1 -1
- package/playwright/playwright-plugin.js +4 -3
- package/playwright/playwright-plugin.js.map +1 -1
- package/puppeteer/puppeteer-controller.d.ts.map +1 -1
- package/puppeteer/puppeteer-controller.js +5 -6
- package/puppeteer/puppeteer-controller.js.map +1 -1
- package/puppeteer/puppeteer-plugin.d.ts.map +1 -1
- package/puppeteer/puppeteer-plugin.js +11 -6
- package/puppeteer/puppeteer-plugin.js.map +1 -1
- package/logger.d.ts +0 -3
- package/logger.d.ts.map +0 -1
- package/logger.js +0 -5
- package/logger.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
+
<p align=center>
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
12
16
|
<p align=center>
|
|
13
17
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
18
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
85
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
86
90
|
|
|
87
91
|
```bash
|
|
88
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
89
93
|
```
|
|
90
94
|
|
|
91
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
94
98
|
{
|
|
95
99
|
"overrides": {
|
|
96
100
|
"apify": {
|
|
97
|
-
"@crawlee/core": "
|
|
98
|
-
"@crawlee/types": "
|
|
99
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
100
104
|
}
|
|
101
105
|
}
|
|
102
106
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type CrawleeLogger } from '@crawlee/core';
|
|
1
2
|
import type { Cookie, Dictionary } from '@crawlee/types';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
@@ -17,6 +18,7 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
|
|
|
17
18
|
*/
|
|
18
19
|
export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
|
|
19
20
|
id: string;
|
|
21
|
+
protected log: CrawleeLogger;
|
|
20
22
|
/**
|
|
21
23
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
22
24
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-controller.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-controller.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAkB,MAAM,eAAe,CAAC;AACnE,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIvF,MAAM,WAAW,uBAAuB,CACpC,OAAO,SAAS,aAAa,EAC7B,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,CAAC,yBAAyB,CAAC,cAAc,CAAC,EAAE,CACxC,UAAU,EAAE,iBAAiB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,KAClG,IAAI,CAAC;CACb;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB,CACnC,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CACpE,SAAQ,YAAY,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IACjH,EAAE,SAAY;IACd,SAAS,CAAC,GAAG,EAAG,aAAa,CAAC;IAE9B;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IAEnG;;OAEG;IACH,OAAO,EAAE,YAAY,CAAc;IAEnC;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAc;IAEhH;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,QAAQ,UAAS;IAEjB,WAAW,SAAK;IAEhB,UAAU,SAAK;IAEf,gBAAgB,SAAc;IAE9B,OAAO,CAAC,SAAS,CAAc;IAE/B,OAAO,CAAC,eAAe,CAEpB;IAEH,OAAO,CAAC,aAAa,CAAc;IAEnC,OAAO,CAAC,iBAAiB,CAEtB;gBAES,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAM9G;;;;;OAKG;IACH,QAAQ,IAAI,IAAI;IAQhB;;OAEG;IACH,aAAa,CACT,OAAO,EAAE,YAAY,EACrB,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,IAAI;IASP;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB5B;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACG,OAAO,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7D,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjE,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIxD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAC1C;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IACzC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAEjF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAErF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAEtE;;OAEG;IACH,QAAQ,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1G"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { serviceLocator } from '@crawlee/core';
|
|
1
2
|
import { nanoid } from 'nanoid';
|
|
2
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
3
4
|
import { tryCancel } from '@apify/timeout';
|
|
4
|
-
import { log } from '../logger.js';
|
|
5
5
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
6
|
/**
|
|
7
7
|
* The `BrowserController` serves two purposes. First, it is the base class that
|
|
@@ -13,6 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
|
13
13
|
*/
|
|
14
14
|
export class BrowserController extends TypedEmitter {
|
|
15
15
|
id = nanoid();
|
|
16
|
+
log;
|
|
16
17
|
/**
|
|
17
18
|
* The `BrowserPlugin` instance used to launch the browser.
|
|
18
19
|
*/
|
|
@@ -49,6 +50,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
49
50
|
});
|
|
50
51
|
constructor(browserPlugin) {
|
|
51
52
|
super();
|
|
53
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
52
54
|
this.browserPlugin = browserPlugin;
|
|
53
55
|
}
|
|
54
56
|
/**
|
|
@@ -89,12 +91,12 @@ export class BrowserController extends TypedEmitter {
|
|
|
89
91
|
this.isActive = false;
|
|
90
92
|
}
|
|
91
93
|
catch (error) {
|
|
92
|
-
log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
94
|
+
this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
93
95
|
}
|
|
94
96
|
this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
|
|
95
97
|
setTimeout(() => {
|
|
96
98
|
this._kill().catch((err) => {
|
|
97
|
-
log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
99
|
+
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
98
100
|
});
|
|
99
101
|
}, PROCESS_KILL_TIMEOUT_MILLIS);
|
|
100
102
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-controller.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"browser-controller.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAE,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO3C,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAczC;;;;;;;GAOG;AACH,MAAM,OAAgB,iBAMpB,SAAQ,YAA2G;IACjH,EAAE,GAAG,MAAM,EAAE,CAAC;IACJ,GAAG,CAAiB;IAE9B;;OAEG;IACH,aAAa,CAAsF;IAEnG;;OAEG;IACH,OAAO,GAAiB,SAAU,CAAC;IAEnC;;OAEG;IACH,aAAa,GAAwF,SAAU,CAAC;IAEhH;;;OAGG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,QAAQ,CAAU;IAElB,QAAQ,GAAG,KAAK,CAAC;IAEjB,WAAW,GAAG,CAAC,CAAC;IAEhB,UAAU,GAAG,CAAC,CAAC;IAEf,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEtB,SAAS,CAAc;IAEvB,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACpD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEK,aAAa,CAAc;IAE3B,iBAAiB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACtD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,YAAY,aAAkG;QAC1G,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,QAAQ;QACJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,aAAa,CACT,OAAqB,EACrB,aAAkG;QAElG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAE7B,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,gDAAgD;YAChD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAqC,KAAe,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACpG,CAAC;QAED,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;QAE1D,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,GAAG,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC7B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,WAA4B;QACtC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,eAAe,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB,EAAE,OAAiB;QACnD,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CA6BJ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CriticalError } from '@crawlee/core';
|
|
1
|
+
import { type CrawleeLogger, CriticalError } from '@crawlee/core';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { LaunchContextOptions } from '../launch-context.js';
|
|
4
4
|
import { LaunchContext } from '../launch-context.js';
|
|
@@ -68,6 +68,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
|
|
|
68
68
|
* Might cause performance issues, as Crawlee might launch too many browser instances.
|
|
69
69
|
*/
|
|
70
70
|
browserPerProxy?: boolean;
|
|
71
|
+
/**
|
|
72
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
73
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
74
|
+
*/
|
|
75
|
+
ignoreProxyCertificate?: boolean;
|
|
71
76
|
}
|
|
72
77
|
export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
|
|
73
78
|
}
|
|
@@ -79,12 +84,14 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
|
|
|
79
84
|
*/
|
|
80
85
|
export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
|
81
86
|
name: string;
|
|
87
|
+
protected log: CrawleeLogger;
|
|
82
88
|
library: Library;
|
|
83
89
|
launchOptions: LibraryOptions;
|
|
84
90
|
proxyUrl?: string;
|
|
85
91
|
userDataDir?: string;
|
|
86
92
|
useIncognitoPages: boolean;
|
|
87
93
|
browserPerProxy?: boolean;
|
|
94
|
+
ignoreProxyCertificate?: boolean;
|
|
88
95
|
constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
|
|
89
96
|
/**
|
|
90
97
|
* Creates a `LaunchContext` with all the information needed
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-plugin.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-plugin.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,aAAa,EAAkB,MAAM,eAAe,CAAC;AAClF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,0HAC4F,CAAC;AAE5H;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,MAAM,MAAM,CAAC;CACvB;AAED,gBAAgB;AAChB,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACpD;AAED,gBAAgB;AAChB,MAAM,WAAW,UAAU;IACvB,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,GAAG,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,cAAc,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B,CACvC,OAAO,SAAS,aAAa,EAC7B,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CACpE,SAAQ,OAAO,CACT,IAAI,CACA,oBAAoB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,EAC1F,eAAe,CAClB,CACJ;CAAG;AAER;;;;;GAKG;AACH,8BAAsB,aAAa,CAC/B,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAChF,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,IAAI,SAAyB;IAC7B,SAAS,CAAC,GAAG,EAAG,aAAa,CAAC;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,sBAAsB,CAAC,EAAE,OAAO,CAAC;gBAErB,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,oBAAoB,CAAC,cAAc,CAAM;IAoBhF;;;;;OAKG;IACH,mBAAmB,CACf,OAAO,GAAE,0BAA0B,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAM,GAC/G,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAyBtF,QAAQ,CAAC,gBAAgB,IAAI,iBAAiB,CAC1C,OAAO,EACP,cAAc,EACd,YAAY,EACZ,cAAc,EACd,aAAa,CAChB;IAED;;OAEG;IACG,MAAM,CACR,aAAa,GAAE,aAAa,CACxB,OAAO,EACP,cAAc,EACd,YAAY,EACZ,cAAc,EACd,aAAa,CACa,GAC/B,OAAO,CAAC,YAAY,CAAC;IAwBxB,OAAO,CAAC,yBAAyB;IAgBjC,SAAS,CAAC,0BAA0B,CAChC,KAAK,EAAE,OAAO,EACd,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,WAAW,EAAE,MAAM,EACnB,oBAAoB,EAAE,MAAM,GAC7B,KAAK;IAuBR;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,wBAAwB,CACvC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CACtC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO;IAEV;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,OAAO,CACtB,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GACnG,OAAO,CAAC,YAAY,CAAC;CAC3B;AAED,qBAAa,kBAAmB,SAAQ,aAAa;gBAC9B,GAAG,IAAI,EAAE,qBAAqB,CAAC,OAAO,aAAa,CAAC;CAgB1E"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CriticalError } from '@crawlee/core';
|
|
1
|
+
import { CriticalError, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import merge from 'lodash.merge';
|
|
3
3
|
import { LaunchContext } from '../launch-context.js';
|
|
4
4
|
/**
|
|
@@ -20,20 +20,24 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
|
|
|
20
20
|
*/
|
|
21
21
|
export class BrowserPlugin {
|
|
22
22
|
name = this.constructor.name;
|
|
23
|
+
log;
|
|
23
24
|
library;
|
|
24
25
|
launchOptions;
|
|
25
26
|
proxyUrl;
|
|
26
27
|
userDataDir;
|
|
27
28
|
useIncognitoPages;
|
|
28
29
|
browserPerProxy;
|
|
30
|
+
ignoreProxyCertificate;
|
|
29
31
|
constructor(library, options = {}) {
|
|
30
|
-
const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
|
|
32
|
+
const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
|
|
33
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
31
34
|
this.library = library;
|
|
32
35
|
this.launchOptions = launchOptions;
|
|
33
36
|
this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
|
|
34
37
|
this.userDataDir = userDataDir;
|
|
35
38
|
this.useIncognitoPages = useIncognitoPages;
|
|
36
39
|
this.browserPerProxy = browserPerProxy;
|
|
40
|
+
this.ignoreProxyCertificate = ignoreProxyCertificate;
|
|
37
41
|
}
|
|
38
42
|
/**
|
|
39
43
|
* Creates a `LaunchContext` with all the information needed
|
|
@@ -42,7 +46,7 @@ export class BrowserPlugin {
|
|
|
42
46
|
* management of the pool and extra features.
|
|
43
47
|
*/
|
|
44
48
|
createLaunchContext(options = {}) {
|
|
45
|
-
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
|
|
49
|
+
const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, proxyTier, } = options;
|
|
46
50
|
return new LaunchContext({
|
|
47
51
|
id,
|
|
48
52
|
launchOptions: merge({}, this.launchOptions, launchOptions),
|
|
@@ -51,6 +55,7 @@ export class BrowserPlugin {
|
|
|
51
55
|
useIncognitoPages,
|
|
52
56
|
userDataDir,
|
|
53
57
|
browserPerProxy,
|
|
58
|
+
ignoreProxyCertificate,
|
|
54
59
|
proxyTier,
|
|
55
60
|
});
|
|
56
61
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-plugin.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"browser-plugin.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAElF,OAAO,KAAK,MAAM,cAAc,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIrD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAC3B,uHAAuH,CAAC;AA8E5H;;;;;GAKG;AACH,MAAM,OAAgB,aAAa;IAO/B,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;IACnB,GAAG,CAAiB;IAC9B,OAAO,CAAU;IACjB,aAAa,CAAiB;IAC9B,QAAQ,CAAU;IAClB,WAAW,CAAU;IACrB,iBAAiB,CAAU;IAC3B,eAAe,CAAW;IAE1B,sBAAsB,CAAW;IAEjC,YAAY,OAAgB,EAAE,UAAgD,EAAE;QAC5E,MAAM,EACF,aAAa,GAAG,EAAoB,EACpC,QAAQ,EACR,WAAW,EACX,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,KAAK,EACvB,sBAAsB,GAAG,KAAK,GACjC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACzD,CAAC;IAED;;;;;OAKG;IACH,mBAAmB,CACf,UAA4G,EAAE;QAE9G,MAAM,EACF,EAAE,EACF,aAAa,GAAG,EAAE,EAClB,QAAQ,GAAG,IAAI,CAAC,QAAQ,EACxB,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAC1C,WAAW,GAAG,IAAI,CAAC,WAAW,EAC9B,eAAe,GAAG,IAAI,CAAC,eAAe,EACtC,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,EACpD,SAAS,GACZ,GAAG,OAAO,CAAC;QAEZ,OAAO,IAAI,aAAa,CAAC;YACrB,EAAE;YACF,aAAa,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC;YAC3D,aAAa,EAAE,IAAI;YACnB,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,eAAe;YACf,sBAAsB;YACtB,SAAS;SACZ,CAAC,CAAC;IACP,CAAC;IAUD;;OAEG;IACH,KAAK,CAAC,MAAM,CACR,gBAMI,IAAI,CAAC,mBAAmB,EAAE;QAE9B,aAAa,CAAC,aAAa,KAAK,EAAoB,CAAC;QAErD,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC;QAElD,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC;YAC9C,4EAA4E;YAC3E,aAA4B,CAAC,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,aAAc,CAAC,IAAI,CAAC,CAAC;YACzF,yEAAyE;YACzE,wEAAwE;YACxE,iEAAiE;YACjE,MAAM,SAAS,GAAG,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC;YAC5F,IAAI,aAAc,CAAC,QAAQ,IAAI,CAAC,aAAa,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtE,aAAc,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,kBAAkB,EAAE,CAAC,CAAC;YACnE,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACvC,CAAC;IAEO,yBAAyB,CAAC,YAAuB;QACrD,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;YACxB,OAAO,CAAC,+CAA+C,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,aAAa,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;QAE3G,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,YAAY,CAAC,aAAa,CAAC,IAAI,uBAAuB,CAAC;QAC3D,CAAC;aAAM,CAAC;YACJ,YAAY,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAES,0BAA0B,CAChC,KAAc,EACd,cAAkC,EAClC,WAAmB,EACnB,oBAA4B;QAE5B,MAAM,YAAY,GAAG,CAAC,uDAAuD,CAAC,CAAC;QAE/E,IAAI,cAAc,EAAE,CAAC;YACjB,YAAY,CAAC,IAAI,CAAC,iDAAiD,cAAc,eAAe,CAAC,CAAC;QACtG,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC/B,YAAY,CAAC,IAAI,CAAC,uCAAuC,WAAW,GAAG,CAAC,CAAC;QAC7E,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,KAAK,oBAAoB,EAAE,CAAC,CAAC;QAE/C,YAAY,CAAC,IAAI,CACb,EAAE,EACF,uHAAuH,EACvH,EAAE,CACL,CAAC;QAEF,oFAAoF;QACpF,MAAM,IAAI,kBAAkB,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAChF,CAAC;CAmBJ;AAED,MAAM,OAAO,kBAAmB,SAAQ,aAAa;IACjD,YAAmB,GAAG,IAAiD;QACnE,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QAEjC,MAAM,CAAC,EAAE,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE;YACjC,GAAG,EAAE,GAAG,EAAE;gBACN,IAAI,IAAI,CAAC,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC9B,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,uBAAuB,QAAQ,EAAE,CAAC;gBACjF,CAAC;gBAED,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC1C,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ"}
|
package/anonymize-proxy.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
type PromiseVoid = () => Promise<void>;
|
|
2
|
-
export
|
|
2
|
+
export interface AnonymizeProxySugarOptions {
|
|
3
|
+
ignoreProxyCertificate?: boolean;
|
|
4
|
+
}
|
|
5
|
+
/** @internal */
|
|
6
|
+
export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
|
|
3
7
|
export {};
|
|
4
8
|
//# sourceMappingURL=anonymize-proxy.d.ts.map
|
package/anonymize-proxy.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize-proxy.d.ts","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAEA,KAAK,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,eAAO,MAAM,mBAAmB,GAC5B,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,WAAW,MAAM,
|
|
1
|
+
{"version":3,"file":"anonymize-proxy.d.ts","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAEA,KAAK,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEvC,MAAM,WAAW,0BAA0B;IACvC,sBAAsB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,gBAAgB;AAChB,eAAO,MAAM,mBAAmB,GAC5B,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,WAAW,MAAM,EACjB,UAAU,0BAA0B,KACrC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,EAAE,WAAW,CAAC,CA8B3C,CAAC"}
|
package/anonymize-proxy.js
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
|
|
2
|
-
|
|
2
|
+
/** @internal */
|
|
3
|
+
export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
|
|
3
4
|
if (proxyUrl) {
|
|
4
5
|
const url = new URL(proxyUrl);
|
|
5
6
|
if (username || password) {
|
|
6
7
|
url.username = username ?? '';
|
|
7
8
|
url.password = password ?? '';
|
|
8
9
|
}
|
|
9
|
-
if (url.username || url.password) {
|
|
10
|
+
if (url.username || url.password || options?.ignoreProxyCertificate) {
|
|
10
11
|
// trim off trailing slash if it's present
|
|
11
12
|
const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
|
|
12
|
-
const anonymized = await anonymizeProxy(
|
|
13
|
+
const anonymized = await anonymizeProxy({
|
|
14
|
+
url: proxyUrlString,
|
|
15
|
+
port: 0,
|
|
16
|
+
ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
|
|
17
|
+
});
|
|
13
18
|
return [
|
|
14
19
|
anonymized,
|
|
15
20
|
async () => {
|
package/anonymize-proxy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize-proxy.js","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"anonymize-proxy.js","sourceRoot":"","sources":["../src/anonymize-proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAQnE,gBAAgB;AAChB,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACpC,QAAiB,EACjB,QAAiB,EACjB,QAAiB,EACjB,OAAoC,EACM,EAAE;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;YAC9B,GAAG,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,IAAI,OAAO,EAAE,sBAAsB,EAAE,CAAC;YAClE,0CAA0C;YAC1C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YACjF,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC;gBACpC,GAAG,EAAE,cAAc;gBACnB,IAAI,EAAE,CAAC;gBACP,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,KAAK;aACnE,CAAC,CAAC;YAEH,OAAO;gBACH,UAAU;gBACV,KAAK,IAAI,EAAE;oBACP,MAAM,oBAAoB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACjD,CAAC;aACJ,CAAC;QACN,CAAC;QAED,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC,CAAC;AACvC,CAAC,CAAC"}
|
package/browser-pool.d.ts
CHANGED
|
@@ -263,6 +263,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
263
263
|
pageCounter: number;
|
|
264
264
|
pages: Map<string, PageReturn>;
|
|
265
265
|
pageIds: WeakMap<PageReturn, string>;
|
|
266
|
+
startingBrowserControllers: Set<BrowserControllerReturn>;
|
|
266
267
|
activeBrowserControllers: Set<BrowserControllerReturn>;
|
|
267
268
|
retiredBrowserControllers: Set<BrowserControllerReturn>;
|
|
268
269
|
pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
|
|
@@ -272,6 +273,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
|
|
|
272
273
|
private browserKillerInterval?;
|
|
273
274
|
private browserRetireInterval?;
|
|
274
275
|
private limiter;
|
|
276
|
+
private log;
|
|
275
277
|
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
|
276
278
|
/**
|
|
277
279
|
* Opens a new page in one of the running browsers or launches
|
|
@@ -404,6 +406,11 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
|
|
|
404
406
|
* Proxy tier.
|
|
405
407
|
*/
|
|
406
408
|
proxyTier?: number;
|
|
409
|
+
/**
|
|
410
|
+
* Disable TLS certificate verification for MITM proxies.
|
|
411
|
+
* Applied both when launching a new browser and when creating a page in an existing one.
|
|
412
|
+
*/
|
|
413
|
+
ignoreTlsErrors?: boolean;
|
|
407
414
|
}
|
|
408
415
|
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
|
|
409
416
|
/**
|
package/browser-pool.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAMlD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAMlD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAKzE,MAAM,WAAW,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI;IACjE,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE,CAAC,iBAAiB,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvF,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,EAAE,CAAC,iBAAiB,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3F;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAC;IAC1D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB,CAAC,MAAM,SAAS,aAAa,GAAG,aAAa;IAC5E;;;;OAIG;IACH,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;;OAQG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;OAMG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC;;;;;;;;OAQG;IACH,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;GAKG;AACH,MAAM,MAAM,aAAa,CAAC,EAAE,SAAS,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAElH;;;;;;;;GAQG;AACH,MAAM,MAAM,cAAc,CAAC,EAAE,SAAS,iBAAiB,IAAI,CACvD,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;;;GAQG;AACH,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,EAAE,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC7F,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,EACrB,WAAW,CAAC,EAAE,EAAE,KACf,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC5G,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;;;GAMG;AACH,MAAM,MAAM,gBAAgB,CAAC,EAAE,SAAS,iBAAiB,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAC1G,IAAI,EAAE,IAAI,EACV,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,iBAAiB,IAAI,CAC1D,MAAM,EAAE,MAAM,EACd,iBAAiB,EAAE,EAAE,KACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B,MAAM,WAAW,gBAAgB,CAC7B,EAAE,SAAS,iBAAiB,EAC5B,EAAE,SAAS,aAAa,EACxB,EAAE,SAAS,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAE9F;;;;;OAKG;IACH,cAAc,CAAC,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;IACrC;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,cAAc,CAAC,EAAE,CAAC,EAAE,CAAC;IACvC;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,CAAC,EAAE,CAAC,EAAE,CAAC;IAC7C;;;;;;;OAOG;IACH,mBAAmB,CAAC,EAAE,kBAAkB,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IACnD;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,gBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IAC/C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,CAAC,EAAE,CAAC,EAAE,CAAC;CAChD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,kBAAkB,GAAG,kBAAkB,EACvD,cAAc,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAC3F,uBAAuB,SAAS,iBAAiB,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC1G,mBAAmB,SAAS,aAAa,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,EACrG,WAAW,GAAG,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/D,UAAU,SAAS,aAAa,CAAC,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,aAAa,CAC5F,UAAU,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CACjD,CACH,SAAQ,YAAY,CAAC,iBAAiB,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;IAC1E,cAAc,EAAE,cAAc,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,2BAA2B,EAAE,MAAM,CAAC;IACpC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,+BAA+B,EAAE,MAAM,CAAC;IACxC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,EAAE,CAAC;IACrD,eAAe,EAAE,cAAc,CAAC,uBAAuB,CAAC,EAAE,CAAC;IAC3D,kBAAkB,EAAE,iBAAiB,CAAC,uBAAuB,EAAE,WAAW,CAAC,EAAE,CAAC;IAC9E,mBAAmB,EAAE,kBAAkB,CAAC,uBAAuB,EAAE,UAAU,CAAC,EAAE,CAAC;IAC/E,iBAAiB,EAAE,gBAAgB,CAAC,uBAAuB,EAAE,UAAU,CAAC,EAAE,CAAC;IAC3E,kBAAkB,EAAE,iBAAiB,CAAC,uBAAuB,CAAC,EAAE,CAAC;IACjE,WAAW,SAAK;IAChB,KAAK,0BAAiC;IACtC,OAAO,8BAAqC;IAC5C,0BAA0B,+BAAsC;IAChE,wBAAwB,+BAAsC;IAC9D,yBAAyB,+BAAsC;IAC/D,uBAAuB,+CAAsD;IAC7E,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IAEnE,OAAO,CAAC,qBAAqB,CAAC,CAG5B;IAEF,OAAO,CAAC,qBAAqB,CAAC,CAAiB;IAE/C,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,GAAG,CAAiB;gBAEhB,OAAO,EAAE,OAAO,GAAG,gBAAgB,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,UAAU,CAAC;IA+FzG;;;;OAIG;IACG,OAAO,CAAC,OAAO,GAAE,yBAAyB,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAmChH;;;;OAIG;IACG,mBAAmB,CACrB,OAAO,GAAE,qCAAqC,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAM,GACzF,OAAO,CAAC,UAAU,CAAC;IAYtB;;;;;;;;;;;;;;;;;;;OAmBG;IACG,qBAAqB,CACvB,WAAW,GAAE,IAAI,CAAC,yBAAyB,CAAC,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,eAAe,CAAC,EAAO,GAC1G,OAAO,CAAC,UAAU,EAAE,CAAC;IAWxB;;;;;;;;;;;;OAYG;IACH,0BAA0B,CAAC,IAAI,EAAE,UAAU,GAAG,uBAAuB,GAAG,SAAS;IAIjF;;;;;OAKG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAI3C;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS;YAIjC,qBAAqB;IAgEnC;;;;OAIG;IACH,uBAAuB,CAAC,iBAAiB,EAAE,uBAAuB,GAAG,IAAI;IAazE;;;OAGG;IACH,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAK3C;;;OAGG;IACH,iBAAiB,IAAI,IAAI;IAMzB;;;OAGG;IACG,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IASvC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAW9B,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,yBAAyB;YAQnB,cAAc;IA+D5B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,4BAA4B;YAkBtB,6BAA6B;IAyB3C,OAAO,CAAC,kBAAkB;YAqBZ,aAAa;IAM3B,OAAO,CAAC,+BAA+B;IAavC,OAAO,CAAC,yBAAyB;IAYjC,OAAO,CAAC,oBAAoB;CAU/B;AAED,MAAM,WAAW,yBAAyB,CAAC,WAAW,EAAE,EAAE,SAAS,aAAa;IAC5E;;;OAGG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,qCAAqC,CAAC,WAAW,EAAE,EAAE,SAAS,aAAa;IACxF;;;OAGG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC;IACnB;;OAEG;IACH,aAAa,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC;CACvC"}
|
package/browser-pool.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { serviceLocator } from '@crawlee/core';
|
|
1
2
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
2
3
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
3
4
|
import { nanoid } from 'nanoid';
|
|
@@ -7,7 +8,6 @@ import QuickLRU from 'quick-lru';
|
|
|
7
8
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
8
9
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
9
10
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
10
|
-
import { log } from './logger.js';
|
|
11
11
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
12
12
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
13
13
|
/**
|
|
@@ -74,6 +74,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
74
74
|
pageCounter = 0;
|
|
75
75
|
pages = new Map();
|
|
76
76
|
pageIds = new WeakMap();
|
|
77
|
+
startingBrowserControllers = new Set();
|
|
77
78
|
activeBrowserControllers = new Set();
|
|
78
79
|
retiredBrowserControllers = new Set();
|
|
79
80
|
pageToBrowserController = new WeakMap();
|
|
@@ -83,8 +84,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
83
84
|
browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
|
|
84
85
|
browserRetireInterval;
|
|
85
86
|
limiter = pLimit(1);
|
|
87
|
+
log;
|
|
86
88
|
constructor(options) {
|
|
87
89
|
super();
|
|
90
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
88
91
|
this.browserKillerInterval.unref();
|
|
89
92
|
ow(options, ow.object.exactShape({
|
|
90
93
|
browserPlugins: ow.array.minLength(1),
|
|
@@ -144,7 +147,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
144
147
|
* or their page limits have been exceeded.
|
|
145
148
|
*/
|
|
146
149
|
async newPage(options = {}) {
|
|
147
|
-
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
|
|
150
|
+
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier, ignoreTlsErrors, } = options;
|
|
148
151
|
if (this.pages.has(id)) {
|
|
149
152
|
throw new Error(`Page with ID: ${id} already exists.`);
|
|
150
153
|
}
|
|
@@ -155,9 +158,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
155
158
|
return this.limiter(async () => {
|
|
156
159
|
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
|
|
157
160
|
if (!browserController)
|
|
158
|
-
browserController = await this._launchBrowser(id, {
|
|
161
|
+
browserController = await this._launchBrowser(id, {
|
|
162
|
+
browserPlugin,
|
|
163
|
+
proxyTier,
|
|
164
|
+
proxyUrl,
|
|
165
|
+
ignoreTlsErrors,
|
|
166
|
+
});
|
|
159
167
|
tryCancel();
|
|
160
|
-
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
|
|
168
|
+
return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
|
|
161
169
|
});
|
|
162
170
|
}
|
|
163
171
|
/**
|
|
@@ -238,7 +246,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
238
246
|
getPageId(page) {
|
|
239
247
|
return this.pageIds.get(page);
|
|
240
248
|
}
|
|
241
|
-
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl) {
|
|
249
|
+
async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
|
|
242
250
|
// This is needed for concurrent newPage calls to wait for the browser launch.
|
|
243
251
|
// It's not ideal though, we need to come up with a better API.
|
|
244
252
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
@@ -247,6 +255,12 @@ export class BrowserPool extends TypedEmitter {
|
|
|
247
255
|
const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
|
|
248
256
|
if (finalPageOptions) {
|
|
249
257
|
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
|
|
258
|
+
if (ignoreTlsErrors) {
|
|
259
|
+
Object.assign(finalPageOptions, {
|
|
260
|
+
ignoreHTTPSErrors: true,
|
|
261
|
+
acceptInsecureCerts: true,
|
|
262
|
+
});
|
|
263
|
+
}
|
|
250
264
|
}
|
|
251
265
|
await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
|
|
252
266
|
tryCancel();
|
|
@@ -278,11 +292,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
278
292
|
*
|
|
279
293
|
*/
|
|
280
294
|
retireBrowserController(browserController) {
|
|
281
|
-
const
|
|
295
|
+
const isStarting = this.startingBrowserControllers.has(browserController);
|
|
296
|
+
const isActive = this.activeBrowserControllers.has(browserController);
|
|
297
|
+
const hasBeenRetiredOrKilled = !isStarting && !isActive;
|
|
282
298
|
if (hasBeenRetiredOrKilled)
|
|
283
299
|
return;
|
|
284
300
|
this.retiredBrowserControllers.add(browserController);
|
|
285
301
|
this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
|
|
302
|
+
this.startingBrowserControllers.delete(browserController);
|
|
286
303
|
this.activeBrowserControllers.delete(browserController);
|
|
287
304
|
}
|
|
288
305
|
/**
|
|
@@ -299,7 +316,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
299
316
|
* closed after all their pages are closed.
|
|
300
317
|
*/
|
|
301
318
|
retireAllBrowsers() {
|
|
302
|
-
this.activeBrowserControllers.forEach((controller) => {
|
|
319
|
+
[...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
|
|
303
320
|
this.retireBrowserController(controller);
|
|
304
321
|
});
|
|
305
322
|
}
|
|
@@ -326,23 +343,37 @@ export class BrowserPool extends TypedEmitter {
|
|
|
326
343
|
this._teardown();
|
|
327
344
|
}
|
|
328
345
|
_teardown() {
|
|
346
|
+
this.startingBrowserControllers.clear();
|
|
329
347
|
this.activeBrowserControllers.clear();
|
|
330
348
|
this.retiredBrowserControllers.clear();
|
|
331
349
|
this.removeAllListeners();
|
|
332
350
|
}
|
|
333
351
|
_getAllBrowserControllers() {
|
|
334
|
-
return new Set([
|
|
352
|
+
return new Set([
|
|
353
|
+
...this.startingBrowserControllers,
|
|
354
|
+
...this.activeBrowserControllers,
|
|
355
|
+
...this.retiredBrowserControllers,
|
|
356
|
+
]);
|
|
335
357
|
}
|
|
336
358
|
async _launchBrowser(pageId, options) {
|
|
337
|
-
const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
|
|
359
|
+
const { browserPlugin, launchOptions, proxyTier, proxyUrl, ignoreTlsErrors } = options;
|
|
338
360
|
const browserController = browserPlugin.createController();
|
|
339
|
-
this.
|
|
361
|
+
this.startingBrowserControllers.add(browserController);
|
|
340
362
|
const launchContext = browserPlugin.createLaunchContext({
|
|
341
363
|
id: pageId,
|
|
342
364
|
launchOptions,
|
|
343
365
|
proxyTier,
|
|
344
366
|
proxyUrl,
|
|
345
367
|
});
|
|
368
|
+
// Disable SSL verification for MITM proxies
|
|
369
|
+
if (ignoreTlsErrors) {
|
|
370
|
+
/**
|
|
371
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
372
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
373
|
+
*/
|
|
374
|
+
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
375
|
+
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
376
|
+
}
|
|
346
377
|
try {
|
|
347
378
|
// If the hooks or the launch fails, we need to delete the controller,
|
|
348
379
|
// because otherwise it would be stuck in limbo without a browser.
|
|
@@ -353,10 +384,10 @@ export class BrowserPool extends TypedEmitter {
|
|
|
353
384
|
browserController.assignBrowser(browser, launchContext);
|
|
354
385
|
}
|
|
355
386
|
catch (err) {
|
|
356
|
-
this.
|
|
387
|
+
this.startingBrowserControllers.delete(browserController);
|
|
357
388
|
throw err;
|
|
358
389
|
}
|
|
359
|
-
log.debug('Launched new browser.', { id: browserController.id });
|
|
390
|
+
this.log.debug('Launched new browser.', { id: browserController.id });
|
|
360
391
|
browserController.proxyTier = proxyTier;
|
|
361
392
|
browserController.proxyUrl = proxyUrl;
|
|
362
393
|
try {
|
|
@@ -365,9 +396,9 @@ export class BrowserPool extends TypedEmitter {
|
|
|
365
396
|
await this._executeHooks(this.postLaunchHooks, pageId, browserController);
|
|
366
397
|
}
|
|
367
398
|
catch (err) {
|
|
368
|
-
this.
|
|
399
|
+
this.startingBrowserControllers.delete(browserController);
|
|
369
400
|
browserController.close().catch((closeErr) => {
|
|
370
|
-
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
401
|
+
this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
|
|
371
402
|
id: browserController.id,
|
|
372
403
|
});
|
|
373
404
|
});
|
|
@@ -375,6 +406,8 @@ export class BrowserPool extends TypedEmitter {
|
|
|
375
406
|
}
|
|
376
407
|
tryCancel();
|
|
377
408
|
browserController.activate();
|
|
409
|
+
this.startingBrowserControllers.delete(browserController);
|
|
410
|
+
this.activeBrowserControllers.add(browserController);
|
|
378
411
|
this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
|
|
379
412
|
return browserController;
|
|
380
413
|
}
|
|
@@ -409,14 +442,14 @@ export class BrowserPool extends TypedEmitter {
|
|
|
409
442
|
const isBrowserEmpty = controller.activePages === 0;
|
|
410
443
|
if (isBrowserIdle || isBrowserEmpty) {
|
|
411
444
|
const { id } = controller;
|
|
412
|
-
log.debug('Closing retired browser.', { id });
|
|
445
|
+
this.log.debug('Closing retired browser.', { id });
|
|
413
446
|
await controller.close();
|
|
414
447
|
this.retiredBrowserControllers.delete(controller);
|
|
415
448
|
closedBrowserIds.push(id);
|
|
416
449
|
}
|
|
417
450
|
}
|
|
418
451
|
if (closedBrowserIds.length) {
|
|
419
|
-
log.debug('Closed retired browsers.', {
|
|
452
|
+
this.log.debug('Closed retired browsers.', {
|
|
420
453
|
count: closedBrowserIds.length,
|
|
421
454
|
closedBrowserIds,
|
|
422
455
|
});
|
|
@@ -429,7 +462,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
429
462
|
page.close = async (...args) => {
|
|
430
463
|
await this._executeHooks(this.prePageCloseHooks, page, browserController);
|
|
431
464
|
await originalPageClose.apply(page, args).catch((err) => {
|
|
432
|
-
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
465
|
+
this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
|
|
433
466
|
});
|
|
434
467
|
await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
435
468
|
this.pages.delete(pageId);
|
|
@@ -447,7 +480,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
447
480
|
// Run this with a delay, otherwise page.close()
|
|
448
481
|
// might fail with "Protocol error (Target.closeTarget): Target closed."
|
|
449
482
|
setTimeout(() => {
|
|
450
|
-
log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
483
|
+
this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
|
|
451
484
|
void browserController.close().finally(() => {
|
|
452
485
|
this.retiredBrowserControllers.delete(browserController);
|
|
453
486
|
});
|