@crawlee/browser 4.0.0-beta.32 → 4.0.0-beta.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
|
@@ -44,6 +44,11 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
|
|
|
44
44
|
* to reduce the chance of detection of the crawler.
|
|
45
45
|
*/
|
|
46
46
|
userAgent?: string;
|
|
47
|
+
/**
|
|
48
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
49
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
50
|
+
*/
|
|
51
|
+
ignoreProxyCertificate?: boolean;
|
|
47
52
|
/**
|
|
48
53
|
* The type of browser to be launched.
|
|
49
54
|
* By default, `chromium` is used. Other browsers like `webkit` or `firefox` can be used.
|
|
@@ -81,6 +86,8 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
81
86
|
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
82
87
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
83
88
|
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
89
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
90
|
+
ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
84
91
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
85
92
|
userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
86
93
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACjF,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAU9D,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,SAAS,UAAU,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC5F,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACjF,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAU9D,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAEjC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,SAAS,UAAU,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC5F,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IA6C5E,QAAQ,CAAC,MAAM;IA3CnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,uBAAuB,EAAE,UAAU,CAAC;IAEpC,MAAM,EAAG,CAAC,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;MAS3B;IAEF,MAAM,CAAC,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC;IAkB7E;;OAEG;gBAEC,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,QAAQ,CAAC,EACnD,MAAM,gBAAkC;IAsBrD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAQ7B;;;OAGG;IACH,MAAM,IAAI,YAAY;IAOtB,mBAAmB,IAAI,UAAU;IA0BjC,SAAS,CAAC,yBAAyB,IAAI,OAAO;IAI9C,SAAS,CAAC,wBAAwB,IAAI,MAAM;IAI5C;;OAEG;IACH,SAAS,CAAC,+BAA+B,IAAI,MAAM;IA6BnD,SAAS,CAAC,yBAAyB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;CAe/D"}
|
|
@@ -27,6 +27,7 @@ export class BrowserLauncher {
|
|
|
27
27
|
useChrome: ow.optional.boolean,
|
|
28
28
|
useIncognitoPages: ow.optional.boolean,
|
|
29
29
|
browserPerProxy: ow.optional.boolean,
|
|
30
|
+
ignoreProxyCertificate: ow.optional.boolean,
|
|
30
31
|
userDataDir: ow.optional.string,
|
|
31
32
|
launchOptions: ow.optional.object,
|
|
32
33
|
userAgent: ow.optional.string,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-launcher.js","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,gBAAgB,GAAG;IACrB,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-launcher.js","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,gBAAgB,GAAG;IACrB,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAwE/C;;;GAGG;AACH,MAAM,OAAgB,eAAe;IAkDpB;IA3Cb,QAAQ,CAAW;IACnB,QAAQ,CAAU;IAClB,SAAS,CAAW;IACpB,aAAa,CAAa;IAC1B,uBAAuB,CAAa;IACpC,mCAAmC;IACnC,MAAM,CAAK;IACX,SAAS,CAAU;IAET,MAAM,CAAC,YAAY,GAAG;QAC5B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAChC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,sBAAsB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC3C,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC;IAEF,MAAM,CAAC,sBAAsB,CAAI,QAAgB,EAAE,cAAsB;QACrE,IAAI,CAAC;YACD,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,GAA+B,CAAC;YAC1C,IAAI,CAAC,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBAChC,MAAM,GAAG,GACL,uBAAuB,QAAQ,+BAA+B,QAAQ,cAAc;oBACpF,uBAAuB,QAAQ,mFAAmF,CAAC;gBACvH,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;oBAC/B,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,6BAA6B,QAAQ,+BAA+B,cAAc,gBAAgB,CAAC;gBACzH,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,YACI,aAA4D,EACnD,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,MAAM,EACF,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,SAAS,EACT,aAAa,GAAG,EAAE,EAClB,GAAG,uBAAuB,EAC7B,GAAG,aAAa,CAAC;QAElB,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAEzC,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,QAAS,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,uBAAqC,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,mBAAmB;QACf,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,mBAAmB,EAAE;YACzC,GAAG,IAAI,CAAC,uBAAuB;SAClC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAE7C,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAiB,CAAC;IAClD,CAAC;IAED,mBAAmB;QACf,MAAM,aAAa,GAAoC;YACnD,IAAI,EAAE,EAAE;YACR,eAAe,EAAE,gBAAgB;YACjC,GAAG,IAAI,CAAC,aAAa;SACxB,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YACjC,aAAa,CAAC,QAAQ,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC;YAClD,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnE,CAAC;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;IAES,yBAAyB;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3E,CAAC;IAES,wBAAwB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACO,+BAA+B;QACrC;;;;;WAKG;QACH,MAAM,YAAY,GAAG,GAAG,EAAE;YACtB,IAAI,oBAAoB,GAAG,4DAA4D,CAAC;YACxF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,2CAA2C,CAAC;YACtF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,2CAA2C,CAAC;YAE9F,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,oBAAoB,GAAG,MAAM,CAAC;YAClC,CAAC;iBAAM,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,oBAAoB,GAAG,MAAM,CAAC;YAClC,CAAC;YACD,OAAO,oBAAoB,CAAC;QAChC,CAAC,CAAC;QACF,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;YACpB,KAAK,QAAQ;gBACT,OAAO,8DAA8D,CAAC;YAC1E,KAAK,OAAO;gBACR,OAAO,YAAY,EAAE,CAAC;YAC1B;gBACI,OAAO,wBAAwB,CAAC;QACxC,CAAC;IACL,CAAC;IAES,yBAAyB,CAAC,QAAiB;QACjD,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,6CAA6C,QAAQ,GAAG,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;YACxG,CAAC;QACL,CAAC;IACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.33",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.33",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.33",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.33",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.33",
|
|
55
55
|
"ow": "^2.0.0",
|
|
56
56
|
"tslib": "^2.8.1",
|
|
57
57
|
"type-fest": "^4.41.0"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "8e8453c225c328d5fda0db45a8d818c3cb3a5fb6"
|
|
79
79
|
}
|