@crawlee/got-scraping-client 4.0.0-beta.24 → 4.0.0-beta.241

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +2 -9
  3. package/index.js +8 -8
  4. package/package.json +6 -6
  5. package/index.d.ts.map +0 -1
  6. package/index.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
package/index.d.ts CHANGED
@@ -3,13 +3,6 @@ import { BaseHttpClient, type CustomFetchOptions } from '@crawlee/http-client';
3
3
  * A HTTP client implementation based on the `got-scraping` library.
4
4
  */
5
5
  export declare class GotScrapingHttpClient extends BaseHttpClient {
6
- /**
7
- * Type guard that validates the HTTP method (excluding CONNECT).
8
- * @param request - The HTTP request to validate
9
- */
10
- private validateRequest;
11
- private iterateHeaders;
12
- private parseHeaders;
13
- fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
6
+ #private;
7
+ protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
14
8
  }
15
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -9,10 +9,10 @@ export class GotScrapingHttpClient extends BaseHttpClient {
9
9
  * Type guard that validates the HTTP method (excluding CONNECT).
10
10
  * @param request - The HTTP request to validate
11
11
  */
12
- validateRequest(request) {
12
+ #validateRequest(request) {
13
13
  return !['CONNECT', 'connect'].includes(request.method);
14
14
  }
15
- *iterateHeaders(headers) {
15
+ *#iterateHeaders(headers) {
16
16
  for (const [key, value] of Object.entries(headers)) {
17
17
  if (key.startsWith(':') || value === undefined)
18
18
  continue;
@@ -25,12 +25,12 @@ export class GotScrapingHttpClient extends BaseHttpClient {
25
25
  }
26
26
  }
27
27
  }
28
- parseHeaders(headers) {
29
- return new Headers([...this.iterateHeaders(headers)]);
28
+ #parseHeaders(headers) {
29
+ return new Headers([...this.#iterateHeaders(headers)]);
30
30
  }
31
31
  async fetch(request, options) {
32
- const { proxyUrl, redirect } = options ?? {};
33
- if (!this.validateRequest(request)) {
32
+ const { proxyUrl, redirect, ignoreTlsErrors } = options ?? {};
33
+ if (!this.#validateRequest(request)) {
34
34
  throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
35
35
  }
36
36
  const gotResult = await gotScraping({
@@ -41,8 +41,9 @@ export class GotScrapingHttpClient extends BaseHttpClient {
41
41
  proxyUrl,
42
42
  signal: options?.signal ?? undefined,
43
43
  followRedirect: redirect === 'follow',
44
+ ...(ignoreTlsErrors ? { https: { rejectUnauthorized: false } } : {}),
44
45
  });
45
- const responseHeaders = this.parseHeaders(gotResult.headers);
46
+ const responseHeaders = this.#parseHeaders(gotResult.headers);
46
47
  return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
47
48
  headers: responseHeaders,
48
49
  status: gotResult.statusCode,
@@ -51,4 +52,3 @@ export class GotScrapingHttpClient extends BaseHttpClient {
51
52
  });
52
53
  }
53
54
  }
54
- //# sourceMappingURL=index.js.map
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/got-scraping-client",
3
- "version": "4.0.0-beta.24",
3
+ "version": "4.0.0-beta.241",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -38,7 +38,7 @@
38
38
  },
39
39
  "homepage": "https://crawlee.dev",
40
40
  "scripts": {
41
- "build": "yarn clean && yarn compile && yarn copy",
41
+ "build": "pnpm clean && pnpm compile && pnpm copy",
42
42
  "clean": "rimraf ./dist",
43
43
  "compile": "tsc -p tsconfig.build.json",
44
44
  "copy": "tsx ../../scripts/copy.ts"
@@ -47,8 +47,8 @@
47
47
  "access": "public"
48
48
  },
49
49
  "dependencies": {
50
- "@crawlee/http-client": "4.0.0-beta.24",
51
- "got-scraping": "^4.1.3"
50
+ "@crawlee/http-client": "4.0.0-beta.241",
51
+ "got-scraping": "^4.2.1"
52
52
  },
53
53
  "lerna": {
54
54
  "command": {
@@ -57,5 +57,5 @@
57
57
  }
58
58
  }
59
59
  },
60
- "gitHead": "6feca12eeacad948d29b71ed1eeb77ac611e2beb"
60
+ "gitHead": "1c337d1ef50c3e28e25e2ae40bc38bb6de67c37e"
61
61
  }
package/index.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,KAAK,kBAAkB,EAAmB,MAAM,sBAAsB,CAAC;AAGhG;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,cAAc;IACrD;;;OAGG;IACH,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAE,cAAc;IAavB,OAAO,CAAC,YAAY;IAIL,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;CA0BxG"}
package/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,cAAc,EAA2B,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,WAAW,EAAgB,MAAM,cAAc,CAAC;AAEzD;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,cAAc;IACrD;;;OAGG;IACK,eAAe,CACnB,OAAgB;QAEhB,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;IAC7D,CAAC;IAEO,CAAC,cAAc,CACnB,OAAsD;QAEtD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS;gBAAE,SAAS;YACzD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,KAAK;oBAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,OAAsD;QACvE,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEQ,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,OAA0C;QAC7E,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE7C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,OAAO,CAAC,GAAI;YACjB,MAAM,EAAE,OAAO,CAAC,MAA2B;YAC3C,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACtD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YACtE,QAAQ;YACR,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,SAAS;YACpC,cAAc,EAAE,QAAQ,KAAK,QAAQ;SACxC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE7D,OAAO,IAAI,eAAe,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;YAC1D,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,SAAS,CAAC,UAAU;YAC5B,UAAU,EAAE,SAAS,CAAC,aAAa,IAAI,EAAE;YACzC,GAAG,EAAE,SAAS,CAAC,GAAG;SACrB,CAAC,CAAC;IACP,CAAC;CACJ"}