@crawlee/http 4.0.0-beta.257 → 4.0.0-beta.259

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,12 +2,12 @@ import { Readable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
4
4
  import { ResponseWithUrl } from '@crawlee/http-client';
5
- import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
5
+ import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
6
6
  import { isTypeValid, parse as parseContentType } from 'content-type';
7
7
  import iconv from 'iconv-lite';
8
8
  import { z } from 'zod';
9
9
  import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
10
- import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
10
+ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
11
11
  /**
12
12
  * Default mime types, which HttpScraper supports.
13
13
  */
@@ -1,8 +1,3 @@
1
- import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
- /**
3
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
- */
5
- export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
6
1
  /**
7
2
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
8
3
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
@@ -1,44 +1,9 @@
1
1
  import { extname } from 'node:path';
2
- import { Readable } from 'node:stream';
3
- import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
2
+ import { parseArgument } from '@crawlee/utils/internal';
4
3
  import { isTypeValid, parse as parseContentType } from 'content-type';
5
4
  import mime from 'mime-types';
6
5
  import { z } from 'zod';
7
6
  const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
8
- /**
9
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
10
- */
11
- export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
12
- const url = new URL(request.url);
13
- const headers = new Headers(request.headers);
14
- applySearchParams(url, searchParams);
15
- if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
16
- throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
17
- }
18
- const body = (() => {
19
- if (form !== undefined) {
20
- return Readable.from(new URLSearchParams(form).toString());
21
- }
22
- if (json !== undefined) {
23
- return Readable.from(JSON.stringify(json));
24
- }
25
- if (request.body !== undefined) {
26
- return Readable.from(request.body);
27
- }
28
- return undefined;
29
- })();
30
- if (form !== undefined && !headers.has('content-type')) {
31
- headers.set('content-type', 'application/x-www-form-urlencoded');
32
- }
33
- if (json !== undefined && !headers.has('content-type')) {
34
- headers.set('content-type', 'application/json');
35
- }
36
- if (username !== undefined || password !== undefined) {
37
- const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
38
- headers.set('authorization', `Basic ${encodedAuth}`);
39
- }
40
- return { ...request, body, url, headers };
41
- }
42
7
  /**
43
8
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
44
9
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.257",
3
+ "version": "4.0.0-beta.259",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.257",
52
- "@crawlee/http-client": "4.0.0-beta.257",
53
- "@crawlee/types": "4.0.0-beta.257",
54
- "@crawlee/utils": "4.0.0-beta.257",
51
+ "@crawlee/basic": "4.0.0-beta.259",
52
+ "@crawlee/http-client": "4.0.0-beta.259",
53
+ "@crawlee/types": "4.0.0-beta.259",
54
+ "@crawlee/utils": "4.0.0-beta.259",
55
55
  "cheerio": "^1.2.0",
56
56
  "content-type": "^3.1.1",
57
57
  "domhandler": "^6.0.1",
@@ -68,5 +68,5 @@
68
68
  }
69
69
  }
70
70
  },
71
- "gitHead": "d8f0dc1eb055b27aa07827a505c8f5a0aaaa5abd"
71
+ "gitHead": "8b6c21145fd782c4618503b633fd45f6ce8e2385"
72
72
  }